refs.bib

-- INTRODUCTION

@book{veith2017universal,
  title={Universal smart grid agent for distributed power generation management},
  author={Veith, Eric},
  year={2017},
  publisher={Logos Verlag Berlin}
}
@inproceedings{frost2020robust,
  title={Robust and deterministic scheduling of power grid actors},
  author={Frost, Emilie and Veith, Eric MSP and Fischer, Lars},
  booktitle={2020 7th International Conference on Control, Decision and Information Technologies (CoDIT)},
  volume={1},
  pages={100--105},
  year={2020},
  organization={IEEE}
}
@article{hinrichs2017distributed,
  title={A distributed combinatorial optimisation heuristic for the scheduling of energy resources represented by self-interested agents},
  author={Hinrichs, Christian and Sonnenschein, Michael},
  journal={International Journal of Bio-Inspired Computation},
  volume={10},
  number={2},
  pages={69--78},
  year={2017},
  publisher={Inderscience Publishers (IEL)}
}
@article{mahela2020comprehensive,
  title={Comprehensive overview of multi-agent systems for controlling smart grids},
  author={Mahela, Om Prakash and Khosravy, Mahdi and Gupta, Neeraj and Khan, Baseem and Alhelou, Hassan Haes and Mahla, Rajendra and Patel, Nilesh and Siano, Pierluigi},
  journal={CSEE Journal of Power and Energy Systems},
  volume={8},
  number={1},
  pages={115--131},
  year={2020},
  publisher={CSEE}
}
@article{holly2020flexibility,
  title={Flexibility management and provision of balancing services with battery-electric automated guided vehicles in the Hamburg container terminal Altenwerder},
  author={Holly, Stefanie and Nie{\ss}e, Astrid and Tr{\"o}schel, Martin and Hammer, Lasse and Franzius, Christoph and Dmitriyev, Viktor and Dorfner, Johannes and Veith, Eric MSP and Harnischmacher, Christine and Greve, Maike and others},
  journal={Energy Informatics},
  volume={3},
  pages={1--20},
  year={2020},
  publisher={Springer}
}
@article{hamilton2016lights,
  title={When the lights went out: Ukraine cybersecurity threat briefing},
  author={Hamilton, Booz Allen},
  journal={http://www. boozallen. com/content/dam/boozallen/documents/2016/09/ukraine-report-when-the-lights-wentout. pdf, checked on},
  volume={12},
  pages={20},
  year={2016}
}
@article{aflaki2021hybrid,
  title={A hybrid framework for detecting and eliminating cyber-attacks in power grids},
  author={Aflaki, Arshia and Gitizadeh, Mohsen and Razavi-Far, Roozbeh and Palade, Vasile and Ghasemi, Ali Akbar},
  journal={Energies},
  volume={14},
  number={18},
  pages={5823},
  year={2021},
  publisher={MDPI}
}
@article{wolgast2021towards,
  title={Towards reinforcement learning for vulnerability analysis in power-economic systems},
  author={Wolgast, Thomas and Veith, Eric MSP and Nie{\ss}e, Astrid},
  journal={Energy Informatics},
  volume={4},
  pages={1--20},
  year={2021},
  publisher={Springer}
}
@article{veith2022learning,
  title={Learning to Attack Powergrids with DERs},
  author={Veith, Eric and Wenninghoff, Nils and Balduin, Stephan and Wolgast, Thomas and Lehnhoff, Sebastian},
  journal={arXiv preprint arXiv:2204.11352},
  year={2022}
}
@book{hanseth2007risk,
  title={Risk, complexity and ICT},
  author={Hanseth, Ole and Ciborra, Claudio},
  year={2007},
  publisher={Edward Elgar Publishing}
}
@inproceedings{diao2019autonomous,
  title={Autonomous voltage control for grid operation using deep reinforcement learning},
  author={Diao, Ruisheng and Wang, Zhiwei and Shi, Di and Chang, Qianyun and Duan, Jiajun and Zhang, Xiaohu},
  booktitle={2019 IEEE Power \& Energy Society General Meeting (PESGM)},
  pages={1--5},
  year={2019},
  organization={IEEE}
}
@article{fischer2018adversarial,
  title={Adversarial resilience learning-towards systemic vulnerability analysis for large and complex systems},
  author={Fischer, Lars and Memmen, Jan-Menno and Veith, Eric and Tr{\"o}schel, Martin},
  journal={arXiv preprint arXiv:1811.06447},
  year={2018}
}
@inproceedings{veith2019analyzing,
  title={Analyzing cyber-physical systems from the perspective of artificial intelligence},
  author={Veith, Eric MSP and Fischer, Lars and Tr{\"o}schel, Martin and Nie{\ss}e, Astrid},
  booktitle={Proceedings of the 2019 International Conference on Artificial Intelligence, Robotics and Control},
  pages={85--95},
  year={2019}
}
@article{schrittwieser2020mastering,
  title={Mastering atari, go, chess and shogi by planning with a learned model},
  author={Schrittwieser, Julian and Antonoglou, Ioannis and Hubert, Thomas and Simonyan, Karen and Sifre, Laurent and Schmitt, Simon and Guez, Arthur and Lockhart, Edward and Hassabis, Demis and Graepel, Thore and others},
  journal={Nature},
  volume={588},
  number={7839},
  pages={604--609},
  year={2020},
  publisher={Nature Publishing Group}
}
@inproceedings{fujimoto2018addressing,
  title={Addressing function approximation error in actor-critic methods},
  author={Fujimoto, Scott and Hoof, Herke and Meger, David},
  booktitle={International conference on machine learning},
  pages={1587--1596},
  year={2018},
  organization={PMLR}
}
@article{schulman2017proximal,
  title={Proximal policy optimization algorithms},
  author={Schulman, John and Wolski, Filip and Dhariwal, Prafulla and Radford, Alec and Klimov, Oleg},
  journal={arXiv preprint arXiv:1707.06347},
  year={2017}
}
@article{haarnoja2018soft,
  title={Soft actor-critic algorithms and applications},
  author={Haarnoja, Tuomas and Zhou, Aurick and Hartikainen, Kristian and Tucker, George and Ha, Sehoon and Tan, Jie and Kumar, Vikash and Zhu, Henry and Gupta, Abhishek and Abbeel, Pieter and others},
  journal={arXiv preprint arXiv:1812.05905},
  year={2018}
}

-- AGENTS
@article{wooldridge1999intelligent,
  title={Intelligent agents},
  author={Wooldridge, Michael},
  journal={Multiagent systems: A modern approach to distributed artificial intelligence},
  volume={1},
  pages={27--73},
  year={1999}
}
@book{wooldridge2009introduction,
  title={An introduction to multiagent systems},
  author={Wooldridge, Michael},
  year={2009},
  publisher={John wiley \& sons}
}
@article{balaji2010introduction,
  title={An introduction to multi-agent systems},
  author={Balaji, Parasumanna Gokulan and Srinivasan, Dipti},
  journal={Innovations in multi-agent systems and applications-1},
  pages={1--27},
  year={2010},
  publisher={Springer}
}
@article{dorri2018multi,
  title={Multi-agent systems: A survey},
  author={Dorri, Ali and Kanhere, Salil S and Jurdak, Raja},
  journal={Ieee Access},
  volume={6},
  pages={28573--28593},
  year={2018},
  publisher={IEEE}
}

-- Powergrid
@article{amin2008electric,
  title={The electric power grid: Today and tomorrow},
  author={Amin, Massoud and Stringer, John},
  journal={MRS bulletin},
  volume={33},
  number={4},
  pages={399--407},
  year={2008},
  publisher={Cambridge University Press}
}

-- MACHINE LEARNING
@article{mccarthy2007artificial,
  title={What is artificial intelligence},
  author={McCarthy, John and others},
  year={2007},
  publisher={Stanford University}
}
@book{mitchell1997machine,
  title={Machine learning},
  author={Mitchell, Tom M and Mitchell, Tom M},
  volume={1},
  number={9},
  year={1997},
  publisher={McGraw-hill New York}
}
@book{goodfellow2016deep,
  title={Deep learning},
  author={Goodfellow, Ian and Bengio, Yoshua and Courville, Aaron},
  year={2016},
  publisher={MIT press}
}

-- DRL
@picture{network_image,
   author={Glossar.ca},
   title={Artificial neural network with layer coloring},
   year={2013},
   urlseen={27.05.2024},
   url={"https://en.wikipedia.org/wiki/File:Colored_neural_network.svg"},
   note={"File:Colored neural network.svg"},
}
@book{anderson1995introduction,
  title={An introduction to neural networks},
  author={Anderson, James A},
  year={1995},
  publisher={MIT press}
}
@inproceedings{barto2004intrinsically,
  title={Intrinsically motivated learning of hierarchical collections of skills},
  author={Barto, Andrew G and Singh, Satinder and Chentanez, Nuttapong and others},
  booktitle={Proceedings of the 3rd International Conference on Development and Learning},
  volume={112},
  pages={19},
  year={2004},
  organization={Citeseer}
}
@inproceedings{puiutta2020explainable,
  title={Explainable reinforcement learning: A survey},
  author={Puiutta, Erika and Veith, Eric MSP},
  booktitle={International cross-domain conference for machine learning and knowledge extraction},
  pages={77--95},
  year={2020},
  organization={Springer}
}
@article{silver2016mastering,
  title={Mastering the game of Go with deep neural networks and tree search},
  author={Silver, David and Huang, Aja and Maddison, Chris J and Guez, Arthur and Sifre, Laurent and Van Den Driessche, George and Schrittwieser, Julian and Antonoglou, Ioannis and Panneershelvam, Veda and Lanctot, Marc and others},
  journal={nature},
  volume={529},
  number={7587},
  pages={484--489},
  year={2016},
  publisher={Nature Publishing Group}
}
@article{arulkumaran2017deep,
  title={Deep reinforcement learning: A brief survey},
  author={Arulkumaran, Kai and Deisenroth, Marc Peter and Brundage, Miles and Bharath, Anil Anthony},
  journal={IEEE Signal Processing Magazine},
  volume={34},
  number={6},
  pages={26--38},
  year={2017},
  publisher={IEEE}
}
-- ARL
@techreport{uther1997adversarial,
  title={Adversarial reinforcement learning},
  author={Uther, William and Veloso, Manuela},
  year={1997},
  institution={Tech. rep., Carnegie Mellon University. Unpublished}
}
-- EA
@book{sampson1976adaptation,
  title={Adaptation in natural and artificial systems (John H. Holland)},
  author={Sampson, Jeffrey R},
  year={1976},
  publisher={Society for Industrial and Applied Mathematics}
}
@article{whitley1996evaluating,
  title={Evaluating evolutionary algorithms},
  author={Whitley, Darrell and Rana, Soraya and Dzubera, John and Mathias, Keith E},
  journal={Artificial intelligence},
  volume={85},
  number={1-2},
  pages={245--276},
  year={1996},
  publisher={Elsevier}
}
-- HYPERPARAMETER OPTIMISATION
@article{feurer2019hyperparameter,
  title={Hyperparameter optimization},
  author={Feurer, Matthias and Hutter, Frank},
  journal={Automated machine learning: Methods, systems, challenges},
  pages={3--33},
  year={2019},
  publisher={Springer International Publishing}
}
-- NETWORK OPTIMISATION
@inproceedings{ALF,
author = {Krauss, Rune and Merten, Marcel and Bockholt, Mirco and Drechsler, Rolf},
title = {ALF: a fitness-based artificial life form for evolving large-scale neural networks},
year = {2021},
isbn = {9781450383516},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
url = {https://doi.org/10.1145/3449726.3459545},
doi = {10.1145/3449726.3459545},
abstract = {Topology and Weight Evolving Artificial Neural Network (TWEANN) is a concept to find the topology and weights of Artificial Neural Networks (ANNs) using genetic algorithms. However, a well-known downside is that TWEANN algorithms often evolve inefficient large ANNs for large-scale problems and require long runtimes.To address this issue, we propose a new TWEANN algorithm called Artificial Life Form (ALF) with the following technical advancements: (1) speciation via structural and semantic similarity to form better candidate solutions, (2) dynamic adaptation of the observed candidate solutions for better convergence properties, and (3) integration of solution quality into genetic reproduction to increase the probability of optimization success. Experiments on large-scale problems confirm that these approaches allow effective solving of these problems and lead to efficient evolved ANNs.},
booktitle = {Proceedings of the Genetic and Evolutionary Computation Conference Companion},
pages = {225–226},
numpages = {2},
keywords = {neuroevolution, neural networks, genetic algorithms},
location = {Lille, France},
series = {GECCO '21}
}
@ARTICLE{NEAT,
  author={Stanley, Kenneth O. and Miikkulainen, Risto},
  journal={Evolutionary Computation}, 
  title={Evolving Neural Networks through Augmenting Topologies}, 
  year={2002},
  volume={10},
  number={2},
  pages={99-127},
  keywords={Genetic algorithms;neural networks;neuroevolution;network topologies;speciation;competing conventions},
  doi={10.1162/106365602320169811}
}
@article{probst2019tunability,
  title={Tunability: Importance of hyperparameters of machine learning algorithms},
  author={Probst, Philipp and Boulesteix, Anne-Laure and Bischl, Bernd},
  journal={Journal of Machine Learning Research},
  volume={20},
  number={53},
  pages={1--32},
  year={2019}
}
@article{white2023neural,
  title={Neural architecture search: Insights from 1000 papers},
  author={White, Colin and Safari, Mahmoud and Sukthanker, Rhea and Ru, Binxin and Elsken, Thomas and Zela, Arber and Dey, Debadeepta and Hutter, Frank},
  journal={arXiv preprint arXiv:2301.08727},
  year={2023}
}
@article{elsken2019neural,
  title={Neural architecture search: A survey},
  author={Elsken, Thomas and Metzen, Jan Hendrik and Hutter, Frank},
  journal={Journal of Machine Learning Research},
  volume={20},
  number={55},
  pages={1--21},
  year={2019}
}
-- NAS_IN_DRL
@article{fu2020auto,
  title={Auto-agent-distiller: Towards efficient deep reinforcement learning agents via neural architecture search},
  author={Fu, Yonggan and Yu, Zhongzhi and Zhang, Yongan and Lin, Yingyan},
  journal={arXiv preprint arXiv:2012.13091},
  year={2020}
}
@inproceedings{mazyavkina2021optimizing,
  title={Optimizing the neural architecture of reinforcement learning agents},
  author={Mazyavkina, Nina and Moustafa, S and Trofimov, Ilya and Burnaev, Evgeny},
  booktitle={Intelligent Computing: Proceedings of the 2021 Computing Conference, Volume 2},
  pages={591--606},
  year={2021},
  organization={Springer}
}
@article{liu2021survey,
  title={A survey on evolutionary neural architecture search},
  author={Liu, Yuqiao and Sun, Yanan and Xue, Bing and Zhang, Mengjie and Yen, Gary G and Tan, Kay Chen},
  journal={IEEE transactions on neural networks and learning systems},
  volume={34},
  number={2},
  pages={550--570},
  year={2021},
  publisher={IEEE}
}
-- EA_IN_DRL
@article{parker2022automated,
  title={Automated reinforcement learning (autorl): A survey and open problems},
  author={Parker-Holder, Jack and Rajan, Raghu and Song, Xingyou and Biedenkapp, Andr{\'e} and Miao, Yingjie and Eimer, Theresa and Zhang, Baohe and Nguyen, Vu and Calandra, Roberto and Faust, Aleksandra and others},
  journal={Journal of Artificial Intelligence Research},
  volume={74},
  pages={517--568},
  year={2022}
}
@inproceedings{stanley2002efficient,
  title={Efficient evolution of neural network topologies},
  author={Stanley, Kenneth O and Miikkulainen, Risto},
  booktitle={Proceedings of the 2002 Congress on Evolutionary Computation. CEC'02 (Cat. No. 02TH8600)},
  volume={2},
  pages={1757--1762},
  year={2002},
  organization={IEEE}
}
@article{stanley2009hypercube,
  title={A hypercube-based encoding for evolving large-scale neural networks},
  author={Stanley, Kenneth O and D'Ambrosio, David B and Gauci, Jason},
  journal={Artificial life},
  volume={15},
  number={2},
  pages={185--212},
  year={2009},
  publisher={MIT Press One Rogers Street, Cambridge, MA 02142-1209, USA journals-info~…}
}
@article{mirjalili2019genetic,
  title={Genetic algorithm},
  author={Mirjalili, Seyedali and Mirjalili, Seyedali},
  journal={Evolutionary algorithms and neural networks: theory and applications},
  pages={43--55},
  year={2019},
  publisher={Springer}
}
@article{mirjalili2016whale,
  title={The whale optimization algorithm},
  author={Mirjalili, Seyedali and Lewis, Andrew},
  journal={Advances in engineering software},
  volume={95},
  pages={51--67},
  year={2016},
  publisher={Elsevier}
}
@inproceedings{elfwing2018online,
  title={Online meta-learning by parallel algorithm competition},
  author={Elfwing, Stefan and Uchibe, Eiji and Doya, Kenji},
  booktitle={Proceedings of the Genetic and Evolutionary Computation Conference},
  pages={426--433},
  year={2018}
}
@article{jaderberg2017population,
  title={Population based training of neural networks},
  author={Jaderberg, Max and Dalibard, Valentin and Osindero, Simon and Czarnecki, Wojciech M and Donahue, Jeff and Razavi, Ali and Vinyals, Oriol and Green, Tim and Dunning, Iain and Simonyan, Karen and others},
  journal={arXiv preprint arXiv:1711.09846},
  year={2017}
}
@article{
bai2023evolutionary,
author = {Hui Bai  and Ran Cheng  and Yaochu Jin },
title = {Evolutionary Reinforcement Learning: A Survey},
journal = {Intelligent Computing},
volume = {2},
number = {},
pages = {0025},
year = {2023},
doi = {10.34133/icomputing.0025},
URL = {https://spj.science.org/doi/abs/10.34133/icomputing.0025},
eprint = {https://spj.science.org/doi/pdf/10.34133/icomputing.0025},
abstract = {Reinforcement learning (RL) is a machine learning approach that trains agents to maximize cumulative rewards through interactions with environments. The integration of RL with deep learning has recently resulted in impressive achievements in a wide range of challenging tasks, including board games, arcade games, and robot control. Despite these successes, several critical challenges remain, such as brittle convergence properties caused by sensitive hyperparameters, difficulties in temporal credit assignment with long time horizons and sparse rewards, a lack of diverse exploration, particularly in continuous search space scenarios, challenges in credit assignment in multi-agent RL, and conflicting objectives for rewards. Evolutionary computation (EC), which maintains a population of learning agents, has demonstrated promising performance in addressing these limitations. This article presents a comprehensive survey of state-of-the-art methods for integrating EC into RL, referred to as evolutionary reinforcement learning (EvoRL). We categorize EvoRL methods according to key research areas in RL, including hyperparameter optimization, policy search, exploration, reward shaping, meta-RL, and multi-objective RL. We then discuss future research directions in terms of efficient methods, benchmarks, and scalable platforms. This survey serves as a resource for researchers and practitioners interested in the field of EvoRL, highlighting the important challenges and opportunities for future research. With the help of this survey, researchers and practitioners can develop more efficient methods and tailored benchmarks for EvoRL, further advancing this promising cross-disciplinary research field.}
}
@article{dalibard2021faster,
  title={Faster improvement rate population based training},
  author={Dalibard, Valentin and Jaderberg, Max},
  journal={arXiv preprint arXiv:2109.13800},
  year={2021}
}
@article{franke2020sample,
  title={Sample-efficient automated deep reinforcement learning},
  author={Franke, J{\"o}rg KH and K{\"o}hler, Gregor and Biedenkapp, Andr{\'e} and Hutter, Frank},
  journal={arXiv preprint arXiv:2009.01555},
  year={2020}
}
@article{cui2018evolutionary,
  title={Evolutionary stochastic gradient descent for optimization of deep neural networks},
  author={Cui, Xiaodong and Zhang, Wei and T5{\"u}ske, Zolt{\'a}n and Picheny, Michael},
  journal={Advances in neural information processing systems},
  volume={31},
  year={2018}
}
@article{Sigaud,
author = {Sigaud, Olivier},
title = {Combining Evolution and Deep Reinforcement Learning for Policy Search: A Survey},
year = {2023},
issue_date = {September 2023},
publisher = {Association for Computing Machinery},
address = {New York, NY, USA},
volume = {3},
number = {3},
url = {https://doi.org/10.1145/3569096},
doi = {10.1145/3569096},
abstract = {Deep neuroevolution and deep Reinforcement Learning have received a lot of attention over the past few years. Some works have compared them, highlighting their pros and cons, but an emerging trend combines them so as to benefit from the best of both worlds. In this article, we provide a survey of this emerging trend by organizing the literature into related groups of works and casting all the existing combinations in each group into a generic framework. We systematically cover all easily available papers irrespective of their publication status, focusing on the combination mechanisms rather than on the experimental results. In total, we cover 45 algorithms more recent than 2017. We hope this effort will favor the growth of the domain by facilitating the understanding of the relationships between the methods, leading to deeper analyses, outlining missing useful comparisons and suggesting new combinations of mechanisms.},
journal = {ACM Trans. Evol. Learn. Optim.},
month = {sep},
articleno = {10},
numpages = {20},
keywords = {Evolutionary algorithms}
}

-- PALAESTRAI
@inproceedings{veith2023palaestrai,
  title={palaestrAI: A training ground for autonomous agents},
  author={Veith, Eric and Balduin, Stephan and Wenninghoff, Nils and Wolgast, Thomas and Baumann, Marcel and Winkler, David and Hammer, Lasse and Salman, Alya and Schulz, Marvin and Raeiszadeh, Amin and others},
  booktitle={Proceedings of the 37th annual European Simulation and Modelling Conference, EUROSIS},
  year={2023}
}
@inproceedings{ofenloch2022mosaik,
  title={Mosaik 3.0: Combining time-stepped and discrete event simulation},
  author={Ofenloch, Annika and Schwarz, Jan S{\"o}ren and Tolk, Deborah and Brandt, Tobias and Eilers, Reef and Ramirez, Rebeca and Raub, Thomas and Lehnhoff, Sebastian},
  booktitle={2022 Open Source Modelling and Simulation of Energy Systems (OSMSES)},
  pages={1--5},
  year={2022},
  organization={IEEE}
}
@article{hintjens2011omq,
  title={{\O}mq-the guide},
  author={Hintjens, Pieter and others},
  journal={Online: http://zguide. zeromq. org/page: all, Accessed on},
  volume={23},
  year={2011}
}
@article{gorski2022uml,
  title={UML profile for messaging patterns in service-oriented architecture, microservices, and internet of things},
  author={G{\'o}rski, Tomasz},
  journal={Applied Sciences},
  volume={12},
  number={24},
  pages={12790},
  year={2022},
  publisher={MDPI}
}
@inproceedings{baker2019emergent,
  title={Emergent tool use from multi-agent autocurricula},
  author={Baker, Bowen and Kanitscheider, Ingmar and Markov, Todor and Wu, Yi and Powell, Glenn and McGrew, Bob and Mordatch, Igor},
  booktitle={International conference on learning representations},
  year={2019}
}


-- BO
@article{shahriari2015taking,
  title={Taking the human out of the loop: A review of Bayesian optimization},
  author={Shahriari, Bobak and Swersky, Kevin and Wang, Ziyu and Adams, Ryan P and De Freitas, Nando},
  journal={Proceedings of the IEEE},
  volume={104},
  number={1},
  pages={148--175},
  year={2015},
  publisher={IEEE}
}

-- Experiment
@INPROCEEDINGS{rudion2006cigre,
  author={Rudion, K. and Orths, A. and Styczynski, Z.A. and Strunz, K.},
  booktitle={2006 IEEE Power Engineering Society General Meeting}, 
  title={Design of benchmark of medium voltage distribution network for investigation of DG integration}, 
  year={2006},
  volume={},
  number={},
  pages={6 pp.-},
  keywords={Medium voltage;Power system modeling;Power system simulation;Power generation;Distributed control;Nuclear power generation;Energy management;Power system analysis computing;Load flow;Fuels;Benchmarking;distributed generation;distributed energy management system;distribution network;power system modeling;power system simulation},
  doi={10.1109/PES.2006.1709447}
}

-- Basics neu
@article{alymov2024monitoring,
  title={Monitoring energy flows for efficient electricity control in low-voltage smart grids},
  author={Alymov, Ivan and Averbukh, Moshe},
  journal={Energies},
  volume={17},
  number={9},
  pages={2123},
  year={2024},
  publisher={MDPI}
}
@article{vilela2021analysis,
  title={Analysis and Adequacy Methodology for Voltage Violations in Distribution Power Grid},
  author={Vilela Junior, Wagner A and Coimbra, Antonio P and Wainer, Gabriel A and Caetano Neto, Joao and Cararo, Jose AG and Reis, Marcio RC and Santos, Paulo V and Calixto, Wesley P},
  journal={Energies},
  volume={14},
  number={14},
  pages={4373},
  year={2021},
  publisher={MDPI}
}
@article{albu2016syncretic,
  title={Syncretic use of smart meters for power quality monitoring in emerging networks},
  author={Albu, Mihaela M and S{\u{a}}nduleac, Mihai and St{\u{a}}nescu, Carmen},
  journal={IEEE Transactions on Smart Grid},
  volume={8},
  number={1},
  pages={485--492},
  year={2016},
  publisher={IEEE}
}
@article{veithpower,
  title={The Power Grid in a Nutshell},
  author={Veith, Eric MSP}
}

@inproceedings{clauset2011brief,
  title={A brief primer on probability distributions},
  author={Clauset, Aaron},
  booktitle={Santa Fe Institute},
  year={2011}
}
@inproceedings{bottou2010large,
  title={Large-scale machine learning with stochastic gradient descent},
  author={Bottou, L{\'e}on},
  booktitle={Proceedings of COMPSTAT'2010: 19th International Conference on Computational StatisticsParis France, August 22-27, 2010 Keynote, Invited and Contributed Papers},
  pages={177--186},
  year={2010},
  organization={Springer}
}
@article{duchi2011adaptive,
  title={Adaptive subgradient methods for online learning and stochastic optimization.},
  author={Duchi, John and Hazan, Elad and Singer, Yoram},
  journal={Journal of machine learning research},
  volume={12},
  number={7},
  year={2011}
}
@article{tieleman2012lecture,
  title={Lecture 6.5-rmsprop: Divide the gradient by a running average of its recent magnitude},
  author={Tieleman, Tijmen},
  journal={COURSERA: Neural networks for machine learning},
  volume={4},
  number={2},
  pages={26},
  year={2012}
}
@article{kingma2014adam,
  title={Adam: A method for stochastic optimization},
  author={Kingma, Diederik P and Ba, Jimmy},
  journal={arXiv preprint arXiv:1412.6980},
  year={2014}
}
@article{loshchilov2017decoupled,
  title={Decoupled weight decay regularization},
  author={Loshchilov, Ilya and Hutter, Frank},
  journal={arXiv preprint arXiv:1711.05101},
  year={2017}
}
@article{tran2019convergence,
  title={On the convergence proof of amsgrad and a new version},
  author={Tran, Phuong Thi and others},
  journal={IEEE Access},
  volume={7},
  pages={61706--61716},
  year={2019},
  publisher={IEEE}
}
@book{russell2016artificial,
  title={Artificial intelligence: a modern approach},
  author={Russell, Stuart J and Norvig, Peter},
  year={2016},
  publisher={pearson}
}
@article{hornik1989multilayer,
  title={Multilayer feedforward networks are universal approximators},
  author={Hornik, Kurt and Stinchcombe, Maxwell and White, Halbert},
  journal={Neural networks},
  volume={2},
  number={5},
  pages={359--366},
  year={1989},
  publisher={Elsevier}
}
@article{heaton2011introduction,
  title={Introduction to the Math of Neural Networks (Beta-1)},
  author={Heaton, Jeff},
  journal={Heaton Research Inc},
  year={2011}
}
@article{bellman1954theory,
  title={The theory of dynamic programming},
  author={Bellman, Richard},
  journal={Bulletin of the American Mathematical Society},
  volume={60},
  number={6},
  pages={503--515},
  year={1954}
}
@article{watkins1992q,
  title={Q-learning},
  author={Watkins, Christopher JCH and Dayan, Peter},
  journal={Machine learning},
  volume={8},
  pages={279--292},
  year={1992},
  publisher={Springer}
}
@article{gu2017interpolated,
  title={Interpolated policy gradient: Merging on-policy and off-policy gradient estimation for deep reinforcement learning},
  author={Gu, Shixiang Shane and Lillicrap, Timothy and Turner, Richard E and Ghahramani, Zoubin and Sch{\"o}lkopf, Bernhard and Levine, Sergey},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}
@article{kober2013reinforcement,
  title={Reinforcement learning in robotics: A survey},
  author={Kober, Jens and Bagnell, J Andrew and Peters, Jan},
  journal={The International Journal of Robotics Research},
  volume={32},
  number={11},
  pages={1238--1274},
  year={2013},
  publisher={SAGE Publications Sage UK: London, England}
}
@article{hambly2023recent,
  title={Recent advances in reinforcement learning in finance},
  author={Hambly, Ben and Xu, Renyuan and Yang, Huining},
  journal={Mathematical Finance},
  volume={33},
  number={3},
  pages={437--503},
  year={2023},
  publisher={Wiley Online Library}
}
@article{chen2021powernet,
  title={Powernet: Multi-agent deep reinforcement learning for scalable powergrid control},
  author={Chen, Dong and Chen, Kaian and Li, Zhaojian and Chu, Tianshu and Yao, Rui and Qiu, Feng and Lin, Kaixiang},
  journal={IEEE Transactions on Power Systems},
  volume={37},
  number={2},
  pages={1007--1017},
  year={2021},
  publisher={IEEE}
}
@article{perera2021applications,
  title={Applications of reinforcement learning in energy systems},
  author={Perera, ATD and Kamalaruban, Parameswaran},
  journal={Renewable and Sustainable Energy Reviews},
  volume={137},
  pages={110618},
  year={2021},
  publisher={Elsevier}
}
@article{hao2022adversarial,
  title={Adversarial attacks on deep learning models in smart grids},
  author={Hao, Jingbo and Tao, Yang},
  journal={Energy Reports},
  volume={8},
  pages={123--129},
  year={2022},
  publisher={Elsevier}
}
@misc{panimproving,
  title={Improving robustness of reinforcement learning for power system control with adversarial training (2021)},
  author={Pan, A and Lee, Y and Zhang, H and Chen, Y and Shi, Y}
}
@article{mnih2015human,
  title={Human-level control through deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Rusu, Andrei A and Veness, Joel and Bellemare, Marc G and Graves, Alex and Riedmiller, Martin and Fidjeland, Andreas K and Ostrovski, Georg and others},
  journal={nature},
  volume={518},
  number={7540},
  pages={529--533},
  year={2015},
  publisher={Nature Publishing Group}
}
@article{lillicrap2015continuous,
  title={Continuous control with deep reinforcement learning},
  author={Lillicrap, Timothy P and Hunt, Jonathan J and Pritzel, Alexander and Heess, Nicolas and Erez, Tom and Tassa, Yuval and Silver, David and Wierstra, Daan},
  journal={arXiv preprint arXiv:1509.02971},
  year={2015}
}
@article{peters2010policy,
  title={Policy gradient methods},
  author={Peters, Jan},
  journal={Scholarpedia},
  volume={5},
  number={11},
  pages={3698},
  year={2010}
}
@inproceedings{haarnoja2018soft2,
  title={Soft actor-critic: Off-policy maximum entropy deep reinforcement learning with a stochastic actor},
  author={Haarnoja, Tuomas and Zhou, Aurick and Abbeel, Pieter and Levine, Sergey},
  booktitle={International conference on machine learning},
  pages={1861--1870},
  year={2018},
  organization={Pmlr}
}
@misc{oreillyNeuroevolutionDifferent,
	author = {Kenneth O. Stanley},
	title = {{N}euroevolution: {A} different kind of deep learning -- oreilly.com},
	howpublished = {\url{https://www.oreilly.com/radar/neuroevolution-a-different-kind-of-deep-learning/}},
	year = {2017},
	note = {[Accessed 24-03-2025]},
}
@article{zhang2015comprehensive,
  title={A comprehensive survey on particle swarm optimization algorithm and its applications},
  author={Zhang, Yudong and Wang, Shuihua and Ji, Genlin},
  journal={Mathematical problems in engineering},
  volume={2015},
  number={1},
  pages={931256},
  year={2015},
  publisher={Wiley Online Library}
}
-- STACK
@inproceedings{sehgal2019deep,
  title={Deep reinforcement learning using genetic algorithm for parameter optimization},
  author={Sehgal, Adarsh and La, Hung and Louis, Sushil and Nguyen, Hai},
  booktitle={2019 Third IEEE International Conference on Robotic Computing (IRC)},
  pages={596--601},
  year={2019},
  organization={IEEE}
}
@article{sehgal2023deep,
  title={Deep reinforcement learning for robotic manipulation tasks using a genetic algorithm-based function optimizer},
  author={Sehgal, Adarsh and Ward, Nicholas and La, Hung Manh and Louis, Sushil},
  journal={Encyclopedia with semantic computing and robotic intelligence},
  year={2023}
}
@article{grigsby2021towards,
  title={Towards automatic actor-critic solutions to continuous control},
  author={Grigsby, Jake and Yoo, Jin Yong and Qi, Yanjun},
  journal={arXiv preprint arXiv:2106.08918},
  year={2021}
}
@article{vincent2023improved,
  title={An improved hyperparameter optimization framework for AutoML systems using evolutionary algorithms},
  author={Vincent, Amala Mary and Jidesh, P},
  journal={Scientific Reports},
  volume={13},
  number={1},
  pages={4737},
  year={2023},
  publisher={Nature Publishing Group UK London}
}
@article{loshchilov2016cma,
  title={CMA-ES for hyperparameter optimization of deep neural networks},
  author={Loshchilov, Ilya and Hutter, Frank},
  journal={arXiv preprint arXiv:1604.07269},
  year={2016}
}
@article{awad2021dehb,
  title={Dehb: Evolutionary hyperband for scalable, robust and efficient hyperparameter optimization},
  author={Awad, Noor and Mallik, Neeratyoy and Hutter, Frank},
  journal={arXiv preprint arXiv:2105.09821},
  year={2021}
}
@article{moriarty1999evolutionary,
  title={Evolutionary algorithms for reinforcement learning},
  author={Moriarty, David E and Schultz, Alan C and Grefenstette, John J},
  journal={Journal of Artificial Intelligence Research},
  volume={11},
  pages={241--276},
  year={1999}
}
@article{khadka2018evolutionary,
  title={Evolutionary reinforcement learning},
  author={Khadka, Shauharda and Tumer, Kagan},
  journal={arXiv preprint arXiv:1805.07917},
  volume={223},
  year={2018}
}

@inproceedings{khadka2019collaborative,
  title={Collaborative evolutionary reinforcement learning},
  author={Khadka, Shauharda and Majumdar, Somdeb and Nassar, Tarek and Dwiel, Zach and Tumer, Evren and Miret, Santiago and Liu, Yinyin and Tumer, Kagan},
  booktitle={International conference on machine learning},
  pages={3341--3350},
  year={2019},
  organization={PMLR}
}
@article{ZHU2023126628,
title = {A survey on Evolutionary Reinforcement Learning algorithms},
journal = {Neurocomputing},
volume = {556},
pages = {126628},
year = {2023},
issn = {0925-2312},
doi = {https://doi.org/10.1016/j.neucom.2023.126628},
url = {https://www.sciencedirect.com/science/article/pii/S0925231223007518},
author = {Qingling Zhu and Xiaoqiang Wu and Qiuzhen Lin and Lijia Ma and Jianqiang Li and Zhong Ming and Jianyong Chen},
keywords = {Evolutionary algorithm, Reinforcement learning, Evolutionary reinforcement learning, Policy optimization},
abstract = {Reinforcement Learning (RL) has proven to be highly effective in various real-world applications. However, in certain scenarios, Evolutionary Algorithms (EAs) have been utilized as an alternative to RL algorithms. Recently, Evolutionary Reinforcement Learning algorithms (ERLs) have emerged as a promising solution that combines the advantages of both RL and EA. This paper presents a comprehensive survey that encompasses a majority of the studies in this exciting research area. We classify these ERLs according to the EA used in their frameworks and analyze the strengths and limitations of various EA components and combination schemes. Additionally, we conduct several experiments to evaluate the performance of some representative ERLs. By categorizing the different approaches and assessing their effectiveness, the paper can assist researchers and practitioners in selecting the most suitable method for their particular application.}
}
@article{cai2018proxylessnas,
  title={Proxylessnas: Direct neural architecture search on target task and hardware},
  author={Cai, Han and Zhu, Ligeng and Han, Song},
  journal={arXiv preprint arXiv:1812.00332},
  year={2018}
}
@article{zoph2016neural,
  title={Neural architecture search with reinforcement learning},
  author={Zoph, Barret and Le, Quoc V},
  journal={arXiv preprint arXiv:1611.01578},
  year={2016}
}
@inproceedings{movckus1975bayesian,
  title={On Bayesian methods for seeking the extremum},
  author={Mo{\v{c}}kus, Jonas},
  booktitle={Optimization Techniques IFIP Technical Conference Novosibirsk, July 1--7, 1974 6},
  pages={400--404},
  year={1975},
  organization={Springer}
}
@article{brock2017smash,
  title={Smash: one-shot model architecture search through hypernetworks},
  author={Brock, Andrew and Lim, Theodore and Ritchie, James M and Weston, Nick},
  journal={arXiv preprint arXiv:1708.05344},
  year={2017}
}
@article{lowell2011comparison,
  title={Comparison of NEAT and HyperNEAT on a Strategic Decision-Making Problem},
  author={Lowell, Jessica and Birger, Kir and Grabkovsky, Sergey},
  journal={URL: http://web. mit. edu/jessiehl/Public/aaai11/fullpaper. pdf [As of: 2017.01. 15]},
  year={2011}
}
@inproceedings{kavzoglu1999determining,
  title={Determining optimum structure for artificial neural networks},
  author={Kavzoglu, Taskin and others},
  booktitle={Proceedings of the 25th Annual Technical Conference and Exhibition of the Remote Sensing Society},
  pages={675--682},
  year={1999},
  organization={Citeseer}
}