@article{sayin2021decentralized,
  title={Decentralized Q-learning in zero-sum Markov games},
  author={Sayin, Muhammed and Zhang, Kaiqing and Leslie, David and Basar, Tamer and Ozdaglar, Asuman},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@article{harris1998rate,
  title={On the rate of convergence of continuous-time fictitious play},
  author={Harris, Christopher},
  journal={Games and Economic Behavior},
  volume={22},
  number={2},
  pages={238--259},
  year={1998},
  publisher={Elsevier}
}

@book{osborne1994course,
  title={A course in game theory},
  author={Osborne, Martin J and Rubinstein, Ariel},
  year={1994},
  publisher={MIT press}
}

@book{fudenberg1998theory,
  title={The theory of learning in games},
  author={Fudenberg, Drew and Drew, Fudenberg and Levine, David K and Levine, David K},
  volume={2},
  year={1998},
  publisher={MIT press}
}

@incollection{littman1994markov,
  title={Markov games as a framework for multi-agent reinforcement learning},
  author={Littman, Michael L},
  booktitle={Machine learning proceedings 1994},
  pages={157--163},
  year={1994},
  publisher={Elsevier}
}

@article{fudenberg1993learning,
  title={Learning mixed equilibria},
  author={Fudenberg, Drew and Kreps, David M},
  journal={Games and economic behavior},
  volume={5},
  number={3},
  pages={320--367},
  year={1993},
  publisher={Elsevier}
}

@article{hofbauer2002global,
  title={On the global convergence of stochastic fictitious play},
  author={Hofbauer, Josef and Sandholm, William H},
  journal={Econometrica},
  volume={70},
  number={6},
  pages={2265--2294},
  year={2002},
  publisher={Wiley Online Library}
}

@article{berger2005fictitious,
  title={Fictitious play in 2$\times$ n games},
  author={Berger, Ulrich},
  journal={Journal of Economic Theory},
  volume={120},
  number={2},
  pages={139--154},
  year={2005},
  publisher={Elsevier}
}

@article{monderer1996fictitious,
  title={Fictitious play property for games with identical interests},
  author={Monderer, Dov and Shapley, Lloyd S},
  journal={Journal of economic theory},
  volume={68},
  number={1},
  pages={258--265},
  year={1996},
  publisher={Elsevier}
}

@techreport{miyasawa1961convergence,
  title={On the convergence of the learning process in a 2 x 2 non-zero-sum two-person game},
  author={Miyasawa, Koichi},
  year={1961},
  institution={PRINCETON UNIV NJ}
}

@article{robinson1951iterative,
  title={An iterative method of solving a game},
  author={Robinson, Julia},
  journal={Annals of mathematics},
  pages={296--301},
  year={1951},
  publisher={JSTOR}
}

@article{brown1951iterative,
  title={Iterative solution of games by fictitious play},
  author={Brown, George W},
  journal={Act. Anal. Prod Allocation},
  volume={13},
  number={1},
  pages={374},
  year={1951}
}

@article{monderer1996potential,
  title={Potential games},
  author={Monderer, Dov and Shapley, Lloyd S},
  journal={Games and economic behavior},
  volume={14},
  number={1},
  pages={124--143},
  year={1996},
  publisher={Elsevier}
}

@article{nash1950equilibrium,
  title={Equilibrium points in n-person games},
  author={Nash Jr, John F},
  journal={Proceedings of the national academy of sciences},
  volume={36},
  number={1},
  pages={48--49},
  year={1950},
  publisher={National Acad Sciences}
}

@article{nash1951non,
  title={Non-cooperative games},
  author={Nash, John},
  journal={Annals of mathematics},
  pages={286--295},
  year={1951},
  publisher={JSTOR}
}

@book{bauso2016game,
  title={Game theory with engineering applications},
  author={Bauso, Dario},
  year={2016},
  publisher={SIAM}
}

@article{shapley1953stochastic,
  title={Stochastic games},
  author={Shapley, Lloyd S},
  journal={Proceedings of the national academy of sciences},
  volume={39},
  number={10},
  pages={1095--1100},
  year={1953},
  publisher={National Acad Sciences}
}

@book{fudenberg1998theory,
  title={The theory of learning in games},
  author={Fudenberg, Drew and Drew, Fudenberg and Levine, David K and Levine, David K},
  volume={2},
  year={1998},
  publisher={MIT press}
}

@article{leonardos2021global,
  title={Global convergence of multi-agent policy gradient in markov potential games},
  author={Leonardos, Stefanos and Overman, Will and Panageas, Ioannis and Piliouras, Georgios},
  journal={arXiv preprint arXiv:2106.01969},
  year={2021}
}

@article{sayin2020fictitious,
  title={Fictitious play in zero-sum stochastic games},
  author={Sayin, Muhammed O and Parise, Francesca and Ozdaglar, Asuman},
  journal={arXiv preprint arXiv:2010.04223},
  year={2020}
}

@article{sayin2021decentralized,
  title={Decentralized Q-learning in zero-sum Markov games},
  author={Sayin, Muhammed and Zhang, Kaiqing and Leslie, David and Basar, Tamer and Ozdaglar, Asuman},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  year={2021}
}

@article{harris1998rate,
  title={On the rate of convergence of continuous-time fictitious play},
  author={Harris, Christopher},
  journal={Games and Economic Behavior},
  volume={22},
  number={2},
  pages={238--259},
  year={1998},
  publisher={Elsevier}
}

@article{friedman1998economic,
  title={On economic applications of evolutionary game theory},
  author={Friedman, Daniel},
  journal={Journal of evolutionary economics},
  volume={8},
  number={1},
  pages={15--43},
  year={1998},
  publisher={Springer}
}

@article{fudenberg2009learning,
  title={Learning and equilibrium},
  author={Fudenberg, Drew and Levine, David K},
  journal={Annual Review of Economics},
  year={2009},
  publisher={Annual Reviews}
}

@incollection{benaim1999dynamics,
  title={Dynamics of stochastic approximation algorithms},
  author={Bena{\"\i}m, Michel},
  booktitle={Seminaire de probabilites XXXIII},
  pages={1--68},
  year={1999},
  publisher={Springer}
}

@article{busoniu2008comprehensive,
  title={A comprehensive survey of multiagent reinforcement learning},
  author={Busoniu, Lucian and Babuska, Robert and De Schutter, Bart},
  journal={IEEE Transactions on Systems, Man, and Cybernetics, Part C (Applications and Reviews)},
  volume={38},
  number={2},
  pages={156--172},
  year={2008},
  publisher={IEEE}
}

@article{iqbal2019coordinated,
  title={Coordinated exploration via intrinsic rewards for multi-agent reinforcement learning},
  author={Iqbal, Shariq and Sha, Fei},
  journal={arXiv preprint arXiv:1905.12127},
  year={2019}
}

@article{papoudakis2019dealing,
  title={Dealing with non-stationarity in multi-agent deep reinforcement learning},
  author={Papoudakis, Georgios and Christianos, Filippos and Rahman, Arrasy and Albrecht, Stefano V},
  journal={arXiv preprint arXiv:1906.04737},
  year={2019}
}


@inproceedings{losey2020learning,
  title={Learning from my partner’s actions: Roles in decentralized robot teams},
  author={Losey, Dylan P and Li, Mengxi and Bohg, Jeannette and Sadigh, Dorsa},
  booktitle={Conference on robot learning},
  pages={752--765},
  year={2020},
  organization={PMLR}
}

@inproceedings{hernandez2017learning,
  title={Learning against sequential opponents in repeated stochastic games},
  author={Hernandez-Leal, Pablo and Kaisers, Michael},
  booktitle={The 3rd Multi-disciplinary Conference on Reinforcement Learning and Decision Making, Ann Arbor},
  volume={25},
  year={2017}
}

@article{leslie2020best,
  title={Best-response dynamics in zero-sum stochastic games},
  author={Leslie, David S and Perkins, Steven and Xu, Zibo},
  journal={Journal of Economic Theory},
  volume={189},
  pages={105095},
  year={2020},
  publisher={Elsevier}
}

@article{candogan2011flows,
  title={Flows and decompositions of games: Harmonic and potential games},
  author={Candogan, Ozan and Menache, Ishai and Ozdaglar, Asuman and Parrilo, Pablo A},
  journal={Mathematics of Operations Research},
  volume={36},
  number={3},
  pages={474--503},
  year={2011},
  publisher={INFORMS}
}

@inproceedings{greenwald2003correlated,
  title={Correlated Q-learning},
  author={Greenwald, Amy and Hall, Keith and Serrano, Roberto and others},
  booktitle={ICML},
  volume={3},
  pages={242--249},
  year={2003}
}

@article{powers2007general,
  title={A general criterion and an algorithmic framework for learning in multi-agent systems},
  author={Powers, Rob and Shoham, Yoav and Vu, Thuc},
  journal={Machine Learning},
  volume={67},
  number={1},
  pages={45--76},
  year={2007},
  publisher={Springer}
}

@inproceedings{hernandez2017learning,
  title={Learning against sequential opponents in repeated stochastic games},
  author={Hernandez-Leal, Pablo and Kaisers, Michael},
  booktitle={The 3rd Multi-disciplinary Conference on Reinforcement Learning and Decision Making, Ann Arbor},
  volume={25},
  year={2017}
}

@article{bowling2002multiagent,
  title={Multiagent learning using a variable learning rate},
  author={Bowling, Michael and Veloso, Manuela},
  journal={Artificial Intelligence},
  volume={136},
  number={2},
  pages={215--250},
  year={2002},
  publisher={Elsevier}
}

@article{foerster2016learning,
  title={Learning to communicate with deep multi-agent reinforcement learning},
  author={Foerster, Jakob and Assael, Ioannis Alexandros and De Freitas, Nando and Whiteson, Shimon},
  journal={Advances in neural information processing systems},
  volume={29},
  year={2016}
}

@article{lowe2017multi,
  title={Multi-agent actor-critic for mixed cooperative-competitive environments},
  author={Lowe, Ryan and Wu, Yi I and Tamar, Aviv and Harb, Jean and Pieter Abbeel, OpenAI and Mordatch, Igor},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}

@inproceedings{wang2022influencing,
  title={Influencing towards stable multi-agent interactions},
  author={Wang, Woodrow Zhouyuan and Shih, Andy and Xie, Annie and Sadigh, Dorsa},
  booktitle={Conference on Robot Learning},
  pages={1132--1143},
  year={2022},
  organization={PMLR}
}

@article{hernandez2017survey,
  title={A survey of learning in multiagent environments: Dealing with non-stationarity},
  author={Hernandez-Leal, Pablo and Kaisers, Michael and Baarslag, Tim and de Cote, Enrique Munoz},
  journal={arXiv preprint arXiv:1707.09183},
  year={2017}
}



@article{fink1964equilibrium,
  title={Equilibrium in a stochastic $ n $-person game},
  author={Fink, Arlington M},
  journal={Journal of science of the hiroshima university, series ai (mathematics)},
  volume={28},
  number={1},
  pages={89--93},
  year={1964},
  publisher={Hiroshima University, Mathematics Program}
}

@inproceedings{long2018towards,
  title={Towards optimally decentralized multi-robot collision avoidance via deep reinforcement learning},
  author={Long, Pinxin and Fan, Tingxiang and Liao, Xinyi and Liu, Wenxi and Zhang, Hao and Pan, Jia},
  booktitle={2018 IEEE International Conference on Robotics and Automation (ICRA)},
  pages={6252--6259},
  year={2018},
  organization={IEEE}
}

@article{lowe2017multi,
  title={Multi-agent actor-critic for mixed cooperative-competitive environments},
  author={Lowe, Ryan and Wu, Yi I and Tamar, Aviv and Harb, Jean and Pieter Abbeel, OpenAI and Mordatch, Igor},
  journal={Advances in neural information processing systems},
  volume={30},
  year={2017}
}

@inproceedings{hessel2018rainbow,
  title={Rainbow: Combining improvements in deep reinforcement learning},
  author={Hessel, Matteo and Modayil, Joseph and Van Hasselt, Hado and Schaul, Tom and Ostrovski, Georg and Dabney, Will and Horgan, Dan and Piot, Bilal and Azar, Mohammad and Silver, David},
  booktitle={Thirty-second AAAI conference on artificial intelligence},
  year={2018}
}

@article{vinyals2019grandmaster,
  title={Grandmaster level in StarCraft II using multi-agent reinforcement learning},
  author={Vinyals, Oriol and Babuschkin, Igor and Czarnecki, Wojciech M and Mathieu, Micha{\"e}l and Dudzik, Andrew and Chung, Junyoung and Choi, David H and Powell, Richard and Ewalds, Timo and Georgiev, Petko and others},
  journal={Nature},
  volume={575},
  number={7782},
  pages={350--354},
  year={2019},
  publisher={Nature Publishing Group}
}

@article{vinyals2017starcraft,
  title={Starcraft ii: A new challenge for reinforcement learning},
  author={Vinyals, Oriol and Ewalds, Timo and Bartunov, Sergey and Georgiev, Petko and Vezhnevets, Alexander Sasha and Yeo, Michelle and Makhzani, Alireza and K{\"u}ttler, Heinrich and Agapiou, John and Schrittwieser, Julian and others},
  journal={arXiv preprint arXiv:1708.04782},
  year={2017}
}

@article{hart1992games,
  title={Games in extensive and strategic forms},
  author={Hart, Sergiu},
  journal={Handbook of game theory with economic applications},
  volume={1},
  pages={19--40},
  year={1992},
  publisher={Elsevier}
}

@article{mguni2020stochastic,
  title={Stochastic potential games},
  author={Mguni, David},
  journal={arXiv preprint arXiv:2005.13527},
  year={2020}
}

@article{shapley1953stochastic,
  title={Stochastic games},
  author={Shapley, Lloyd S},
  journal={Proceedings of the national academy of sciences},
  volume={39},
  number={10},
  pages={1095--1100},
  year={1953},
  publisher={National Acad Sciences}
}

@book{fudenberg1998theory,
  title={The theory of learning in games},
  author={Fudenberg, Drew and Drew, Fudenberg and Levine, David K and Levine, David K},
  volume={2},
  year={1998},
  publisher={MIT press}
}

@article{leonardos2021global,
  title={Global convergence of multi-agent policy gradient in markov potential games},
  author={Leonardos, Stefanos and Overman, Will and Panageas, Ioannis and Piliouras, Georgios},
  journal={arXiv preprint arXiv:2106.01969},
  year={2021}
}

@article{sayin2020fictitious,
  title={Fictitious play in zero-sum stochastic games},
  author={Sayin, Muhammed O and Parise, Francesca and Ozdaglar, Asuman},
  journal={arXiv preprint arXiv:2010.04223},
  year={2020}
}

@book{sutton2018reinforcement,
  title={Reinforcement learning: An introduction},
  author={Sutton, Richard S and Barto, Andrew G},
  year={2018},
  publisher={MIT press}
}

@article{yang2020overview,
  title={An overview of multi-agent reinforcement learning from game theoretical perspective},
  author={Yang, Yaodong and Wang, Jun},
  journal={arXiv preprint arXiv:2011.00583},
  year={2020}
}

@article{silver2021reward,
  title={Reward is enough},
  author={Silver, David and Singh, Satinder and Precup, Doina and Sutton, Richard S},
  journal={Artificial Intelligence},
  volume={299},
  pages={103535},
  year={2021},
  publisher={Elsevier}
}

@article{bellman1966dynamic,
  title={Dynamic programming},
  author={Bellman, Richard},
  journal={Science},
  volume={153},
  number={3731},
  pages={34--37},
  year={1966},
  publisher={American Association for the Advancement of Science}
}

@inproceedings{ding2022independent,
  title={Independent policy gradient for large-scale Markov potential games: Sharper rates, function approximation, and game-agnostic convergence},
  author={Ding, Dongsheng and Wei, Chen-Yu and Zhang, Kaiqing and Jovanovic, Mihailo},
  booktitle={International Conference on Machine Learning},
  pages={5166--5220},
  year={2022},
  organization={PMLR}
}

@article{heikkinen2006potential,
  title={A potential game approach to distributed power control and scheduling},
  author={Heikkinen, Tiina},
  journal={Computer Networks},
  volume={50},
  number={13},
  pages={2295--2311},
  year={2006},
  publisher={Elsevier}
}

@article{yu1995game,
  title={A game strategy approach for image labeling},
  author={Yu, Shan and Berthod, Marc},
  journal={Computer Vision and Image Understanding},
  volume={61},
  number={1},
  pages={32--37},
  year={1995},
  publisher={Elsevier}
}

@article{berthod1996bayesian,
  title={Bayesian image classification using Markov random fields},
  author={Berthod, Marc and Kato, Zoltan and Yu, Shan and Zerubia, Josiane},
  journal={Image and vision computing},
  volume={14},
  number={4},
  pages={285--295},
  year={1996},
  publisher={Elsevier}
}

@article{rosenthal1973class,
  title={A class of games possessing pure-strategy Nash equilibria},
  author={Rosenthal, Robert W},
  journal={International Journal of Game Theory},
  volume={2},
  number={1},
  pages={65--67},
  year={1973},
  publisher={Physica-Verlag}
}

@article{babichenko2016graphical,
  title={Graphical potential games},
  author={Babichenko, Yakov and Tamuz, Omer},
  journal={Journal of Economic Theory},
  volume={163},
  pages={889--899},
  year={2016},
  publisher={Elsevier}
}

@inproceedings{mguni2021learning,
  title={Learning in nonzero-sum stochastic games with potentials},
  author={Mguni, David H and Wu, Yutong and Du, Yali and Yang, Yaodong and Wang, Ziyi and Li, Minne and Wen, Ying and Jennings, Joel and Wang, Jun},
  booktitle={International Conference on Machine Learning},
  pages={7688--7699},
  year={2021},
  organization={PMLR}
}

@article{roughgarden2007routing,
  title={Routing games},
  author={Roughgarden, Tim},
  journal={Algorithmic game theory},
  volume={18},
  pages={459--484},
  year={2007},
  publisher={Cambridge University Press Cambridge}
}

@article{hernandez2017exploration,
  title={An exploration strategy for non-stationary opponents},
  author={Hernandez-Leal, Pablo and Zhan, Yusen and Taylor, Matthew E and Sucar, L Enrique and Munoz de Cote, Enrique},
  journal={Autonomous Agents and Multi-Agent Systems},
  volume={31},
  number={5},
  pages={971--1002},
  year={2017},
  publisher={Springer}
}

@article{zhao2021provably,
  title={Provably efficient policy gradient methods for two-player zero-sum Markov games},
  author={Zhao, Yulai and Tian, Yuandong and Lee, Jason D and Du, Simon S},
  journal={arXiv preprint arXiv:2102.08903},
  year={2021}
}

@inproceedings{guo2021decentralized,
  title={Decentralized Single-Timescale Actor-Critic on Zero-Sum Two-Player Stochastic Games},
  author={Guo, Hongyi and Fu, Zuyue and Yang, Zhuoran and Wang, Zhaoran},
  booktitle={International Conference on Machine Learning},
  pages={3899--3909},
  year={2021},
  organization={PMLR}
}

@article{leslie2003convergent,
  title={Convergent multiple-timescales reinforcement learning algorithms in normal form games},
  author={Leslie, David S and Collins, EJ},
  journal={The Annals of Applied Probability},
  volume={13},
  number={4},
  pages={1231--1251},
  year={2003},
  publisher={Institute of Mathematical Statistics}
}

@inproceedings{guo2021decentralized,
  title={Decentralized Single-Timescale Actor-Critic on Zero-Sum Two-Player Stochastic Games},
  author={Guo, Hongyi and Fu, Zuyue and Yang, Zhuoran and Wang, Zhaoran},
  booktitle={International Conference on Machine Learning},
  pages={3899--3909},
  year={2021},
  organization={PMLR}
}

@inproceedings{elidrisi2014fast,
  title={Fast adaptive learning in repeated stochastic games by game abstraction.},
  author={Elidrisi, Mohamed and Johnson, Nicholas and Gini, Maria L and Crandall, Jacob W},
  booktitle={AAMAS},
  pages={1141--1148},
  year={2014}
}

@article{bowling2002multiagent,
  title={Multiagent learning using a variable learning rate},
  author={Bowling, Michael and Veloso, Manuela},
  journal={Artificial Intelligence},
  volume={136},
  number={2},
  pages={215--250},
  year={2002},
  publisher={Elsevier}
}

@inproceedings{wunder2012framework,
  title={A framework for modeling population strategies by depth of reasoning.},
  author={Wunder, Michael and Yaros, John Robert and Kaisers, Michael and Littman, Michael L},
  booktitle={AAMAS},
  pages={947--954},
  year={2012},
  organization={Citeseer}
}

@article{gmytrasiewicz2005framework,
  title={A framework for sequential planning in multi-agent settings},
  author={Gmytrasiewicz, Piotr J and Doshi, Prashant},
  journal={Journal of Artificial Intelligence Research},
  volume={24},
  pages={49--79},
  year={2005}
}

@inproceedings{fox2022independent,
  title={Independent natural policy gradient always converges in Markov potential games},
  author={Fox, Roy and Mcaleer, Stephen M and Overman, Will and Panageas, Ioannis},
  booktitle={International Conference on Artificial Intelligence and Statistics},
  pages={4414--4425},
  year={2022},
  organization={PMLR}
}

@article{mnih2013playing,
  title={Playing atari with deep reinforcement learning},
  author={Mnih, Volodymyr and Kavukcuoglu, Koray and Silver, David and Graves, Alex and Antonoglou, Ioannis and Wierstra, Daan and Riedmiller, Martin},
  journal={arXiv preprint arXiv:1312.5602},
  year={2013}
}

@article{szita2006learning,
  title={Learning Tetris using the noisy cross-entropy method},
  author={Szita, Istv{\'a}n and L{\"o}rincz, Andr{\'a}s},
  journal={Neural computation},
  volume={18},
  number={12},
  pages={2936--2941},
  year={2006},
  publisher={MIT Press}
}

@inproceedings{lu2022aw,
  title={AW-Opt: Learning Robotic Skills with Imitation andReinforcement at Scale},
  author={Lu, Yao and Hausman, Karol and Chebotar, Yevgen and Yan, Mengyuan and Jang, Eric and Herzog, Alexander and Xiao, Ted and Irpan, Alex and Khansari, Mohi and Kalashnikov, Dmitry and others},
  booktitle={Conference on Robot Learning},
  pages={1078--1088},
  year={2022},
  organization={PMLR}
}

@inproceedings{kalashnikov2018scalable,
  title={Scalable deep reinforcement learning for vision-based robotic manipulation},
  author={Kalashnikov, Dmitry and Irpan, Alex and Pastor, Peter and Ibarz, Julian and Herzog, Alexander and Jang, Eric and Quillen, Deirdre and Holly, Ethan and Kalakrishnan, Mrinal and Vanhoucke, Vincent and others},
  booktitle={Conference on Robot Learning},
  pages={651--673},
  year={2018},
  organization={PMLR}
}

@article{lillicrap2015continuous,
  title={Continuous control with deep reinforcement learning},
  author={Lillicrap, Timothy P and Hunt, Jonathan J and Pritzel, Alexander and Heess, Nicolas and Erez, Tom and Tassa, Yuval and Silver, David and Wierstra, Daan},
  journal={arXiv preprint arXiv:1509.02971},
  year={2015}
}


@book{dong2020deep,
  title={Deep Reinforcement Learning},
  author={Dong, Hao and Dong, Hao and Ding, Zihan and Zhang, Shanghang and Chang},
  year={2020},
  publisher={Springer}
}

@article{szepesvari1997asymptotic,
  title={The asymptotic convergence-rate of Q-learning},
  author={Szepesv{\'a}ri, Csaba},
  journal={Advances in neural information processing systems},
  volume={10},
  year={1997}
}

@article{hernandez2017survey,
  title={A survey of learning in multiagent environments: Dealing with non-stationarity},
  author={Hernandez-Leal, Pablo and Kaisers, Michael and Baarslag, Tim and de Cote, Enrique Munoz},
  journal={arXiv preprint arXiv:1707.09183},
  year={2017}
}

@incollection{hastie2009unsupervised,
  title={Unsupervised learning},
  author={Hastie, Trevor and Tibshirani, Robert and Friedman, Jerome},
  booktitle={The elements of statistical learning},
  pages={485--585},
  year={2009},
  publisher={Springer}
}

@article{ha2018world,
  title={World models},
  author={Ha, David and Schmidhuber, J{\"u}rgen},
  journal={arXiv preprint arXiv:1803.10122},
  year={2018}
}

@article{sutton1995generalization,
  title={Generalization in reinforcement learning: Successful examples using sparse coarse coding},
  author={Sutton, Richard S},
  journal={Advances in neural information processing systems},
  volume={8},
  year={1995}
}

@book{rummery1994line,
  title={On-line Q-learning using connectionist systems},
  author={Rummery, Gavin A and Niranjan, Mahesan},
  volume={37},
  year={1994},
  publisher={Citeseer}
}

@ARTICLE{Peters:2010,
AUTHOR = {Peters, J. },
TITLE   = {{P}olicy gradient methods},
YEAR    = {2010},
JOURNAL = {Scholarpedia},
VOLUME  = {5},
NUMBER  = {11},
PAGES   = {3698},
DOI     = {10.4249/scholarpedia.3698},
NOTE    = {revision \#137199}
}

@article{szepesvari2010algorithms,
  title={Algorithms for reinforcement learning},
  author={Szepesv{\'a}ri, Csaba},
  journal={Synthesis lectures on artificial intelligence and machine learning},
  volume={4},
  number={1},
  pages={1--103},
  year={2010},
  publisher={Morgan \& Claypool Publishers}
}

@incollection{hastie2009overview,
  title={Overview of supervised learning},
  author={Hastie, Trevor and Tibshirani, Robert and Friedman, Jerome},
  booktitle={The elements of statistical learning},
  pages={9--41},
  year={2009},
  publisher={Springer}
}

@article{dayan1999unsupervised,
  title={Unsupervised learning},
  author={Dayan, Peter and Sahani, Maneesh and Deback, Gr{\'e}goire},
  journal={The MIT encyclopedia of the cognitive sciences},
  pages={857--859},
  year={1999},
  publisher={MIT Press}
}

@article{busoniu2008comprehensive,
  title={A comprehensive survey of multiagent reinforcement learning},
  author={Busoniu, Lucian and Babuska, Robert and De Schutter, Bart},
  journal={IEEE Transactions on Systems, Man, and Cybernetics, Part C (Applications and Reviews)},
  volume={38},
  number={2},
  pages={156--172},
  year={2008},
  publisher={IEEE}
}

@article{vinyals2019grandmaster,
  title={Grandmaster level in StarCraft II using multi-agent reinforcement learning},
  author={Vinyals, Oriol and Babuschkin, Igor and Czarnecki, Wojciech M and Mathieu, Micha{\"e}l and Dudzik, Andrew and Chung, Junyoung and Choi, David H and Powell, Richard and Ewalds, Timo and Georgiev, Petko and others},
  journal={Nature},
  volume={575},
  number={7782},
  pages={350--354},
  year={2019},
  publisher={Nature Publishing Group}
}

@article{silver2017mastering,
  title={Mastering chess and shogi by self-play with a general reinforcement learning algorithm},
  author={Silver, David and Hubert, Thomas and Schrittwieser, Julian and Antonoglou, Ioannis and Lai, Matthew and Guez, Arthur and Lanctot, Marc and Sifre, Laurent and Kumaran, Dharshan and Graepel, Thore and others},
  journal={arXiv preprint arXiv:1712.01815},
  year={2017}
}

@article{watkins1992q,
  title={Q-learning},
  author={Watkins, Christopher JCH and Dayan, Peter},
  journal={Machine learning},
  volume={8},
  number={3},
  pages={279--292},
  year={1992},
  publisher={Springer}
}

@article{moerland2020model,
  title={Model-based reinforcement learning: A survey},
  author={Moerland, Thomas M and Broekens, Joost and Jonker, Catholijn M},
  journal={arXiv preprint arXiv:2006.16712},
  year={2020}
}

@article{leslie2005individual,
  title={Individual Q-learning in normal form games},
  author={Leslie, David S and Collins, Edmund J},
  journal={SIAM Journal on Control and Optimization},
  volume={44},
  number={2},
  pages={495--514},
  year={2005},
  publisher={SIAM}
}

@inproceedings{zhang2018fully,
  title={Fully decentralized multi-agent reinforcement learning with networked agents},
  author={Zhang, Kaiqing and Yang, Zhuoran and Liu, Han and Zhang, Tong and Basar, Tamer},
  booktitle={International Conference on Machine Learning},
  pages={5872--5881},
  year={2018},
  organization={PMLR}
}

@article{zhang2021finite,
  title={Finite-sample analysis for decentralized batch multiagent reinforcement learning with networked agents},
  author={Zhang, Kaiqing and Yang, Zhuoran and Liu, Han and Zhang, Tong and Ba{\c{s}}ar, Tamer},
  journal={IEEE Transactions on Automatic Control},
  volume={66},
  number={12},
  pages={5925--5940},
  year={2021},
  publisher={IEEE}
}

@inproceedings{zhang2018model,
  title={Model-free reinforcement learning for fully cooperative multi-agent graphical games},
  author={Zhang, Qichao and Zhao, Dongbin and Lewis, Frank L},
  booktitle={2018 International Joint Conference on Neural Networks (IJCNN)},
  pages={1--6},
  year={2018},
  organization={IEEE}
}

@article{zhao2021provably,
  title={Provably efficient policy gradient methods for two-player zero-sum Markov games},
  author={Zhao, Yulai and Tian, Yuandong and Lee, Jason D and Du, Simon S},
  journal={arXiv preprint arXiv:2102.08903},
  year={2021}
}

@inproceedings{zhang2018networked,
  title={Networked multi-agent reinforcement learning in continuous spaces},
  author={Zhang, Kaiqing and Yang, Zhuoran and Basar, Tamer},
  booktitle={2018 IEEE conference on decision and control (CDC)},
  pages={2771--2776},
  year={2018},
  organization={IEEE}
}

@inproceedings{zhang2019distributed,
  title={Distributed off-policy actor-critic reinforcement learning with policy consensus},
  author={Zhang, Yan and Zavlanos, Michael M},
  booktitle={2019 IEEE 58th Conference on Decision and Control (CDC)},
  pages={4674--4679},
  year={2019},
  organization={IEEE}
}

@inproceedings{wei2021last,
  title={Last-iterate convergence of decentralized optimistic gradient descent/ascent in infinite-horizon competitive markov games},
  author={Wei, Chen-Yu and Lee, Chung-Wei and Zhang, Mengxiao and Luo, Haipeng},
  booktitle={Conference on learning theory},
  pages={4259--4299},
  year={2021},
  organization={PMLR}
}

@book{gonzalez2013discrete,
  title={Discrete--time stochastic control and dynamic potential games: the Euler--Equation approach},
  author={Gonz{\'a}lez-S{\'a}nchez, David and Hern{\'a}ndez-Lerma, On{\'e}simo},
  year={2013},
  publisher={Springer Science \& Business Media}
}

@inproceedings{mguni2018decentralised,
  title={Decentralised learning in systems with many, many strategic agents},
  author={Mguni, David and Jennings, Joel and de Cote, Enrique Munoz},
  booktitle={Thirty-second AAAI conference on artificial intelligence},
  year={2018}
}

@article{zhang2021gradient,
  title={Gradient play in multi-agent markov stochastic games: Stationary points and convergence},
  author={Zhang, Runyu and Ren, Zhaolin and Li, Na},
  journal={arXiv preprint arXiv:2106.00198},
  year={2021}
}

@article{leonardos2021exploration,
  title={Exploration-exploitation in multi-agent competition: convergence with bounded rationality},
  author={Leonardos, Stefanos and Piliouras, Georgios and Spendlove, Kelly},
  journal={Advances in Neural Information Processing Systems},
  volume={34},
  pages={26318--26331},
  year={2021}
}

@book{borkar2009stochastic,
  title={Stochastic approximation: a dynamical systems viewpoint},
  author={Borkar, Vivek S},
  volume={48},
  year={2009},
  publisher={Springer}
}

@book{kelley2010theory,
  title={The theory of differential equations: classical and qualitative},
  author={Kelley, Walter G and Peterson, Allan C},
  year={2010},
  publisher={Springer Science \& Business Media}
}

@inproceedings{wei2021last,
  title={Last-iterate convergence of decentralized optimistic gradient descent/ascent in infinite-horizon competitive markov games},
  author={Wei, Chen-Yu and Lee, Chung-Wei and Zhang, Mengxiao and Luo, Haipeng},
  booktitle={Conference on learning theory},
  pages={4259--4299},
  year={2021},
  organization={PMLR}
}

@article{leonardos2022exploration,
  title={Exploration-exploitation in multi-agent learning: Catastrophe theory meets game theory},
  author={Leonardos, Stefanos and Piliouras, Georgios},
  journal={Artificial Intelligence},
  volume={304},
  pages={103653},
  year={2022},
  publisher={Elsevier}
}

@article{agarwal2021theory,
  title={On the Theory of Policy Gradient Methods: Optimality, Approximation, and Distribution Shift.},
  author={Agarwal, Alekh and Kakade, Sham M and Lee, Jason D and Mahajan, Gaurav},
  journal={J. Mach. Learn. Res.},
  volume={22},
  number={98},
  pages={1--76},
  year={2021}
}

@article{zazo2016dynamic,
  title={Dynamic potential games with constraints: Fundamentals and applications in communications},
  author={Zazo, Santiago and Macua, Sergio Valcarcel and S{\'a}nchez-Fern{\'a}ndez, Matilde and Zazo, Javier},
  journal={IEEE Transactions on Signal Processing},
  volume={64},
  number={14},
  pages={3806--3821},
  year={2016},
  publisher={IEEE}
}

@article{macua2018learning,
  title={Learning parametric closed-loop policies for markov potential games},
  author={Macua, Sergio Valcarcel and Zazo, Javier and Zazo, Santiago},
  journal={arXiv preprint arXiv:1802.00899},
  year={2018}
}