@inproceedings{NIPS2017_3f5ee243,
 author = {Vaswani, Ashish and Shazeer, Noam and Parmar, Niki and Uszkoreit, Jakob and Jones, Llion and Gomez, Aidan N and Kaiser, \L ukasz and Polosukhin, Illia},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {I. Guyon and U. Von Luxburg and S. Bengio and H. Wallach and R. Fergus and S. Vishwanathan and R. Garnett},
 pages = {},
 publisher = {Curran Associates, Inc.},
 title = {Attention is All you Need},
 url = {https://proceedings.neurips.cc/paper_files/paper/2017/file/3f5ee243547dee91fbd053c1c4a845aa-Paper.pdf},
 volume = {30},
 year = {2017}
}

@inproceedings{NEURIPS2020_1457c0d6,
 author = {Brown, Tom and Mann, Benjamin and Ryder, Nick and Subbiah, Melanie and Kaplan, Jared D and Dhariwal, Prafulla and Neelakantan, Arvind and Shyam, Pranav and Sastry, Girish and Askell, Amanda and Agarwal, Sandhini and Herbert-Voss, Ariel and Krueger, Gretchen and Henighan, Tom and Child, Rewon and Ramesh, Aditya and Ziegler, Daniel and Wu, Jeffrey and Winter, Clemens and Hesse, Chris and Chen, Mark and Sigler, Eric and Litwin, Mateusz and Gray, Scott and Chess, Benjamin and Clark, Jack and Berner, Christopher and McCandlish, Sam and Radford, Alec and Sutskever, Ilya and Amodei, Dario},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {H. Larochelle and M. Ranzato and R. Hadsell and M.F. Balcan and H. Lin},
 pages = {1877--1901},
 publisher = {Curran Associates, Inc.},
 title = {Language Models are Few-Shot Learners},
 url = {https://proceedings.neurips.cc/paper_files/paper/2020/file/1457c0d6bfcb4967418bfb8ac142f64a-Paper.pdf},
 volume = {33},
 year = {2020}
}

@misc{kaplan2020scalinglawsneurallanguage,
      title={Scaling Laws for Neural Language Models}, 
      author={Jared Kaplan and Sam McCandlish and Tom Henighan and Tom B. Brown and Benjamin Chess and Rewon Child and Scott Gray and Alec Radford and Jeffrey Wu and Dario Amodei},
      year={2020},
      eprint={2001.08361},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2001.08361}, 
}

@misc{han2015learningweightsconnectionsefficient,
      title={Learning both Weights and Connections for Efficient Neural Networks}, 
      author={Song Han and Jeff Pool and John Tran and William J. Dally},
      year={2015},
      eprint={1506.02626},
      archivePrefix={arXiv},
      primaryClass={cs.NE},
      url={https://arxiv.org/abs/1506.02626}, 
}

@misc{gholami2021surveyquantizationmethodsefficient,
      title={A Survey of Quantization Methods for Efficient Neural Network Inference}, 
      author={Amir Gholami and Sehoon Kim and Zhen Dong and Zhewei Yao and Michael W. Mahoney and Kurt Keutzer},
      year={2021},
      eprint={2103.13630},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/2103.13630}, 
}

@misc{han2016deepcompressioncompressingdeep,
      title={Deep Compression: Compressing Deep Neural Networks with Pruning, Trained Quantization and Huffman Coding}, 
      author={Song Han and Huizi Mao and William J. Dally},
      year={2016},
      eprint={1510.00149},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/1510.00149}, 
}

@misc{hu2021loralowrankadaptationlarge,
      title={LoRA: Low-Rank Adaptation of Large Language Models}, 
      author={Edward J. Hu and Yelong Shen and Phillip Wallis and Zeyuan Allen-Zhu and Yuanzhi Li and Shean Wang and Lu Wang and Weizhu Chen},
      year={2021},
      eprint={2106.09685},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2106.09685}, 
}

@misc{zeng2024expressivepowerlowrankadaptation,
      title={The Expressive Power of Low-Rank Adaptation}, 
      author={Yuchen Zeng and Kangwook Lee},
      year={2024},
      eprint={2310.17513},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2310.17513}, 
}

@misc{zhao2024loraland310finetuned,
      title={LoRA Land: 310 Fine-tuned LLMs that Rival GPT-4, A Technical Report}, 
      author={Justin Zhao and Timothy Wang and Wael Abid and Geoffrey Angus and Arnav Garg and Jeffery Kinnison and Alex Sherstinsky and Piero Molino and Travis Addair and Devvret Rishi},
      year={2024},
      eprint={2405.00732},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2405.00732}, 
}

@misc{jacob2017quantizationtrainingneuralnetworks,
      title={Quantization and Training of Neural Networks for Efficient Integer-Arithmetic-Only Inference}, 
      author={Benoit Jacob and Skirmantas Kligys and Bo Chen and Menglong Zhu and Matthew Tang and Andrew Howard and Hartwig Adam and Dmitry Kalenichenko},
      year={2017},
      eprint={1712.05877},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/1712.05877}, 
}

@misc{shim2021layerwisepruningtransformerattention,
      title={Layer-wise Pruning of Transformer Attention Heads for Efficient Language Modeling}, 
      author={Kyuhong Shim and Iksoo Choi and Wonyong Sung and Jungwook Choi},
      year={2021},
      eprint={2110.03252},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2110.03252}, 
}

@misc{alizadeh2024llmflashefficientlarge,
      title={LLM in a flash: Efficient Large Language Model Inference with Limited Memory}, 
      author={Keivan Alizadeh and Iman Mirzadeh and Dmitry Belenko and Karen Khatamifard and Minsik Cho and Carlo C Del Mundo and Mohammad Rastegari and Mehrdad Farajtabar},
      year={2024},
      eprint={2312.11514},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2312.11514}, 
}

@misc{yang2018adaptationsrougebleubetter,
      title={Adaptations of ROUGE and BLEU to Better Evaluate Machine Reading Comprehension Task}, 
      author={An Yang and Kai Liu and Jing Liu and Yajuan Lyu and Sujian Li},
      year={2018},
      eprint={1806.03578},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/1806.03578}, 
}

@misc{alon2023detectinglanguagemodelattacks,
      title={Detecting Language Model Attacks with Perplexity}, 
      author={Gabriel Alon and Michael Kamfonas},
      year={2023},
      eprint={2308.14132},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2308.14132}, 
}

@inproceedings{yacouby-axman-2020-probabilistic,
    title = "Probabilistic Extension of Precision, Recall, and F1 Score for More Thorough Evaluation of Classification Models",
    author = "Yacouby, Reda  and
      Axman, Dustin",
    editor = "Eger, Steffen  and
      Gao, Yang  and
      Peyrard, Maxime  and
      Zhao, Wei  and
      Hovy, Eduard",
    booktitle = "Proceedings of the First Workshop on Evaluation and Comparison of NLP Systems",
    month = nov,
    year = "2020",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://aclanthology.org/2020.eval4nlp-1.9/",
    doi = "10.18653/v1/2020.eval4nlp-1.9",
    pages = "79--91",
    abstract = "In pursuit of the perfect supervised NLP classifier, razor thin margins and low-resource test sets can make modeling decisions difficult. Popular metrics such as Accuracy, Precision, and Recall are often insufficient as they fail to give a complete picture of the model{'}s behavior. We present a probabilistic extension of Precision, Recall, and F1 score, which we refer to as confidence-Precision (cPrecision), confidence-Recall (cRecall), and confidence-F1 (cF1) respectively. The proposed metrics address some of the challenges faced when evaluating large-scale NLP systems, specifically when the model{'}s confidence score assignments have an impact on the system{'}s behavior. We describe four key benefits of our proposed metrics as compared to their threshold-based counterparts. Two of these benefits, which we refer to as robustness to missing values and sensitivity to model confidence score assignments are self-evident from the metrics' definitions; the remaining benefits, generalization, and functional consistency are demonstrated empirically."
}

@misc{frantar2023gptqaccurateposttrainingquantization,
      title={GPTQ: Accurate Post-Training Quantization for Generative Pre-trained Transformers}, 
      author={Elias Frantar and Saleh Ashkboos and Torsten Hoefler and Dan Alistarh},
      year={2023},
      eprint={2210.17323},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2210.17323}, 
}

@misc{bengio2013estimatingpropagatinggradientsstochastic,
      title={Estimating or Propagating Gradients Through Stochastic Neurons}, 
      author={Yoshua Bengio},
      year={2013},
      eprint={1305.2982},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/1305.2982}, 
}

@misc{choi2018pactparameterizedclippingactivation,
      title={PACT: Parameterized Clipping Activation for Quantized Neural Networks}, 
      author={Jungwook Choi and Zhuo Wang and Swagath Venkataramani and Pierce I-Jen Chuang and Vijayalakshmi Srinivasan and Kailash Gopalakrishnan},
      year={2018},
      eprint={1805.06085},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/1805.06085}, 
}

@misc{esser2020learnedstepsizequantization,
      title={Learned Step Size Quantization}, 
      author={Steven K. Esser and Jeffrey L. McKinstry and Deepika Bablani and Rathinakumar Appuswamy and Dharmendra S. Modha},
      year={2020},
      eprint={1902.08153},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/1902.08153}, 
}

@misc{nagel2021whitepaperneuralnetwork,
      title={A White Paper on Neural Network Quantization}, 
      author={Markus Nagel and Marios Fournarakis and Rana Ali Amjad and Yelysei Bondarenko and Mart van Baalen and Tijmen Blankevoort},
      year={2021},
      eprint={2106.08295},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2106.08295}, 
}

@misc{banner2019posttraining4bitquantizationconvolution,
      title={Post-training 4-bit quantization of convolution networks for rapid-deployment}, 
      author={Ron Banner and Yury Nahshan and Elad Hoffer and Daniel Soudry},
      year={2019},
      eprint={1810.05723},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/1810.05723}, 
}

@misc{choukroun2019lowbitquantizationneuralnetworks,
      title={Low-bit Quantization of Neural Networks for Efficient Inference}, 
      author={Yoni Choukroun and Eli Kravchik and Fan Yang and Pavel Kisilev},
      year={2019},
      eprint={1902.06822},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/1902.06822}, 
}

@misc{zhao2019improvingneuralnetworkquantization,
      title={Improving Neural Network Quantization without Retraining using Outlier Channel Splitting}, 
      author={Ritchie Zhao and Yuwei Hu and Jordan Dotzel and Christopher De Sa and Zhiru Zhang},
      year={2019},
      eprint={1901.09504},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/1901.09504}, 
}

@misc{gao2024spuqperturbationbaseduncertaintyquantification,
      title={SPUQ: Perturbation-Based Uncertainty Quantification for Large Language Models}, 
      author={Xiang Gao and Jiaxin Zhang and Lalla Mouatadid and Kamalika Das},
      year={2024},
      eprint={2403.02509},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2403.02509}, 
}

@misc{chen2024nlperturbatorstudyingrobustnesscode,
      title={NLPerturbator: Studying the Robustness of Code LLMs to Natural Language Variations}, 
      author={Junkai Chen and Zhenhao Li and Xing Hu and Xin Xia},
      year={2024},
      eprint={2406.19783},
      archivePrefix={arXiv},
      primaryClass={cs.SE},
      url={https://arxiv.org/abs/2406.19783}, 
}

@misc{chinova2025stochastic,
  title        = {Stochastic Token Permutation in Large Language Models for Controlled Contextual Perturbation},
  author       = {Chinova, Onem and Stratton, Victor and Kingswell, Dominic and Whitmore, Evelyn and Coleridge, Zephyr},
  year         = {2025},
  howpublished = {OSF Preprints},
  url          = {https://osf.io/}
}

@misc{dong2019hawqv2hessianawaretraceweighted,
      title={HAWQ-V2: Hessian Aware trace-Weighted Quantization of Neural Networks}, 
      author={Zhen Dong and Zhewei Yao and Yaohui Cai and Daiyaan Arfeen and Amir Gholami and Michael W. Mahoney and Kurt Keutzer},
      year={2019},
      eprint={1911.03852},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/1911.03852}, 
}

@misc{hinton2015distillingknowledgeneuralnetwork,
      title={Distilling the Knowledge in a Neural Network}, 
      author={Geoffrey Hinton and Oriol Vinyals and Jeff Dean},
      year={2015},
      eprint={1503.02531},
      archivePrefix={arXiv},
      primaryClass={stat.ML},
      url={https://arxiv.org/abs/1503.02531}, 
}

@misc{chen2023gradientbasedbileveloptimizationdeep,
      title={Gradient-based Bi-level Optimization for Deep Learning: A Survey}, 
      author={Can Chen and Xi Chen and Chen Ma and Zixuan Liu and Xue Liu},
      year={2023},
      eprint={2207.11719},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2207.11719}, 
}

@misc{luo2024benchmarkingdissectingnvidiahopper,
      title={Benchmarking and Dissecting the Nvidia Hopper GPU Architecture}, 
      author={Weile Luo and Ruibo Fan and Zeyu Li and Dayou Du and Qiang Wang and Xiaowen Chu},
      year={2024},
      eprint={2402.13499},
      archivePrefix={arXiv},
      primaryClass={cs.AR},
      url={https://arxiv.org/abs/2402.13499}, 
}

@misc{dettmers2023qloraefficientfinetuningquantized,
      title={QLoRA: Efficient Finetuning of Quantized LLMs}, 
      author={Tim Dettmers and Artidoro Pagnoni and Ari Holtzman and Luke Zettlemoyer},
      year={2023},
      eprint={2305.14314},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2305.14314}, 
}

@misc{liu2025spinquantllmquantizationlearned,
      title={SpinQuant: LLM quantization with learned rotations}, 
      author={Zechun Liu and Changsheng Zhao and Igor Fedorov and Bilge Soran and Dhruv Choudhary and Raghuraman Krishnamoorthi and Vikas Chandra and Yuandong Tian and Tijmen Blankevoort},
      year={2025},
      eprint={2405.16406},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2405.16406}, 
}

@misc{li2020efficientriemannianoptimizationstiefel,
      title={Efficient Riemannian Optimization on the Stiefel Manifold via the Cayley Transform}, 
      author={Jun Li and Li Fuxin and Sinisa Todorovic},
      year={2020},
      eprint={2002.01113},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2002.01113}, 
}

@misc{xiao2024smoothquantaccurateefficientposttraining,
      title={SmoothQuant: Accurate and Efficient Post-Training Quantization for Large Language Models}, 
      author={Guangxuan Xiao and Ji Lin and Mickael Seznec and Hao Wu and Julien Demouth and Song Han},
      year={2024},
      eprint={2211.10438},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2211.10438}, 
}

@misc{liu2023llmqatdatafreequantizationaware,
      title={LLM-QAT: Data-Free Quantization Aware Training for Large Language Models}, 
      author={Zechun Liu and Barlas Oguz and Changsheng Zhao and Ernie Chang and Pierre Stock and Yashar Mehdad and Yangyang Shi and Raghuraman Krishnamoorthi and Vikas Chandra},
      year={2023},
      eprint={2305.17888},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2305.17888}, 
}

@misc{ashkboos2024quarotoutlierfree4bitinference,
      title={QuaRot: Outlier-Free 4-Bit Inference in Rotated LLMs}, 
      author={Saleh Ashkboos and Amirkeivan Mohtashami and Maximilian L. Croci and Bo Li and Pashmina Cameron and Martin Jaggi and Dan Alistarh and Torsten Hoefler and James Hensman},
      year={2024},
      eprint={2404.00456},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2404.00456}, 
}

@inproceedings{NIPS1989_6c9882bb,
 author = {LeCun, Yann and Denker, John and Solla, Sara},
 booktitle = {Advances in Neural Information Processing Systems},
 editor = {D. Touretzky},
 pages = {},
 publisher = {Morgan-Kaufmann},
 title = {Optimal Brain Damage},
 url = {https://proceedings.neurips.cc/paper_files/paper/1989/file/6c9882bbac1c7093bd25041881277658-Paper.pdf},
 volume = {2},
 year = {1989}
}

@misc{michel2019sixteenheadsreallybetter,
      title={Are Sixteen Heads Really Better than One?}, 
      author={Paul Michel and Omer Levy and Graham Neubig},
      year={2019},
      eprint={1905.10650},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/1905.10650}, 
}

@misc{frankle2019lotterytickethypothesisfinding,
      title={The Lottery Ticket Hypothesis: Finding Sparse, Trainable Neural Networks}, 
      author={Jonathan Frankle and Michael Carbin},
      year={2019},
      eprint={1803.03635},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/1803.03635}, 
}

@misc{lin2022teachingmodelsexpressuncertainty,
      title={Teaching Models to Express Their Uncertainty in Words}, 
      author={Stephanie Lin and Jacob Hilton and Owain Evans},
      year={2022},
      eprint={2205.14334},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2205.14334}, 
}

@misc{kadavath2022languagemodelsmostlyknow,
      title={Language Models (Mostly) Know What They Know}, 
      author={Saurav Kadavath and Tom Conerly and Amanda Askell and Tom Henighan and Dawn Drain and Ethan Perez and Nicholas Schiefer and Zac Hatfield-Dodds and Nova DasSarma and Eli Tran-Johnson and Scott Johnston and Sheer El-Showk and Andy Jones and Nelson Elhage and Tristan Hume and Anna Chen and Yuntao Bai and Sam Bowman and Stanislav Fort and Deep Ganguli and Danny Hernandez and Josh Jacobson and Jackson Kernion and Shauna Kravec and Liane Lovitt and Kamal Ndousse and Catherine Olsson and Sam Ringer and Dario Amodei and Tom Brown and Jack Clark and Nicholas Joseph and Ben Mann and Sam McCandlish and Chris Olah and Jared Kaplan},
      year={2022},
      eprint={2207.05221},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2207.05221}, 
}

@misc{wang2023selfconsistencyimproveschainthought,
      title={Self-Consistency Improves Chain of Thought Reasoning in Language Models}, 
      author={Xuezhi Wang and Jason Wei and Dale Schuurmans and Quoc Le and Ed Chi and Sharan Narang and Aakanksha Chowdhery and Denny Zhou},
      year={2023},
      eprint={2203.11171},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2203.11171}, 
}

@misc{lin2024generatingconfidenceuncertaintyquantification,
      title={Generating with Confidence: Uncertainty Quantification for Black-box Large Language Models}, 
      author={Zhen Lin and Shubhendu Trivedi and Jimeng Sun},
      year={2024},
      eprint={2305.19187},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2305.19187}, 
}

@article{chen1998,
  title={Learning and using probabilistic relational models},
  author={Chen, Hsing-Hen and Getoor, Lise and others},
  journal={Proceedings of the AAAI},
  year={1998}
}

@misc{zhang2024alleviatinghallucinationslargelanguage,
      title={Alleviating Hallucinations of Large Language Models through Induced Hallucinations}, 
      author={Yue Zhang and Leyang Cui and Wei Bi and Shuming Shi},
      year={2024},
      eprint={2312.15710},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2312.15710}, 
}

@article{H_llermeier_2021,
   title={Aleatoric and epistemic uncertainty in machine learning: an introduction to concepts and methods},
   volume={110},
   ISSN={1573-0565},
   url={http://dx.doi.org/10.1007/s10994-021-05946-3},
   DOI={10.1007/s10994-021-05946-3},
   number={3},
   journal={Machine Learning},
   publisher={Springer Science and Business Media LLC},
   author={Hüllermeier, Eyke and Waegeman, Willem},
   year={2021},
   month=mar, pages={457–506} }

@article{JMLR:v15:srivastava14a,
  author  = {Nitish Srivastava and Geoffrey Hinton and Alex Krizhevsky and Ilya Sutskever and Ruslan Salakhutdinov},
  title   = {Dropout: A Simple Way to Prevent Neural Networks from Overfitting},
  journal = {Journal of Machine Learning Research},
  year    = {2014},
  volume  = {15},
  number  = {56},
  pages   = {1929--1958},
  url     = {http://jmlr.org/papers/v15/srivastava14a.html}
}


@InProceedings{pmlr-v48-gal16,
  title = 	 {Dropout as a Bayesian Approximation: Representing Model Uncertainty in Deep Learning},
  author = 	 {Gal, Yarin and Ghahramani, Zoubin},
  booktitle = 	 {Proceedings of The 33rd International Conference on Machine Learning},
  pages = 	 {1050--1059},
  year = 	 {2016},
  editor = 	 {Balcan, Maria Florina and Weinberger, Kilian Q.},
  volume = 	 {48},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {New York, New York, USA},
  month = 	 {20--22 Jun},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v48/gal16.pdf},
  url = 	 {https://proceedings.mlr.press/v48/gal16.html},
  abstract = 	 {Deep learning tools have gained tremendous attention in applied machine learning. However such tools for regression and classification do not capture model uncertainty. In comparison, Bayesian models offer a mathematically grounded framework to reason about model uncertainty, but usually come with a prohibitive computational cost. In this paper we develop a new theoretical framework casting dropout training in deep neural networks (NNs) as approximate Bayesian inference in deep Gaussian processes. A direct result of this theory gives us tools to model uncertainty with dropout NNs – extracting information from existing models that has been thrown away so far. This mitigates the problem of representing uncertainty in deep learning without sacrificing either computational complexity or test accuracy. We perform an extensive study of the properties of dropout’s uncertainty. Various network architectures and non-linearities are assessed on tasks of regression and classification, using MNIST as an example. We show a considerable improvement in predictive log-likelihood and RMSE compared to existing state-of-the-art methods, and finish by using dropout’s uncertainty in deep reinforcement learning.}
}

@misc{alistarh2017qsgdcommunicationefficientsgdgradient,
      title={QSGD: Communication-Efficient SGD via Gradient Quantization and Encoding}, 
      author={Dan Alistarh and Demjan Grubic and Jerry Li and Ryota Tomioka and Milan Vojnovic},
      year={2017},
      eprint={1610.02132},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/1610.02132}, 
}


@InProceedings{pmlr-v28-wan13,
  title = 	 {Regularization of Neural Networks using DropConnect},
  author = 	 {Wan, Li and Zeiler, Matthew and Zhang, Sixin and Le Cun, Yann and Fergus, Rob},
  booktitle = 	 {Proceedings of the 30th International Conference on Machine Learning},
  pages = 	 {1058--1066},
  year = 	 {2013},
  editor = 	 {Dasgupta, Sanjoy and McAllester, David},
  volume = 	 {28},
  number =       {3},
  series = 	 {Proceedings of Machine Learning Research},
  address = 	 {Atlanta, Georgia, USA},
  month = 	 {17--19 Jun},
  publisher =    {PMLR},
  pdf = 	 {http://proceedings.mlr.press/v28/wan13.pdf},
  url = 	 {https://proceedings.mlr.press/v28/wan13.html},
  abstract = 	 {We introduce DropConnect, a generalization of DropOut, for regularizing large fully-connected layers within neural networks. When training with Dropout, a randomly selected subset of activations are set to zero within each layer. DropConnect instead sets a randomly selected subset of weights within the network to zero. Each unit thus receives input from a random subset of units in the previous layer. We derive a bound on the generalization performance of both Dropout and DropConnect. We then evaluate DropConnect on a range of datasets, comparing to Dropout, and show state-of-the-art results on several image recoginition benchmarks can be obtained by aggregating multiple DropConnect-trained models.}
}

@misc{chen2016trainingdeepnetssublinear,
      title={Training Deep Nets with Sublinear Memory Cost}, 
      author={Tianqi Chen and Bing Xu and Chiyuan Zhang and Carlos Guestrin},
      year={2016},
      eprint={1604.06174},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/1604.06174}, 
}

@misc{gomez2017reversibleresidualnetworkbackpropagation,
      title={The Reversible Residual Network: Backpropagation Without Storing Activations}, 
      author={Aidan N. Gomez and Mengye Ren and Raquel Urtasun and Roger B. Grosse},
      year={2017},
      eprint={1707.04585},
      archivePrefix={arXiv},
      primaryClass={cs.CV},
      url={https://arxiv.org/abs/1707.04585}, 
}

@misc{ouyang2022traininglanguagemodelsfollow,
      title={Training language models to follow instructions with human feedback}, 
      author={Long Ouyang and Jeff Wu and Xu Jiang and Diogo Almeida and Carroll L. Wainwright and Pamela Mishkin and Chong Zhang and Sandhini Agarwal and Katarina Slama and Alex Ray and John Schulman and Jacob Hilton and Fraser Kelton and Luke Miller and Maddie Simens and Amanda Askell and Peter Welinder and Paul Christiano and Jan Leike and Ryan Lowe},
      year={2022},
      eprint={2203.02155},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2203.02155}, 
}

@misc{alpaca,
  author = {Rohan Taori and Ishaan Gulrajani and Tianyi Zhang and Yann Dubois and Xuechen Li and Carlos Guestrin and Percy Liang and Tatsunori B. Hashimoto },
  title = {Stanford Alpaca: An Instruction-following LLaMA model},
  year = {2023},
  publisher = {GitHub},
  journal = {GitHub repository},
  howpublished = {\url{https://github.com/tatsu-lab/stanford_alpaca}},
}

@online{DatabricksBlog2023DollyV2,
    author    = {Mike Conover and Matt Hayes and Ankit Mathur and Jianwei Xie and Jun Wan and Sam Shah and Ali Ghodsi and Patrick Wendell and Matei Zaharia and Reynold Xin},
    title     = {Free Dolly: Introducing the World's First Truly Open Instruction-Tuned LLM},
    year      = {2023},
    url       = {https://www.databricks.com/blog/2023/04/12/dolly-first-open-commercially-viable-instruction-tuned-llm},
    urldate   = {2023-06-30}
}

@misc{zeng2023agenttuning,
      title={AgentTuning: Enabling Generalized Agent Abilities for LLMs}, 
      author={Aohan Zeng and Mingdao Liu and Rui Lu and Bowen Wang and Xiao Liu and Yuxiao Dong and Jie Tang},
      year={2023},
      eprint={2310.12823},
      archivePrefix={arXiv},
      primaryClass={cs.CL}
}

@misc{wang2023selfinstructaligninglanguagemodels,
      title={Self-Instruct: Aligning Language Models with Self-Generated Instructions}, 
      author={Yizhong Wang and Yeganeh Kordi and Swaroop Mishra and Alisa Liu and Noah A. Smith and Daniel Khashabi and Hannaneh Hajishirzi},
      year={2023},
      eprint={2212.10560},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2212.10560}, 
}

@misc{touvron2023llamaopenefficientfoundation,
      title={LLaMA: Open and Efficient Foundation Language Models}, 
      author={Hugo Touvron and Thibaut Lavril and Gautier Izacard and Xavier Martinet and Marie-Anne Lachaux and Timothée Lacroix and Baptiste Rozière and Naman Goyal and Eric Hambro and Faisal Azhar and Aurelien Rodriguez and Armand Joulin and Edouard Grave and Guillaume Lample},
      year={2023},
      eprint={2302.13971},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2302.13971}, 
}

@misc{shazeer2019fasttransformerdecodingwritehead,
      title={Fast Transformer Decoding: One Write-Head is All You Need}, 
      author={Noam Shazeer},
      year={2019},
      eprint={1911.02150},
      archivePrefix={arXiv},
      primaryClass={cs.NE},
      url={https://arxiv.org/abs/1911.02150}, 
}

@misc{su2023roformerenhancedtransformerrotary,
      title={RoFormer: Enhanced Transformer with Rotary Position Embedding}, 
      author={Jianlin Su and Yu Lu and Shengfeng Pan and Ahmed Murtadha and Bo Wen and Yunfeng Liu},
      year={2023},
      eprint={2104.09864},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2104.09864}, 
}

@misc{shazeer2020gluvariantsimprovetransformer,
      title={GLU Variants Improve Transformer}, 
      author={Noam Shazeer},
      year={2020},
      eprint={2002.05202},
      archivePrefix={arXiv},
      primaryClass={cs.LG},
      url={https://arxiv.org/abs/2002.05202}, 
}

@misc{gemmateam2025gemma3technicalreport,
      title={Gemma 3 Technical Report}, 
      author={Gemma Team and Aishwarya Kamath and Johan Ferret and Shreya Pathak and Nino Vieillard and Ramona Merhej and Sarah Perrin and Tatiana Matejovicova and Alexandre Ramé and Morgane Rivière and Louis Rouillard and Thomas Mesnard and Geoffrey Cideron and Jean-bastien Grill and Sabela Ramos and Edouard Yvinec and Michelle Casbon and Etienne Pot and Ivo Penchev and Gaël Liu and Francesco Visin and Kathleen Kenealy and Lucas Beyer and Xiaohai Zhai and Anton Tsitsulin and Robert Busa-Fekete and Alex Feng and Noveen Sachdeva and Benjamin Coleman and Yi Gao and Basil Mustafa and Iain Barr and Emilio Parisotto and David Tian and Matan Eyal and Colin Cherry and Jan-Thorsten Peter and Danila Sinopalnikov and Surya Bhupatiraju and Rishabh Agarwal and Mehran Kazemi and Dan Malkin and Ravin Kumar and David Vilar and Idan Brusilovsky and Jiaming Luo and Andreas Steiner and Abe Friesen and Abhanshu Sharma and Abheesht Sharma and Adi Mayrav Gilady and Adrian Goedeckemeyer and Alaa Saade and Alex Feng and Alexander Kolesnikov and Alexei Bendebury and Alvin Abdagic and Amit Vadi and András György and André Susano Pinto and Anil Das and Ankur Bapna and Antoine Miech and Antoine Yang and Antonia Paterson and Ashish Shenoy and Ayan Chakrabarti and Bilal Piot and Bo Wu and Bobak Shahriari and Bryce Petrini and Charlie Chen and Charline Le Lan and Christopher A. Choquette-Choo and CJ Carey and Cormac Brick and Daniel Deutsch and Danielle Eisenbud and Dee Cattle and Derek Cheng and Dimitris Paparas and Divyashree Shivakumar Sreepathihalli and Doug Reid and Dustin Tran and Dustin Zelle and Eric Noland and Erwin Huizenga and Eugene Kharitonov and Frederick Liu and Gagik Amirkhanyan and Glenn Cameron and Hadi Hashemi and Hanna Klimczak-Plucińska and Harman Singh and Harsh Mehta and Harshal Tushar Lehri and Hussein Hazimeh and Ian Ballantyne and Idan Szpektor and Ivan Nardini and Jean Pouget-Abadie and Jetha Chan and Joe Stanton and John Wieting and Jonathan Lai and Jordi Orbay and Joseph Fernandez and Josh Newlan and Ju-yeong Ji and Jyotinder Singh and Kat Black and Kathy Yu and Kevin Hui and Kiran Vodrahalli and Klaus Greff and Linhai Qiu and Marcella Valentine and Marina Coelho and Marvin Ritter and Matt Hoffman and Matthew Watson and Mayank Chaturvedi and Michael Moynihan and Min Ma and Nabila Babar and Natasha Noy and Nathan Byrd and Nick Roy and Nikola Momchev and Nilay Chauhan and Noveen Sachdeva and Oskar Bunyan and Pankil Botarda and Paul Caron and Paul Kishan Rubenstein and Phil Culliton and Philipp Schmid and Pier Giuseppe Sessa and Pingmei Xu and Piotr Stanczyk and Pouya Tafti and Rakesh Shivanna and Renjie Wu and Renke Pan and Reza Rokni and Rob Willoughby and Rohith Vallu and Ryan Mullins and Sammy Jerome and Sara Smoot and Sertan Girgin and Shariq Iqbal and Shashir Reddy and Shruti Sheth and Siim Põder and Sijal Bhatnagar and Sindhu Raghuram Panyam and Sivan Eiger and Susan Zhang and Tianqi Liu and Trevor Yacovone and Tyler Liechty and Uday Kalra and Utku Evci and Vedant Misra and Vincent Roseberry and Vlad Feinberg and Vlad Kolesnikov and Woohyun Han and Woosuk Kwon and Xi Chen and Yinlam Chow and Yuvein Zhu and Zichuan Wei and Zoltan Egyed and Victor Cotruta and Minh Giang and Phoebe Kirk and Anand Rao and Kat Black and Nabila Babar and Jessica Lo and Erica Moreira and Luiz Gustavo Martins and Omar Sanseviero and Lucas Gonzalez and Zach Gleicher and Tris Warkentin and Vahab Mirrokni and Evan Senter and Eli Collins and Joelle Barral and Zoubin Ghahramani and Raia Hadsell and Yossi Matias and D. Sculley and Slav Petrov and Noah Fiedel and Noam Shazeer and Oriol Vinyals and Jeff Dean and Demis Hassabis and Koray Kavukcuoglu and Clement Farabet and Elena Buchatskaya and Jean-Baptiste Alayrac and Rohan Anil and Dmitry and Lepikhin and Sebastian Borgeaud and Olivier Bachem and Armand Joulin and Alek Andreev and Cassidy Hardin and Robert Dadashi and Léonard Hussenot},
      year={2025},
      eprint={2503.19786},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/2503.19786}, 
}

@Misc{peft,
  title =        {{PEFT}: State-of-the-art Parameter-Efficient Fine-Tuning methods},
  author =       {Sourab Mangrulkar and Sylvain Gugger and Lysandre Debut and Younes Belkada and Sayak Paul and Benjamin Bossan},
  howpublished = {\url{https://github.com/huggingface/peft}},
  year =         {2022}
}

@article{article_prun,
author = {Wang, Jiajun},
year = {2023},
month = {10},
pages = {152-158},
title = {Research on pruning optimization techniques for neural networks},
volume = {19},
journal = {Applied and Computational Engineering},
doi = {10.54254/2755-2721/19/20231025}
}

@misc{reimers2019sentencebertsentenceembeddingsusing,
      title={Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks}, 
      author={Nils Reimers and Iryna Gurevych},
      year={2019},
      eprint={1908.10084},
      archivePrefix={arXiv},
      primaryClass={cs.CL},
      url={https://arxiv.org/abs/1908.10084}, 
}