@inproceedings{davoudi-etal-2026-amnesia, title = "{AMNESIA}: A Large-Scale Medical Unlearning Benchmark Suite with Disease-Informed Analysis", author = "Davoudi, Saeedeh and Iranmanesh, Reihaneh and Frieder, Ophir and Goharian, Nazli", editor = "Lee, Hung-Yi and Dev, Sunipa and Li, Junyi Jessy and Elliott, Desmond", booktitle = "Proceedings of the 2026 Conference on Empirical Methods in Natural Language Processing", month = oct, year = "2026", address = "Budapest, Hungary", publisher = "Association for Computational Linguistics", url = "https://aclanthology.org/2026.emnlp-main.2597/", doi = "10.18653/2026.emnlp-main.2597", pages = "58968--58986", ISBN = "979-8-89176-459-0", abstract = "Medical knowledge is continuously evolving. This creates a need to update or selectively forget information encoded in already-trained medical LLMs. Machine unlearning aims to remove the influence of specific training data from a model without full retraining. Yet, existing unlearning benchmarks rely on synthetic or small-scale general data, leaving clinical unlearning understudied. We introduce AMNESIA, the first large-scale, open source benchmark for medical unlearning, with 70,560 question-answer pairs (QAs) from 8,820 patient notes across 11 disease categories. AMNESIA includes both factual questions testing direct recall and reasoning questions testing clinical inference. We use it to evaluate four widely used unlearning methods at both random patient and disease-level, and introduce a new metric for detecting leakage of medical terminology. We show that unlearning individual patients erodes knowledge of others with the same condition, calling for methods that can better separate patients from shared clinical knowledge." }