@inproceedings{iranmanesh-etal-2026-taraz, title = "{TARAZ}: {P}ersian Short-Answer Question Benchmark for Cultural Evaluation of Language Models", author = "Iranmanesh, Reihaneh and Davoudi, Saeedeh and Abrishamchian, Pasha and Frieder, Ophir and Goharian, Nazli", editor = "Piperidis, Stelios and Bel, N{\'u}ria and van den Heuvel, Henk and Ide, Nancy and Krek, Simon and Toral, Antonio", booktitle = "Proceedings of the Fifteenth Language Resources and Evaluation Conference", month = may, year = "2026", address = "Palma de Mallorca, Spain", publisher = "ELRA Language Resource Association", url = "https://aclanthology.org/2026.lrec-1.411/", doi = "10.63317/2ophy9wqyzrd", pages = "5256--5267", abstract = "This paper presents a comprehensive evaluation framework for assessing the cultural competence of large language models (LLMs) in Persian. Existing Persian cultural benchmarks rely predominantly on multiple-choice formats and English-centric metrics that fail to capture Persian{'}s morphological complexity and semantic nuance. Our framework introduces a Persian-specific short-answer evaluation that combines rule-based morphological normalization with a hybrid syntactic and semantic similarity module, enabling robust soft-match scoring beyond exact string overlap. Through systematic evaluation of 15 state-of-the-art open- and closed-source models across three culturally grounded Persian datasets, we demonstrate that our hybrid evaluation improves scoring consistency by +10 compared to exact-match baselines by capturing meaning that surface-level methods cannot detect. Our human evaluation further confirms that the proposed semantic similarity metric achieves higher agreement with human judgments than LLM-based judges. We publicly release our evaluation framework, providing the first standardized benchmark for measuring cultural understanding in Persian and establishing a reproducible foundation for cross-cultural LLM evaluation research." }