%Aigaion2 BibTeX export from Idiap Publications
%Thursday 05 November 2026 04:23:20 PM

@ARTICLE{Nissim_IJCOL_2026,
                      author = {Nissim, Malvina and Croce, Danilo and Patti, Viviana and Basile, Pierpaolo and Attanasio, Giuseppe and Musacchio, Elio and Rinaldi, Matteo and Borazio, Federico and Francis, Maria and Gili, Jacopo and Scalena, Daniel and Altuna, Begona and Azurmendi, Ekhi and Basile, Valerio and Bentivogli, Luisa and Bisazza, Arianna and Bolognesi, Mariana and Brunato, Dominique and Caselli, Tommaso and Casola, Silvia and Cassese, Maria and Cettolo, Mauro and Collacciani, Claudia and De Cosmo, Leonardo and Di Buono, Maria Pia and Esuli, Andrea and Etxaniz, Julen and Ferrando, Chiara and Fidelangeli, Alessia and Frenda, Simona and Fusco, Achille and Gaido, Marco and Galassi, Andrea and Galli, Federico and Giordano, Luca and Goffetti, Mattia and Gonzalez-Dios, Itziar and Gregori, Lorenzo and Grundler, Giulia and Iannaccone, Sandro and Jiang, Chunyang and La Quatra, Moreno and Lagioia, Francesca and Lo, Soda Marem and Madeddu, Marco and Magnini, Bernardo and Manna, Raffaele and Mercorio, Fabio and Merlo, Paola and Muti, Arianna and Nastase, Vivi and Negri, Matteo and Onorati, Dario and Palmieri, Elena and Papi, Sara and Passaro, Lucia and Pensa, Giulia and Piergentili, Andrea and Poterti, Daniele and Puccetti, Giovanni and Ranaldi, Federico and Ranaldi, Leonardo and Ravelli, Andrea Amelio and Rosola, Martina and Ruzzetti, Elena Sofia and Samo, Giuseppe and Santilli, Andrea and Santin, Piera and Sarti, Gabriele and Sartor, Giovanni and Savoldi, Beatrice and Serino, Antonio and Seveso, Andrea and Siciliani, Lucia and Torroni, Paolo and Varvara, Rossella and Zaninello, Andrea and Zanollo, Asya and Zanzotto, Fabio Massimo and Zeinalipour, Kamyar and Zugarini, Andrea},
                    projects = {Idiap},
         mainresearchprogram = {AI for Everyone},
                       title = {Challenging the Abilities of Large Language Models in Italian: a Community Initiative},
                     journal = {Italian Journal of Computational Linguistics},
                      volume = {12},
                      number = {1},
                        year = {2026},
                    abstract = {The rapid progress of Large Language Models (LLMs) has transformed natural language processing and broadened its impact across research and society. Yet, systematic evaluation of these models, especially for languages beyond English, remains limited. "Challenging the Abilities of LAnguage Models in ITAlian" (CALAMITA) is a large-scale collaborative benchmarking initiative for Italian, coordinated under the Italian Association for Computational Linguistics. Unlike existing efforts that focus on leaderboards, CALAMITA foregrounds methodology: it federates more than 80 contributors from academia, industry, and the public sector to design, document, and evaluate a diverse collection of tasks, covering linguistic competence, commonsense reasoning, factual consistency, fairness, summarization, translation, and code generation. Through this process, we not only assembled a benchmark of over 20 tasks and almost 100 subtasks, but also established a centralized evaluation pipeline that supports heterogeneous datasets and metrics. We report results for four open-weight LLMs, highlighting systematic strengths and weaknesses across abilities, as well as challenges in task-specific evaluation. Beyond quantitative results, CALAMITA exposes methodological lessons: the necessity of fine-grained, task-representative metrics, the importance of harmonized pipelines, and the benefits and limitations of broad community engagement. CALAMITA is conceived as a rolling benchmark, enabling continuous integration of new tasks and models. This makes it both a resource – the most comprehensive and diverse benchmark for Italian to date – and a framework for sustainable, community-driven evaluation. We argue that this combination offers a blueprint for other languages and communities seeking inclusive and rigorous LLM evaluation practices.}
}