%Aigaion2 BibTeX export from Idiap Publications
%Sunday 14 July 2024 03:16:39 AM

         author = {Valentino, Marco and Mendes, Deborah and Thayaparan, Mokanarangan and Freitas, Andre and Ustalov, Dmitry},
       projects = {Idiap},
          title = {TextGraphs 2022 Shared Task on Natural Language Premise Selection},
      booktitle = {Proceedings of TextGraphs-16: Graph-based Methods for Natural Language Processing},
           year = {2022},
            url = {https://aclanthology.org/2022.textgraphs-1.11},
       abstract = {The Shared Task on Natural Language Premise Selection (NLPS) asks participants to retrieve the set of premises that are most likely to be useful for proving a given mathematical statement from a supporting knowledge base. While previous editions of the TextGraphs shared tasks series targeted multi-hop inference for explanation regeneration in the context of science questions (Thayaparan et al., 2021; Jansen and Ustalov, 2020, 2019), NLPS aims to assess the ability of state-of-the-art approaches to operate on a mixture of natural and mathematical language and model complex multi-hop reasoning dependencies between statements. To this end, this edition of the shared task makes use of a large set of approximately 21k mathematical statements extracted from the PS-ProofWiki dataset (Ferreira and Freitas, 2020a). In this summary paper, we present the results of the 1st edition of the NLPS task, providing a description of the evaluation data, and the participating systems. Additionally, we perform a detailed analysis of the results, evaluating various aspects involved in mathematical language processing and multi-hop inference. The best-performing system achieved a MAP of 15.39, improving the performance of a TF-IDF baseline by approximately 3.0 MAP.}