%Aigaion2 BibTeX export from Idiap Publications
%Thursday 06 August 2026 04:21:55 PM

@PHDTHESIS{He_THESIS_2026,
                      author = {He, Mutian},
                    projects = {Idiap},
                       month = jun,
                       title = {Foundation Models for Spoken Language Understanding : Capability, Efficiency, and Application},
                        year = {2026},
                      school = {EPFL},
                    abstract = {At a rather high level, this thesis is motivated by the goal of inference of job suitability from
audio recordings of interviews and work samples. This in turn defines the technical goal of
analysis and understanding of long audio recordings in French with a limited amount of data.
Of particular relevance, the emergence of large-scale pretrained generalist models, including
large language models (LLMs), is reshaping the landscape of speech processing. Such models
offer an effective solution to our goal, whilst several issues remain: (i) how capable such
models are for speech, (ii) how to best apply or adapt them to the target tasks in the speech
domain, and (iii) how to efficiently process long speech sequences that often exceed text by
an order of magnitude. We explore them through a series of evaluations, model architecture
design, and training techniques, as presented in this thesis.
We begin by studying an auxiliary supervision stage for pretrained speech models prior to
task-specific fine-tuning. We show that speech translation mid-training substantially improves
semantic understanding and few-shot cross-lingual transfer for models such as wav2vec2.
We further demonstrate that preserving pretraining knowledge via Bayesian regularizers can
improve downstream performance.
With the emergence of LLMs starting from ChatGPT, the next part of the thesis evaluates the
role of ChatGPT and other text-only LMs on spoken language understanding. We demonstrate
their strong zero- and few-shot performance, approaching supervised baselines, and confirm
the emergent in-context learning behavior unique in LLMs. Meanwhile we also identify
limitations of such models including sensitivity to ASR errors and degradation on under-
specified and vague tasks.
Third, we address the computational bottleneck of long-form speech modeling arising from
high sampling rates and the information-sparse nature of speech, compounded by the O(N 2 )
time and O(N ) space demands of transformer-based models. Although efficient sequence
models such as linear attention are advancing rapidly, few pretrained speech models exist
in these architectures. To bridge this gap without re-pretraining, we propose a systematic
layer-wise distillation framework that converts pretrained speech and language Transformers
into target efficient architectures (e.g., Linformer, Mamba) during fine-tuning on downstream
speech or NLP data. We also develop distillation strategies that improve downstream perfor-
mance by better preserving pretraining knowledge.
Fourth, we analyze “forgetfulness” in linear attention that current linear attention models
gradually lose information of distant past, due to limited recurrent state or memory capacity.
We propose hybrid linear–sparse mechanisms that grant direct access to distant tokens without sacrificing efficiency. Particularly, we investigated query-aware sparse attention mechanisms,
as well as a novel contextualized learnable token eviction approach that maintains linear
attention’s O(1) space appeal. Empirical studies confirmed the advantage of our methods
over existing hybrid baselines (e.g. with sliding-window attention) in long-context scenarios,
without compromising efficiency thanks to our hardware-efficient implementation.
In the end, we apply these methods to long French interviews recording. We show that
pretrained speech models can be adapted to identify storytelling, an important aspect of
interview evaluation. We also show the critical role of contextual information in this task.
In summary, the thesis contributes to the advancement of speech and language understanding
through improved model architecture, pretraining, and adaptation techniques. In particular,
we investigate efficient attention approaches including linear attention, sparse attention, and
their hybrids, which can be essential for long-context scenarios.},
                         pdf = {https://publications.idiap.ch/attachments/papers/2026/He_THESIS_2026.pdf}
}