%Aigaion2 BibTeX export from Idiap Publications
%Friday 05 December 2025 04:36:16 PM

@INPROCEEDINGS{Mai_ACL2023_2023,
                      author = {Mai, Florian and Pannatier, Arnaud and Fehr, Fabio and Chen, Haolin and Marelli, Fran{\c c}ois and Fleuret, Francois and Henderson, James},
                    projects = {Idiap, LAOS, NAST, COMPBIO},
                       month = jul,
                       title = {HyperMixer: An MLP-based Low Cost Alternative to Transformers},
                   booktitle = {Proc. of the 61st Annual Meeting of the Association for Computational Linguistics},
                        year = {2023},
                       pages = {15632-15654},
                    location = {Toronto, Canada},
                organization = {Association for Computational Linguistics},
                        isbn = {978-1-959429-72-2},
                         doi = {http://dx.doi.org/10.18653/v1/2023.acl-long.871},
                    abstract = {Transformer-based architectures are the model of choice for natural language understanding, but they come at a significant cost, as they have quadratic complexity in the input length, require a lot of training data, and can be difficult to tune. In the pursuit of lower costs, we investigate simple MLP-based architectures. We find that existing architectures such as MLPMixer, which achieves token mixing through a static MLP applied to each feature independently, are too detached from the inductive biases required for natural language understanding. In this paper, we propose a simple variant, HyperMixer, which forms the token mixing MLP dynamically using hypernetworks. Empirically, we demonstrate that our model performs better than alternative MLP-based models, and on par with Transformers. In contrast to Transformers, HyperMixer achieves these results at substantially lower costs in terms of processing time, training data, and hyperparameter tuning.},
                         pdf = {https://publications.idiap.ch/attachments/papers/2023/Mai_ACL2023_2023.pdf}
}