@inproceedings{26bdb47ef4b149a58f75a4f08e80f85c,
title = "Penn & BGU BabyBERTa+ for Strict-Small BabyLM Challenge",
abstract = "The BabyLM Challenge aims at pre-training a language model on a small-scale dataset of inputs intended for children. In this work, we adapted the architecture and masking policy of BabyBERTa (Huebner et al., 2021) to solve the strict-small track of the BabyLM challenge. Our model, Penn & BGU BabyBERTa+, was pre-trained and evaluated on the three benchmarks of the BabyLM Challenge. Experimental results indicate that our model achieves higher or comparable performance in predicting 17 grammatical phenomena, compared to the RoBERTa baseline.",
author = "Yahan Yang and Insup Lee and Elior Sulem and Dan Roth",
note = "Publisher Copyright: {\textcopyright} 2023 Association for Computational Linguistics.; BabyLM Challenge at the 27th Conference on Computational Natural Language Learning, CoNLL 2023 ; Conference date: 06-12-2023 Through 07-12-2023",
year = "2023",
month = jan,
day = "1",
language = "American English",
series = "CoNLL 2023 - BabyLM Challenge at the 27th Conference on Computational Natural Language Learning, Proceedings",
publisher = "Association for Computational Linguistics (ACL)",
pages = "86--88",
editor = "Alex Warstadt and Aaron Mueller and Leshem Choshen and Ethan Wilcox and Chengxu Zhuang and Juan Ciro and Rafael Mosquera and Bhargavi Paranjabe and Adina Williams and Tal Linzen and Ryan Cotterell",
booktitle = "CoNLL 2023 - BabyLM Challenge at the 27th Conference on Computational Natural Language Learning, Proceedings",
address = "United States",
}