@inproceedings{06db7e9da4074e47bd42071491419b06,
title = "CDLM: Cross-Document Language Modeling",
abstract = "We introduce a new pretraining approach geared for multi-document language modeling, incorporating two key ideas into the masked language modeling self-supervised objective. First, instead of considering documents in isolation, we pretrain over sets of multiple related documents, encouraging the model to learn cross-document relationships. Second, we improve over recent long-range transformers by introducing dynamic global attention that has access to the entire input to predict masked tokens. We release CDLM (Cross-Document Language Model), a new general language model for multi-document setting that can be easily applied to downstream tasks. Our extensive analysis shows that both ideas are essential for the success of CDLM, and work in synergy to set new state-of-the-art results for several multi-text tasks.",
author = "Avi Caciularu and Arman Cohan and Iz Beltagy and Peters, {Matthew E.} and Arie Cattan and Ido Dagan",
note = "Publisher Copyright: {\textcopyright} 2021 Association for Computational Linguistics.; 2021 Findings of the Association for Computational Linguistics, Findings of ACL: EMNLP 2021 ; Conference date: 07-11-2021 Through 11-11-2021",
year = "2021",
doi = "10.18653/v1/2021.findings-emnlp.225",
language = "الإنجليزيّة",
series = "Findings of the Association for Computational Linguistics, Findings of ACL: EMNLP 2021",
publisher = "Association for Computational Linguistics (ACL)",
pages = "2648--2662",
editor = "Marie-Francine Moens and Xuanjing Huang and Lucia Specia and Yih, {Scott Wen-Tau}",
booktitle = "Findings of the Association for Computational Linguistics, Findings of ACL",
address = "الولايات المتّحدة",
}