new version of dl-hard qrels for msmarco-document

Author: seanmacavaney

ir_datasets/datasets/msmarco_document.py

@@ -1,6 +1,6 @@
 from typing import NamedTuple
 import ir_datasets
-from ir_datasets.util import Cache, DownloadConfig, GzipExtract, Lazy
+from ir_datasets.util import Cache, DownloadConfig, GzipExtract, Lazy, Migrator
 from ir_datasets.datasets.base import Dataset, YamlDocumentation, FilteredQueries, FilteredScoredDocs, FilteredQrels
 from ir_datasets.formats import TrecDocs, TsvQueries, TrecQrels, TrecScoredDocs
 from ir_datasets.datasets.msmarco_passage import DUA, QRELS_DEFS, DL_HARD_QIDS_BYFOLD, DL_HARD_QIDS
@@ -114,14 +114,17 @@ def _init():
     )
 
     # DL-Hard
+    dl_hard_qrels_migrator = Migrator(base_path/'trec-dl-hard'/'irds_version.txt', 'v2',
+        affected_files=[base_path/'trec-dl-hard'/'qrels'],
+        message='Updating trec-dl-hard qrels')
     hard_qids = Lazy(lambda: DL_HARD_QIDS)
     dl_hard_base_queries = TsvQueries([
             Cache(GzipExtract(dlc['trec-dl-2019/queries']), base_path/'trec-dl-2019/queries.tsv'),
             Cache(GzipExtract(dlc['trec-dl-2020/queries']), base_path/'trec-dl-2020/queries.tsv')], namespace='msmarco', lang='en')
     subsets['trec-dl-hard'] = Dataset(
         collection,
         FilteredQueries(dl_hard_base_queries, hard_qids),
-        TrecQrels(dlc['trec-dl-hard/qrels'], TREC_DL_QRELS_DEFS),
+        dl_hard_qrels_migrator(TrecQrels(dlc['trec-dl-hard/qrels'], TREC_DL_QRELS_DEFS)),
         documentation('trec-dl-hard')
     )
     hard_qids = Lazy(lambda: DL_HARD_QIDS_BYFOLD['1'])

ir_datasets/etc/downloads.yaml

@@ -1028,7 +1028,7 @@ msmarco-document:
     cache_path: 'orcas/ms.run.gz'
   trec-dl-hard/qrels:
     url: 'https://raw.githubusercontent.com/grill-lab/DL-Hard/main/dataset/dl_hard-doc.qrels'
-    expected_md5: 'da2366aef13c11b352fe4587d96ee9f8'
+    expected_md5: '06dfe71d497e081a7c4c1294979edb7d'
     cache_path: 'trec-dl-hard/qrels'
 
 

test/integration/msmarco_document.py

@@ -59,10 +59,10 @@ def test_msmarco_document_queries(self):
             9: GenericQuery('1105792', 'define: geon'),
             44: GenericQuery('997622', 'where is the show shameless filmed'),
         })
-        self._test_queries('msmarco-document/trec-dl-hard', count=49, items={
+        self._test_queries('msmarco-document/trec-dl-hard', count=50, items={
             0: GenericQuery('1108939', 'what slows down the flow of blood'),
             9: GenericQuery('451602', "medicare's definition of mechanical ventilation"),
-            48: GenericQuery('88495', 'causes of stroke?'),
+            49: GenericQuery('88495', 'causes of stroke?'),
         })
         self._test_queries('msmarco-document/trec-dl-hard/fold1', count=10, items={
             0: GenericQuery('966413', 'where are the benefits of cinnamon as a supplement?'),
@@ -80,9 +80,9 @@ def test_msmarco_document_queries(self):
             0: GenericQuery('1108100', 'what type of movement do bacteria exhibit?'),
             9: GenericQuery('88495', 'causes of stroke?'),
         })
-        self._test_queries('msmarco-document/trec-dl-hard/fold5', count=9, items={
+        self._test_queries('msmarco-document/trec-dl-hard/fold5', count=10, items={
             0: GenericQuery('190044', 'foods to detox liver naturally'),
-            8: GenericQuery('877809', 'what metal are hip replacements made of'),
+            9: GenericQuery('877809', 'what metal are hip replacements made of'),
         })
 
 
@@ -122,10 +122,10 @@ def test_msmarco_document_qrels(self):
             9: TrecQrel('42255', 'D1168483', 0, '0'),
             9097: TrecQrel('1136962', 'D96742', 0, '0'),
         })
-        self._test_qrels('msmarco-document/trec-dl-hard', count=8540, items={
+        self._test_qrels('msmarco-document/trec-dl-hard', count=8544, items={
             0: TrecQrel('1117817', 'D192188', 0, 'Q0'),
             9: TrecQrel('801118', 'D579461', 1, 'Q0'),
-            8539: TrecQrel('315637', 'D655701', 3, 'Q0'),
+            8543: TrecQrel('273695', 'D736968', 2, 'Q0'),
         })
         self._test_qrels('msmarco-document/trec-dl-hard/fold1', count=1557, items={
             0: TrecQrel('1056204', 'D1891649', 1, 'Q0'),
@@ -147,10 +147,10 @@ def test_msmarco_document_qrels(self):
             9: TrecQrel('87452', 'D1000458', 2, 'Q0'),
             1053: TrecQrel('1108100', 'D3318246', 3, 'Q0'),
         })
-        self._test_qrels('msmarco-document/trec-dl-hard/fold5', count=4110, items={
+        self._test_qrels('msmarco-document/trec-dl-hard/fold5', count=4114, items={
             0: TrecQrel('489204', 'D1002646', 0, 'Q0'),
             9: TrecQrel('489204', 'D1025842', 1, 'Q0'),
-            4109: TrecQrel('190044', 'D910793', 3, 'Q0'),
+            4113: TrecQrel('273695', 'D736968', 2, 'Q0'),
         })
 
     def test_msmarco_document_scoreddocs(self):