moved clirmatrix_downloads.json to itself be a downloaded file

Author: seanmacavaney

MANIFEST.in

@@ -1,5 +1,4 @@
 include ir_datasets/docs/*.yaml
 include ir_datasets/etc/*.yaml
-include ir_datasets/etc/*.json
 include requirements.txt
 include LICENSE

ir_datasets/datasets/clirmatrix.py

@@ -2,7 +2,7 @@
 from pathlib import Path
 from typing import NamedTuple
 import ir_datasets
-from ir_datasets.util import GzipExtract, DownloadConfig_CM
+from ir_datasets.util import GzipExtract, DownloadConfig, _DownloadConfig
 from ir_datasets.datasets.base import Dataset, YamlDocumentation
 from ir_datasets.formats import TsvDocs, CLIRMatrixQueries, CLIRMatrixQrels
 
@@ -29,7 +29,9 @@ def _init():
     base_path = ir_datasets.util.home_path()/NAME
 
     def _dlc_init():
-        return DownloadConfig_CM
+        dlc = DownloadConfig.context(NAME, base_path)
+        clirmatrix_dlc = _DownloadConfig(dlc['downloads'].path(), parser='json')
+        return clirmatrix_dlc
 
     _dlc = ir_datasets.util.Lazy(_dlc_init)
 

ir_datasets/etc/clirmatrix_downloads.json

@@ -105,6 +105,13 @@ clinicaltrials:
     cache_path: 'trec-pm-2019/queries.xml'
 
 
+clirmatrix:
+  downloads:
+    url: 'https://macavaney.us/clirmatrix_downloads.json' # TODO: move this to JHU server?
+    expected_md5: '9e70cd85ec45caa8c16061c42d1ce9b8'
+    cache_path: 'clirmatrix_downloads.json'
+
+
 clueweb09:
   docs:
     instructions: 'ClueWeb09 is available by hard drives from CMU here: <https://lemurproject.org/clueweb09/>

ir_datasets/etc/downloads.yaml

@@ -7,7 +7,7 @@
 from pathlib import Path
 from .. import log
 from .fileio import IterStream, Cache, TarExtract, TarExtractAll, RelativePath, GzipExtract, ZipExtract, ZipExtractCache, StringFile, ReTar, Bz2Extract
-from .download import Download, DownloadConfig, BaseDownload, RequestsDownload, LocalDownload, DownloadConfig_CM
+from .download import Download, DownloadConfig, BaseDownload, RequestsDownload, LocalDownload, _DownloadConfig
 from .hash import HashVerificationError, HashVerifier, HashStream
 from .registry import Registry
 

ir_datasets/util/__init__.py

@@ -232,7 +232,10 @@ def __init__(self, file=None, base_path=None, contents=None, dua=None, parser="y
 
     def contents(self):
         if self._contents is None:
-            data = pkgutil.get_data('ir_datasets', self._file)
+            if self._file.startswith('/'):
+                data = open(self._file).read()
+            else:
+                data = pkgutil.get_data('ir_datasets', self._file)
             if self._parser == "json":
                 json = ir_datasets.lazy_libs.json()
                 self._contents = json.loads(data)
@@ -284,4 +287,3 @@ def __getitem__(self, key):
 
 
 DownloadConfig = _DownloadConfig(file='etc/downloads.yaml')
-DownloadConfig_CM = _DownloadConfig(file='etc/clirmatrix_downloads.json', parser="json")

ir_datasets/util/download.py

@@ -22,7 +22,7 @@
         'console_scripts': ['ir_datasets=ir_datasets:main_cli'],
     },
     package_data={
-        'ir_datasets': glob('docs/*.yaml') + glob('etc/*.yaml') + glob('etc/*.json'),
+        'ir_datasets': glob('docs/*.yaml') + glob('etc/*.yaml'),
         '': ['requirements.txt', 'LICENSE'],
     },
 )