Add support for QQP dataset with unit tests

docs/source/datasets.rst

@@ -62,6 +62,11 @@ MRPC
 
 .. autofunction:: MRPC
 
+QQP
+~~~~
+
+.. autofunction:: QQP
+
 SogouNews
 ~~~~~~~~~
 

test/datasets/test_qqp.py

@@ -0,0 +1,61 @@
+import os
+from unittest.mock import patch
+
+from torchtext.datasets.qqp import QQP
+
+from ..common.case_utils import TempDirMixin, zip_equal, get_random_unicode
+from ..common.torchtext_test_case import TorchtextTestCase
+
+
+def _get_mock_dataset(root_dir):
+    """
+    root_dir: directory to the mocked dataset
+    """
+    base_dir = os.path.join(root_dir, "QQP")
+    os.makedirs(base_dir, exist_ok=True)
+
+    seed = 1
+    file_name = "quora_duplicate_questions.tsv"
+    txt_file = os.path.join(base_dir, file_name)
+    mocked_data = []
+    print(txt_file)
+    with open(txt_file, "w", encoding="utf-8") as f:
+        f.write("id\tqid1\tqid2\tquestion1\tquestion2\tis_duplicate\n")
+        for i in range(5):
+            label = seed % 2
+            rand_string_1 = get_random_unicode(seed)
+            rand_string_2 = get_random_unicode(seed + 1)
+            dataset_line = (label, rand_string_1, rand_string_2)
+            # append line to correct dataset split
+            mocked_data.append(dataset_line)
+            f.write(f"{i}\t{i}\t{i}\t{rand_string_1}\t{rand_string_2}\t{label}\n")
+            seed += 1
+
+    return mocked_data
+
+
+class TestQQP(TempDirMixin, TorchtextTestCase):
+    root_dir = None
+    samples = []
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        cls.root_dir = cls.get_base_temp_dir()
+        print(cls.root_dir)
+        cls.samples = _get_mock_dataset(cls.root_dir)
+        cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True)
+        cls.patcher.start()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.patcher.stop()
+        super().tearDownClass()
+
+    def test_qqp(self):
+        dataset = QQP(root=self.root_dir)
+
+        samples = list(dataset)
+        expected_samples = self.samples
+        for sample, expected_sample in zip_equal(samples, expected_samples):
+            self.assertEqual(sample, expected_sample)

torchtext/datasets/__init__.py

@@ -14,6 +14,7 @@
 from .mrpc import MRPC
 from .multi30k import Multi30k
 from .penntreebank import PennTreebank
+from .qqp import QQP
 from .sogounews import SogouNews
 from .squad1 import SQuAD1
 from .squad2 import SQuAD2
@@ -40,6 +41,7 @@
     "MRPC": MRPC,
     "Multi30k": Multi30k,
     "PennTreebank": PennTreebank,
+    "QQP": QQP,
     "SQuAD1": SQuAD1,
     "SQuAD2": SQuAD2,
     "SogouNews": SogouNews,

torchtext/datasets/qqp.py

@@ -0,0 +1,53 @@
+import os
+
+from torchtext._internal.module_utils import is_module_available
+from torchtext.data.datasets_utils import _create_dataset_directory
+
+if is_module_available("torchdata"):
+    from torchdata.datapipes.iter import FileOpener, IterableWrapper
+    from torchtext._download_hooks import HttpReader
+
+URL = "http://qim.fs.quoracdn.net/quora_duplicate_questions.tsv"
+
+MD5 = "b6d5672bd9dc1e66ab2bb020ebeafb8d"
+
+_PATH = "quora_duplicate_questions.tsv"
+
+NUM_LINES = {"train": 404290}
+
+DATASET_NAME = "QQP"
+
+
+@_create_dataset_directory(dataset_name=DATASET_NAME)
+def QQP(root: str):
+    """QQP dataset
+    For additional details refer to https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs
+
+    Args:
+        root: Directory where the datasets are saved. Default: os.path.expanduser('~/.torchtext/cache')
+
+    :returns: DataPipe that yields rows from QQP dataset (label (int), question1 (str), question2 (str))
+    :rtype: (int, str, str)
+    """
+    if not is_module_available("torchdata"):
+        raise ModuleNotFoundError(
+            "Package `torchdata` not found. Please install following instructions at `https://github.com/pytorch/data`"
+        )
+
+    def _filepath_fn(_=None):
+        return os.path.join(root, _PATH)
+
+    def _modify_res(x):
+        return (int(x[-1]), x[3], x[4])
+
+    url_dp = IterableWrapper([URL])
+    cache_dp = url_dp.on_disk_cache(
+        filepath_fn=_filepath_fn,
+        hash_dict={_filepath_fn(): MD5},
+        hash_type="md5",
+    )
+    cache_dp = HttpReader(cache_dp).end_caching(mode="wb", same_filepath_fn=True)
+    cache_dp = FileOpener(cache_dp, encoding="utf-8")
+    # some context stored at top of the file needs to be removed
+    parsed_data = cache_dp.parse_csv(skip_lines=1, delimiter="\t").map(_modify_res)
+    return parsed_data.shuffle().set_shuffle(False).sharding_filter()