Skip to content

Add support for QQP dataset with unit tests #1713

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 9 commits into from
May 18, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions docs/source/datasets.rst
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@ MRPC

.. autofunction:: MRPC

QQP
~~~~

.. autofunction:: QQP

SogouNews
~~~~~~~~~

Expand Down
61 changes: 61 additions & 0 deletions test/datasets/test_qqp.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
import os
from unittest.mock import patch

from torchtext.datasets.qqp import QQP

from ..common.case_utils import TempDirMixin, zip_equal, get_random_unicode
from ..common.torchtext_test_case import TorchtextTestCase


def _get_mock_dataset(root_dir):
"""
root_dir: directory to the mocked dataset
"""
base_dir = os.path.join(root_dir, "QQP")
os.makedirs(base_dir, exist_ok=True)

seed = 1
file_name = "quora_duplicate_questions.tsv"
txt_file = os.path.join(base_dir, file_name)
mocked_data = []
print(txt_file)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we remove the print statement here

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Solved with #1734

with open(txt_file, "w", encoding="utf-8") as f:
f.write("id\tqid1\tqid2\tquestion1\tquestion2\tis_duplicate\n")
for i in range(5):
label = seed % 2
rand_string_1 = get_random_unicode(seed)
rand_string_2 = get_random_unicode(seed + 1)
dataset_line = (label, rand_string_1, rand_string_2)
# append line to correct dataset split
mocked_data.append(dataset_line)
f.write(f"{i}\t{i}\t{i}\t{rand_string_1}\t{rand_string_2}\t{label}\n")
seed += 1

return mocked_data


class TestQQP(TempDirMixin, TorchtextTestCase):
root_dir = None
samples = []

@classmethod
def setUpClass(cls):
super().setUpClass()
cls.root_dir = cls.get_base_temp_dir()
print(cls.root_dir)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove print

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Solved with #1734

cls.samples = _get_mock_dataset(cls.root_dir)
cls.patcher = patch("torchdata.datapipes.iter.util.cacheholder._hash_check", return_value=True)
cls.patcher.start()

@classmethod
def tearDownClass(cls):
cls.patcher.stop()
super().tearDownClass()

def test_qqp(self):
dataset = QQP(root=self.root_dir)

samples = list(dataset)
expected_samples = self.samples
for sample, expected_sample in zip_equal(samples, expected_samples):
self.assertEqual(sample, expected_sample)
2 changes: 2 additions & 0 deletions torchtext/datasets/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from .mrpc import MRPC
from .multi30k import Multi30k
from .penntreebank import PennTreebank
from .qqp import QQP
from .sogounews import SogouNews
from .squad1 import SQuAD1
from .squad2 import SQuAD2
Expand All @@ -40,6 +41,7 @@
"MRPC": MRPC,
"Multi30k": Multi30k,
"PennTreebank": PennTreebank,
"QQP": QQP,
"SQuAD1": SQuAD1,
"SQuAD2": SQuAD2,
"SogouNews": SogouNews,
Expand Down
53 changes: 53 additions & 0 deletions torchtext/datasets/qqp.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
import os

from torchtext._internal.module_utils import is_module_available
from torchtext.data.datasets_utils import _create_dataset_directory

if is_module_available("torchdata"):
from torchdata.datapipes.iter import FileOpener, IterableWrapper
from torchtext._download_hooks import HttpReader

URL = "http://qim.fs.quoracdn.net/quora_duplicate_questions.tsv"

MD5 = "b6d5672bd9dc1e66ab2bb020ebeafb8d"

_PATH = "quora_duplicate_questions.tsv"

NUM_LINES = {"train": 404290}

DATASET_NAME = "QQP"


@_create_dataset_directory(dataset_name=DATASET_NAME)
def QQP(root: str):
"""QQP dataset
For additional details refer to https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs

Args:
root: Directory where the datasets are saved. Default: os.path.expanduser('~/.torchtext/cache')

:returns: DataPipe that yields rows from QQP dataset (label (int), question1 (str), question2 (str))
:rtype: (int, str, str)
"""
if not is_module_available("torchdata"):
raise ModuleNotFoundError(
"Package `torchdata` not found. Please install following instructions at `https://github.com/pytorch/data`"
)

def _filepath_fn(_=None):
return os.path.join(root, _PATH)

def _modify_res(x):
return (int(x[-1]), x[3], x[4])

url_dp = IterableWrapper([URL])
cache_dp = url_dp.on_disk_cache(
filepath_fn=_filepath_fn,
hash_dict={_filepath_fn(): MD5},
hash_type="md5",
)
cache_dp = HttpReader(cache_dp).end_caching(mode="wb", same_filepath_fn=True)
cache_dp = FileOpener(cache_dp, encoding="utf-8")
# some context stored at top of the file needs to be removed
parsed_data = cache_dp.parse_csv(skip_lines=1, delimiter="\t").map(_modify_res)
return parsed_data.shuffle().set_shuffle(False).sharding_filter()