re-parameterization of Multi30K dataset

docs/source/datasets.rst

@@ -101,11 +101,6 @@ PennTreebank
 Machine Translation
 ^^^^^^^^^^^^^^^^^^^
 
-Multi30k
-~~~~~~~~
-
-.. autofunction:: Multi30k
-
 
 IWSLT2016
 ~~~~~~~~~

docs/source/experimental_datasets_raw.rst

@@ -0,0 +1,38 @@
+torchtext.experimental.datasets.raw
+===================================
+
+.. currentmodule:: torchtext.experimental.datasets.raw
+
+General use cases are as follows: ::
+
+
+    # import datasets
+    from torchtext.experimental.datasets.raw import Multi30k
+
+    train_iter = Multi30k(split='train')
+
+    def tokenize(label, line):
+        return line.split()
+
+    tokens_src = []
+    tokens_tgt = []
+
+    for line in train_iter:
+        src, tgt = line
+        tokens_src += tokenize(src)
+        tokens_tgt += tokenize(tgt)
+
+The following datasets are available:
+
+.. contents:: Datasets
+    :local:
+
+
+Machine Translation
+^^^^^^^^^^^^^^^^^^^
+
+Multi30k
+~~~~~~~~
+
+.. autofunction:: Multi30k
+

docs/source/index.rst

@@ -40,6 +40,7 @@ popular datasets for natural language.
    torchtext.vocab <vocab>
    torchtext.utils <utils>
    experimental_datasets
+   experimental_datasets_raw
    experimental_transforms
    experimental_vectors
    experimental_vocab

test/data/test_builtin_datasets.py

@@ -174,12 +174,19 @@ def test_raw_text_classification(self, info):
             return
 
         split = info['split']
-        data_iter = torchtext.datasets.DATASETS[dataset_name](split=split)
+        if dataset_name == "Multi30k":
+            data_iter = torchtext.experimental.datasets.raw.DATASETS[dataset_name](split=split)
+        else:
+            data_iter = torchtext.datasets.DATASETS[dataset_name](split=split)
         self.assertEqual(len(data_iter), info['NUM_LINES'])
         self.assertEqual(next(data_iter), info['first_line'])
         if dataset_name == "AG_NEWS":
             self.assertEqual(torchtext.datasets.URLS[dataset_name][split], info['URL'])
             self.assertEqual(torchtext.datasets.MD5[dataset_name][split], info['MD5'])
+        elif dataset_name == "Multi30k":
+            self.assertEqual(torchtext.experimental.datasets.raw.URLS[dataset_name][split], info['URL'])
+            self.assertEqual(torchtext.experimental.datasets.raw.MD5[dataset_name][split], info['MD5'])
+
         else:
             self.assertEqual(torchtext.datasets.URLS[dataset_name], info['URL'])
             self.assertEqual(torchtext.datasets.MD5[dataset_name], info['MD5'])
@@ -352,7 +359,7 @@ def test_multi30k(self):
                          [18, 24, 1168, 807, 16, 56, 83, 335, 1338])
 
         # Add test for the subset of the standard datasets
-        train_iter, valid_iter = torchtext.datasets.Multi30k(split=('train', 'valid'))
+        train_iter, valid_iter = torchtext.experimental.datasets.raw.Multi30k(split=('train', 'valid'))
         self._helper_test_func(len(train_iter), 29000, ' '.join(next(train_iter)),
                                ' '.join(['Zwei junge weiße Männer sind im Freien in der Nähe vieler Büsche.\n',
                                          'Two young, White males are outside near many bushes.\n']))

torchtext/datasets/__init__.py

@@ -8,7 +8,6 @@
 from .imdb import IMDB
 from .iwslt2016 import IWSLT2016
 from .iwslt2017 import IWSLT2017
-from .multi30k import Multi30k
 from .penntreebank import PennTreebank
 from .sogounews import SogouNews
 from .squad1 import SQuAD1
@@ -31,7 +30,6 @@
     'IMDB': IMDB,
     'IWSLT2016': IWSLT2016,
     'IWSLT2017': IWSLT2017,
-    'Multi30k': Multi30k,
     'PennTreebank': PennTreebank,
     'SQuAD1': SQuAD1,
     'SQuAD2': SQuAD2,

torchtext/experimental/datasets/raw/__init__.py

@@ -1,8 +1,10 @@
 import importlib
 from .wmtnewscrawl import WMTNewsCrawl
+from .multi30k import Multi30k
 
 DATASETS = {
     'WMTNewsCrawl': WMTNewsCrawl,
+    'Multi30k': Multi30k,
 }
 
 URLS = {}