Add unicode generation to IWSLT tests (followup to #1608) (#1642)

* meaningless change to make XML well formed (though it is not important).

* Remove closing doc tag from xml list. Update how close tags are created

Co-authored-by: Elijah Rippeth <[email protected]>
Co-authored-by: nayef211 <[email protected]>

Author: 3 people

test/datasets/test_iwslt2016.py

@@ -39,15 +39,15 @@ def _generate_uncleaned_train():
         "<title",
         "<speaker",
         "<doc",
-        "</doc",
     ]
     for i in range(100):
         rand_string = " ".join(random.choice(string.ascii_letters) for i in range(10))
         # With a 10% change, add one of the XML tags which is cleaned
         # to ensure cleaning happens appropriately
         if random.random() < 0.1:
             open_tag = random.choice(xml_tags) + ">"
-            close_tag = "</" + open_tag[1:] + ">"
+            # Open tag already contains the closing >
+            close_tag = "</" + open_tag[1:]
             file_contents.append(open_tag + rand_string + close_tag)
         else:
             examples.append(rand_string + "\n")

test/datasets/test_iwslt2017.py

@@ -36,15 +36,15 @@ def _generate_uncleaned_train():
         "<title",
         "<speaker",
         "<doc",
-        "</doc",
     ]
     for i in range(100):
         rand_string = " ".join(random.choice(string.ascii_letters) for i in range(10))
         # With a 10% change, add one of the XML tags which is cleaned
         # to ensure cleaning happens appropriately
         if random.random() < 0.1:
             open_tag = random.choice(xml_tags) + ">"
-            close_tag = "</" + open_tag[1:] + ">"
+            # Open tag already contains the closing >
+            close_tag = "</" + open_tag[1:]
             file_contents.append(open_tag + rand_string + close_tag)
         else:
             examples.append(rand_string + "\n")