Fixing Percentage Issue

Author: codertimo

bert_pytorch/__main__.py

@@ -10,29 +10,30 @@
 def train():
     parser = argparse.ArgumentParser()
 
-    parser.add_argument("-c", "--train_dataset", required=True, type=str)
-    parser.add_argument("-t", "--test_dataset", type=str, default=None)
-    parser.add_argument("-v", "--vocab_path", required=True, type=str)
-    parser.add_argument("-o", "--output_path", required=True, type=str)
-
-    parser.add_argument("-hs", "--hidden", type=int, default=256)
-    parser.add_argument("-l", "--layers", type=int, default=8)
-    parser.add_argument("-a", "--attn_heads", type=int, default=8)
-    parser.add_argument("-s", "--seq_len", type=int, default=20)
-
-    parser.add_argument("-b", "--batch_size", type=int, default=64)
-    parser.add_argument("-e", "--epochs", type=int, default=10)
-    parser.add_argument("-w", "--num_workers", type=int, default=5)
-
-    parser.add_argument("--with_cuda", type=bool, default=True)
-    parser.add_argument("--log_freq", type=int, default=10)
-    parser.add_argument("--corpus_lines", type=int, default=None)
-    parser.add_argument("--cuda_devices", type=int, nargs='+', default=None)
-
-    parser.add_argument("--lr", type=float, default=1e-3)
-    parser.add_argument("--adam_weight_decay", type=float, default=0.01)
-    parser.add_argument("--adam_beta1", type=float, default=0.9)
-    parser.add_argument("--adam_beta2", type=float, default=0.999)
+    parser.add_argument("-c", "--train_dataset", required=True, type=str, help="train dataset for train bert")
+    parser.add_argument("-t", "--test_dataset", type=str, default=None, help="test set for evaluate train set")
+    parser.add_argument("-v", "--vocab_path", required=True, type=str, help="built vocab model path with bert-vocab")
+    parser.add_argument("-o", "--output_path", required=True, type=str, help="ex)output/bert.model")
+
+    parser.add_argument("-hs", "--hidden", type=int, default=256, help="hidden size of transformer model")
+    parser.add_argument("-l", "--layers", type=int, default=8, help="number of layers")
+    parser.add_argument("-a", "--attn_heads", type=int, default=8, help="number of attention heads")
+    parser.add_argument("-s", "--seq_len", type=int, default=20, help="maximum sequence len")
+
+    parser.add_argument("-b", "--batch_size", type=int, default=64, help="number of batch_size")
+    parser.add_argument("-e", "--epochs", type=int, default=10, help="number of epochs")
+    parser.add_argument("-w", "--num_workers", type=int, default=5, help="dataloader worker size")
+
+    parser.add_argument("--with_cuda", type=bool, default=True, help="training with CUDA: true, or false")
+    parser.add_argument("--log_freq", type=int, default=10, help="printing loss every n iter: setting n")
+    parser.add_argument("--corpus_lines", type=int, default=None, help="total number of lines in corpus")
+    parser.add_argument("--cuda_devices", type=int, nargs='+', default=None, help="CUDA device ids")
+    parser.add_argument("--on_memory", type=bool, default=True, help="Loading on memory: true or false")
+
+    parser.add_argument("--lr", type=float, default=1e-3, help="learning rate of adam")
+    parser.add_argument("--adam_weight_decay", type=float, default=0.01, help="weight_decay of adam")
+    parser.add_argument("--adam_beta1", type=float, default=0.9, help="adam first beta value")
+    parser.add_argument("--adam_beta2", type=float, default=0.999, help="adam first beta value")
 
     args = parser.parse_args()
 
@@ -41,11 +42,12 @@ def train():
     print("Vocab Size: ", len(vocab))
 
     print("Loading Train Dataset", args.train_dataset)
-    train_dataset = BERTDataset(args.train_dataset, vocab, seq_len=args.seq_len, corpus_lines=args.corpus_lines)
+    train_dataset = BERTDataset(args.train_dataset, vocab, seq_len=args.seq_len,
+                                corpus_lines=args.corpus_lines, on_memory=args.on_memory)
 
     print("Loading Test Dataset", args.test_dataset)
-    test_dataset = BERTDataset(args.test_dataset, vocab,
-                               seq_len=args.seq_len) if args.test_dataset is not None else None
+    test_dataset = BERTDataset(args.test_dataset, vocab, seq_len=args.seq_len, on_memory=args.on_memory) \
+        if args.test_dataset is not None else None
 
     print("Creating Dataloader")
     train_data_loader = DataLoader(train_dataset, batch_size=args.batch_size, num_workers=args.num_workers)

bert_pytorch/dataset/dataset.py

@@ -5,19 +5,27 @@
 
 
 class BERTDataset(Dataset):
-    def __init__(self, corpus_path, vocab, seq_len, encoding="utf-8", corpus_lines=None):
+    def __init__(self, corpus_path, vocab, seq_len, encoding="utf-8", corpus_lines=None, on_memory=True):
         self.vocab = vocab
         self.seq_len = seq_len
+        self.on_memory = on_memory
+        self.corpus_lines = corpus_lines
 
         with open(corpus_path, "r", encoding=encoding) as f:
-            self.datas = [line[:-1].split("\t")
-                          for line in tqdm.tqdm(f, desc="Loading Dataset", total=corpus_lines)]
+            if self.corpus_lines is None and not on_memory:
+                for _ in tqdm.tqdm(f, desc="Loading Dataset", total=corpus_lines):
+                    self.corpus_lines += 1
+
+            if on_memory:
+                self.lines = [line[:-1].split("\t")
+                              for line in tqdm.tqdm(f, desc="Loading Dataset", total=corpus_lines)]
+                self.corpus_lines = len(self.lines)
 
     def __len__(self):
-        return len(self.datas)
+        return self.corpus_lines
 
     def __getitem__(self, item):
-        t1, (t2, is_next_label) = self.datas[item][0], self.random_sent(item)
+        t1, t2, is_next_label = self.random_sent(item)
         t1_random, t1_label = self.random_word(t1)
         t2_random, t2_label = self.random_word(t2)
 
@@ -54,7 +62,7 @@ def random_word(self, sentence):
                     tokens[i] = self.vocab.mask_index
 
                 # 10% randomly change token to random token
-                elif prob * 0.8 <= prob < prob * 0.9:
+                elif 0.15 * 0.8 <= prob < 0.15 * 0.9:
                     tokens[i] = random.randrange(len(self.vocab))
 
                 # 10% randomly change token to current token
@@ -72,6 +80,6 @@ def random_word(self, sentence):
     def random_sent(self, index):
         # output_text, label(isNotNext:0, isNext:1)
         if random.random() > 0.5:
-            return self.datas[index][1], 1
+            return self.datas[index][0], self.datas[index][1], 1
         else:
-            return self.datas[random.randrange(len(self.datas))][1], 0
+            return self.datas[index][0], self.datas[random.randrange(len(self.datas))][1], 0

requirements.txt

@@ -1,4 +1,4 @@
 tqdm
 numpy
 torch>=0.4.0
-torch-encoding
+torch-encodin