note

Author: songyingxin

BERT-Pytorch 源码阅读.md

@@ -0,0 +1,37 @@
+# BERT-Pytorch 源码阅读
+
+---
+
+## 0. 数据准备
+
+由于只是用来测试，因此，数据随便选了一个文本相似度数据集
+
+## 1.  整体描述
+
+BERT-Pytorch 在分发包时，主要设置了两大功能：
+
+- bert-vocab ：统计词频，token2idx, idx2token 等信息。对应 `bert_pytorch.dataset.vocab` 中的 `build` 函数。
+- bert：对应 `bert_pytorch.__main__` 下的 train 函数。
+
+### 1. bert-vocab
+
+```
+python3 -m ipdb test_bert_vocab.py  # 调试 bert-vocab
+```
+
+其实 bert-vocab 内部并没有什么重要信息，无非就是一些自然语言处理中常见的预处理手段， 自己花个十分钟调试一下就明白了， 我加了少部分注释， 很容易就能明白。
+
+内部继承关系为： 
+
+```
+TorchVocab --> Vocab --> WordVocab
+```
+
+### 2. bert
+
+#### 1. Bert Model
+
+![整体结构图](.\img\all.png)
+
+
+

bert_pytorch/dataset/vocab.py

@@ -15,17 +15,12 @@ class TorchVocab(object):
 
     def __init__(self, counter, max_size=None, min_freq=1, specials=['<pad>', '<oov>'],
                  vectors=None, unk_init=None, vectors_cache=None):
-        """Create a Vocab object from a collections.Counter.
-        Arguments:
-            counter: collections.Counter object holding the frequencies of
-                each value found in the data.
-            max_size: The maximum size of the vocabulary, or None for no
-                maximum. Default: None.
-            min_freq: The minimum frequency needed to include a token in the
-                vocabulary. Values less than 1 will be set to 1. Default: 1.
-            specials: The list of special tokens (e.g., padding or eos) that
-                will be prepended to the vocabulary in addition to an <unk>
-                token. Default: ['<pad>']
+        """用一个 collections.Counter 对象简历 Vocab
+        Args:
+            counter: collections.Counter 对象。预训练文件中的 token 统计 {'token': 10} 
+            max_size: 词表最大长度。 None for no maximum. Default: None.
+            min_freq: 最小词频。 Default: 1.
+            specials: 列表， 包含一系列特殊字符，如['<pad', 'unk']等。  Default: ['<pad>']
             vectors: One of either the available pretrained vectors
                 or custom pretrained vectors (see Vocab.load_vectors);
                 or a list of aforementioned vectors
@@ -39,23 +34,24 @@ def __init__(self, counter, max_size=None, min_freq=1, specials=['<pad>', '<oov>
         min_freq = max(min_freq, 1)
 
         self.itos = list(specials)
-        # frequencies of special tokens are not counted when building vocabulary
-        # in frequency order
+
+        # 特殊字符不计入统计词频
         for tok in specials:
             del counter[tok]
 
         max_size = None if max_size is None else max_size + len(self.itos)
 
-        # sort by frequency, then alphabetically
+        # 先按照字典序排列，然后按照词频排列
         words_and_frequencies = sorted(counter.items(), key=lambda tup: tup[0])
         words_and_frequencies.sort(key=lambda tup: tup[1], reverse=True)
 
+        # 依据词频和字典长度过滤数据
         for word, freq in words_and_frequencies:
             if freq < min_freq or len(self.itos) == max_size:
                 break
             self.itos.append(word)
 
-        # stoi is simply a reverse dict for itos
+        # token2idx
         self.stoi = {tok: i for i, tok in enumerate(self.itos)}
 
         self.vectors = None
@@ -163,6 +159,7 @@ def from_seq(self, seq, join=False, with_pad=False):
 
     @staticmethod
     def load_vocab(vocab_path: str) -> 'WordVocab':
+        """将 WordVocab 对象序列化到 vocab_path 文件中 """
         with open(vocab_path, "rb") as f:
             return pickle.load(f)
 

bert_pytorch/model/bert.py

@@ -10,12 +10,13 @@ class BERT(nn.Module):
     """
 
     def __init__(self, vocab_size, hidden=768, n_layers=12, attn_heads=12, dropout=0.1):
-        """
-        :param vocab_size: vocab_size of total words
-        :param hidden: BERT model hidden size
-        :param n_layers: numbers of Transformer blocks(layers)
-        :param attn_heads: number of attention heads
-        :param dropout: dropout rate
+        """ Bert 模型
+        Args:
+            vocab_size: 词表大小
+            hidden: BERT 的 hidden size
+            n_layers: Transformer 的层数
+            attn_heads: Multi-head Attention 中的 head 数
+            dropout: dropout rate
         """
 
         super().__init__()
@@ -26,10 +27,10 @@ def __init__(self, vocab_size, hidden=768, n_layers=12, attn_heads=12, dropout=0
         # paper noted they used 4*hidden_size for ff_network_hidden_size
         self.feed_forward_hidden = hidden * 4
 
-        # embedding for BERT, sum of positional, segment, token embeddings
+        # BERT的输入embedding, 由 positional, segment, token embeddings 三部分组成
         self.embedding = BERTEmbedding(vocab_size=vocab_size, embed_size=hidden)
 
-        # multi-layers transformer blocks, deep network
+        # 多层的 Transformer (Encoder)
         self.transformer_blocks = nn.ModuleList(
             [TransformerBlock(hidden, attn_heads, hidden * 4, dropout) for _ in range(n_layers)])
 

bert_pytorch/model/embedding/bert.py

@@ -6,19 +6,18 @@
 
 class BERTEmbedding(nn.Module):
     """
-    BERT Embedding which is consisted with under features
-        1. TokenEmbedding : normal embedding matrix
-        2. PositionalEmbedding : adding positional information using sin, cos
-        2. SegmentEmbedding : adding sentence segment info, (sent_A:1, sent_B:2)
-
-        sum of all these features are output of BERTEmbedding
+    BERT Embedding 由以下三部分组成：
+        1. TokenEmbedding : token embedding matrix
+        2. PositionalEmbedding : 位置信息编码
+        2. SegmentEmbedding : 句子信息编码, (sent_A:1, sent_B:2)
     """
 
     def __init__(self, vocab_size, embed_size, dropout=0.1):
         """
-        :param vocab_size: total vocab size
-        :param embed_size: embedding size of token embedding
-        :param dropout: dropout rate
+        Args:
+            vocab_size: 词表大小
+            embed_size: token embedding 的维度
+            dropout: dropout rate
         """
         super().__init__()
         self.token = TokenEmbedding(vocab_size=vocab_size, embed_size=embed_size)

bert_pytorch/model/embedding/segment.py

@@ -3,4 +3,5 @@
 
 class SegmentEmbedding(nn.Embedding):
     def __init__(self, embed_size=512):
+        """ 3 为 padding_idx, sent_A, sent_B """
         super().__init__(3, embed_size, padding_idx=0)

img/all.png

@@ -0,0 +1,94 @@
+import argparse
+
+from torch.utils.data import DataLoader
+
+from bert_pytorch.model import BERT
+from bert_pytorch.trainer import BERTTrainer
+from bert_pytorch.dataset import BERTDataset, WordVocab
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument("-c", "--train_dataset", required=True,
+                        type=str, help="train dataset for train bert")
+    parser.add_argument("-t", "--test_dataset", type=str,
+                        default=None, help="test set for evaluate train set")
+    parser.add_argument("-v", "--vocab_path", required=True,
+                        type=str, help="built vocab model path with bert-vocab")
+    parser.add_argument("-o", "--output_path", required=True,
+                        type=str, help="ex)output/bert.model")
+
+    parser.add_argument("-hs", "--hidden", type=int,
+                        default=256, help="hidden size of transformer model")
+    parser.add_argument("-l", "--layers", type=int,
+                        default=8, help="number of layers")
+    parser.add_argument("-a", "--attn_heads", type=int,
+                        default=8, help="number of attention heads")
+    parser.add_argument("-s", "--seq_len", type=int,
+                        default=20, help="maximum sequence len")
+
+    parser.add_argument("-b", "--batch_size", type=int,
+                        default=64, help="number of batch_size")
+    parser.add_argument("-e", "--epochs", type=int,
+                        default=10, help="number of epochs")
+    parser.add_argument("-w", "--num_workers", type=int,
+                        default=5, help="dataloader worker size")
+
+    parser.add_argument("--with_cuda", type=bool, default=True,
+                        help="training with CUDA: true, or false")
+    parser.add_argument("--log_freq", type=int, default=10,
+                        help="printing loss every n iter: setting n")
+    parser.add_argument("--corpus_lines", type=int,
+                        default=None, help="total number of lines in corpus")
+    parser.add_argument("--cuda_devices", type=int, nargs='+',
+                        default=None, help="CUDA device ids")
+    parser.add_argument("--on_memory", type=bool, default=True,
+                        help="Loading on memory: true or false")
+
+    parser.add_argument("--lr", type=float, default=1e-3,
+                        help="learning rate of adam")
+    parser.add_argument("--adam_weight_decay", type=float,
+                        default=0.01, help="weight_decay of adam")
+    parser.add_argument("--adam_beta1", type=float,
+                        default=0.9, help="adam first beta value")
+    parser.add_argument("--adam_beta2", type=float,
+                        default=0.999, help="adam first beta value")
+
+    args = parser.parse_args()
+
+    print("Loading Vocab", args.vocab_path)
+    vocab = WordVocab.load_vocab(args.vocab_path)
+    print("Vocab Size: ", len(vocab))
+
+    print("Loading Train Dataset", args.train_dataset)
+    train_dataset = BERTDataset(args.train_dataset, vocab, seq_len=args.seq_len,
+                                corpus_lines=args.corpus_lines, on_memory=args.on_memory)
+
+    print("Loading Test Dataset", args.test_dataset)
+    test_dataset = BERTDataset(args.test_dataset, vocab, seq_len=args.seq_len, on_memory=args.on_memory) \
+        if args.test_dataset is not None else None
+
+    print("Creating Dataloader")
+    train_data_loader = DataLoader(
+        train_dataset, batch_size=args.batch_size, num_workers=args.num_workers)
+    test_data_loader = DataLoader(test_dataset, batch_size=args.batch_size, num_workers=args.num_workers) \
+        if test_dataset is not None else None
+
+    print("Building BERT model")
+    bert = BERT(len(vocab), hidden=args.hidden,
+                n_layers=args.layers, attn_heads=args.attn_heads)
+
+    print("Creating BERT Trainer")
+    trainer = BERTTrainer(bert, len(vocab), train_dataloader=train_data_loader, test_dataloader=test_data_loader,
+                          lr=args.lr, betas=(
+                              args.adam_beta1, args.adam_beta2), weight_decay=args.adam_weight_decay,
+                          with_cuda=args.with_cuda, cuda_devices=args.cuda_devices, log_freq=args.log_freq)
+
+    print("Training Start")
+    for epoch in range(args.epochs):
+        trainer.train(epoch)
+        trainer.save(epoch, args.output_path)
+
+        if test_data_loader is not None:
+            trainer.test(epoch)

test_bert.py

@@ -0,0 +1,20 @@
+
+from bert_pytorch.dataset.vocab import *
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-c", "--corpus_path", required=True, type=str)
+    parser.add_argument("-o", "--output_path", required=True, type=str)
+    parser.add_argument("-s", "--vocab_size", type=int, default=None)
+    parser.add_argument("-e", "--encoding", type=str, default="utf-8")
+    parser.add_argument("-m", "--min_freq", type=int, default=1)
+    args = parser.parse_args()
+
+    with open(args.corpus_path, "r", encoding=args.encoding) as f:
+        vocab = WordVocab(f, max_size=args.vocab_size, min_freq=args.min_freq)
+
+    print("VOCAB SIZE:", len(vocab))
+    vocab.save_vocab(args.output_path)