基于BERT实现机器阅读理解

!pip install paddlenlp --upgrade

import siximport functoolsimport inspectimport paddleimport osimport jsonimport paddleimport paddle.nn as nnimport paddle.tensor as tensorimport paddle.nn.functional as Ffrom paddle.nn import Layerfrom paddle.nn import TransformerEncoder, Linear, Layer, Embedding, LayerNorm, Tanhfrom paddlenlp.transformers.tokenizer_utils import PretrainedTokenizerfrom paddlenlp.transformers.bert.tokenizer import BasicTokenizer,WordpieceTokenizerfrom paddlenlp.data import Pad, Stack, Tuple, Dictfrom paddle.io import DataLoaderfrom paddle.optimizer.lr import LambdaDecayimport sysimport math

class BertTokenizer(PretrainedTokenizer):
    resource_files_names = {"vocab_file": "vocab.txt"}  # for save_pretrained
    pretrained_resource_files_map = {        "vocab_file": {            "bert-base-chinese":            "https://paddle-hapi.bj.bcebos.com/models/bert/bert-base-chinese-vocab.txt",
        }
    }
    pretrained_init_configuration = {        "bert-base-chinese": {            "do_lower_case": False
        },
    }
    padding_side = 'right'

    def __init__(self,vocab_file,do_lower_case=True,unk_token="[UNK]",sep_token="[SEP]",
                 pad_token="[PAD]",
                 cls_token="[CLS]",
                 mask_token="[MASK]"):

        if not os.path.isfile(vocab_file):            raise ValueError(                "Can't find a vocabulary file at path '{}'. To load the "
                "vocabulary from a pretrained model please use "
                "`tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`"
                .format(vocab_file))        # 加载词汇文件vocab.txt，返回一个有序字典
        self.vocab = self.load_vocabulary(vocab_file, unk_token=unk_token)        # 定义 BasicTokenizer
        self.basic_tokenizer = BasicTokenizer(do_lower_case=do_lower_case)        # 定义 WordpieceTokenizer
        self.wordpiece_tokenizer = WordpieceTokenizer(
            vocab=self.vocab, unk_token=unk_token)    @property
    def vocab_size(self):
        # 返回词汇表的大小
        return len(self.vocab)    def _tokenize(self, text):
        split_tokens = []        # 进行unicode转换、标点符号分割、小写转换、中文字符分割、去除重音符号等操作，最后返回的是关于词的数组（中文是字的数组）
        for token in self.basic_tokenizer.tokenize(text):            # 将合成词分解成类似词根一样的词片，例如将"unwanted"分解成["un", "##want", "##ed"]
            for sub_token in self.wordpiece_tokenizer.tokenize(token):
                split_tokens.append(sub_token)        return split_tokens    def tokenize(self, text):
        # 对文本进行切分和wordPiece化
        return self._tokenize(text)    def convert_tokens_to_string(self, tokens):
        # 对tokens的list进行拼接，并去除里面的##符号
        out_string = " ".join(tokens).replace(" ##", "").strip()        return out_string    def num_special_tokens_to_add(self, pair=False):
        token_ids_0 = []
        token_ids_1 = []        return len(
            self.build_inputs_with_special_tokens(token_ids_0, token_ids_1                                                  if pair else None))    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
        # 给输入文本加上cls开始符号，和sep分隔符号
        if token_ids_1 is None:            return [self.cls_token_id] + token_ids_0 + [self.sep_token_id]
        _cls = [self.cls_token_id]
        _sep = [self.sep_token_id]        return _cls + token_ids_0 + _sep + token_ids_1 + _sep    def build_offset_mapping_with_special_tokens(self,offset_mapping_0,offset_mapping_1=None):
        # 用来记录每个词起始字符和结束字符的索引
        if offset_mapping_1 is None:            return [(0, 0)] + offset_mapping_0 + [(0, 0)]        return [(0, 0)] + offset_mapping_0 + [(0, 0)
                                              ] + offset_mapping_1 + [(0, 0)]    def create_token_type_ids_from_sequences(self,
                                             token_ids_0,
                                             token_ids_1=None):
        # 从传递的两个序列创建一个掩码，用于序列对分类任务。
        _sep = [self.sep_token_id]
        _cls = [self.cls_token_id]        if token_ids_1 is None:            return len(_cls + token_ids_0 + _sep) * [0]        return len(_cls + token_ids_0 + _sep) * [0] + len(token_ids_1 +
                                                          _sep) * [1]    def get_special_tokens_mask(self,
                                token_ids_0,
                                token_ids_1=None,
                                already_has_special_tokens=False):
        # 从没有添加特殊标记的标记列表中检索序列ID。添加时调用此方法使用标记器“encode”方法的特殊标记。
        if already_has_special_tokens:            if token_ids_1 is not None:                raise ValueError(                    "You should not supply a second sequence if the provided sequence of "
                    "ids is already formatted with special tokens for the model."
                )            return list(                map(lambda x: 1 if x in [self.sep_token_id, self.cls_token_id] else 0,
                    token_ids_0))        if token_ids_1 is not None:            return [1] + ([0] * len(token_ids_0)) + [1] + (
                [0] * len(token_ids_1)) + [1]        return [1] + ([0] * len(token_ids_0)) + [1]

model_name_or_path='bert-base-chinese'tokenizer = BertTokenizer.from_pretrained(model_name_or_path)

from paddlenlp.datasets import load_dataset

doc_stride=128  # 滑动窗口的大小max_seq_length=384  # 分词后的最大长度task_name='dureader_robust'

def prepare_train_features(examples):
    contexts = [examples[i]['context'] for i in range(len(examples))]
    questions = [examples[i]['question'] for i in range(len(examples))]

    tokenized_examples = tokenizer(
            questions,
            contexts,
            stride=doc_stride,
            max_seq_len=max_seq_length)    for i, tokenized_example in enumerate(tokenized_examples):        # 把不可能的答案用CLS字符来索引
        input_ids = tokenized_example["input_ids"]
        cls_index = input_ids.index(tokenizer.cls_token_id)        # offset mapping会建立一个从token到字符在原文中的位置的映射，这帮助我们计算开始位置和结束位置
        offsets = tokenized_example['offset_mapping']        # 抓取样本对应的序列（知道上下文是什么，问题是什么）
        sequence_ids = tokenized_example['token_type_ids']        # 一个样本可以有多个span，这是包含此文本span的示例的索引。
        sample_index = tokenized_example['overflow_to_sample']
        answers = examples[sample_index]['answers']
        answer_starts = examples[sample_index]['answer_starts']        # 文本中答案所在的开始字符和结束字符的索引
        start_char = answer_starts[0]
        end_char = start_char + len(answers[0])        # 文本中当前span的开始token的索引
        token_start_index = 0
        while sequence_ids[token_start_index] != 1:
            token_start_index += 1

        # 文本中当前span的结束token的索引
        token_end_index = len(input_ids) - 1
        while sequence_ids[token_end_index] != 1:
            token_end_index -= 1
        token_end_index -= 1

        # 检查答案是否超过了span（特征用cls的索引标注），如果超过了，答案的位置用cls的位置代替
        if not (offsets[token_start_index][0] <= start_char and
                offsets[token_end_index][1] >= end_char):
            tokenized_examples[i]["start_positions"] = cls_index
            tokenized_examples[i]["end_positions"] = cls_index        else:            # 把token_start_index和token_end_index移动到答案的两端
            while token_start_index < len(offsets) and offsets[
                    token_start_index][0] <= start_char:
                token_start_index += 1
            tokenized_examples[i]["start_positions"] = token_start_index - 1
            while offsets[token_end_index][1] >= end_char:
                token_end_index -= 1
            tokenized_examples[i]["end_positions"] = token_end_index + 1

    return tokenized_examples

train_ds = load_dataset(task_name, splits='train')print(train_ds[:2])

train_ds.map(prepare_train_features, batched=True)

for idx in range(2):    print('input_ids:{}'.format(train_ds[idx]['input_ids']))    print('token_type_ids:{}'.format(train_ds[idx]['token_type_ids']))    print('overflow_to_sample:{}'.format(train_ds[idx]['overflow_to_sample']))    print('offset_mapping:{}'.format(train_ds[idx]['offset_mapping']))    print('start_positions:{}'.format(train_ds[idx]['start_positions']))    print('end_positions:{}'.format(train_ds[idx]['end_positions']))    print()

def prepare_validation_features(examples):
    contexts = [examples[i]['context'] for i in range(len(examples))]
    questions = [examples[i]['question'] for i in range(len(examples))]

    tokenized_examples = tokenizer(
            questions,
            contexts,
            stride=doc_stride,
            max_seq_len=max_seq_length)    # 对于验证，不需要计算开始和结束位置
    for i, tokenized_example in enumerate(tokenized_examples):        # 抓取样本对应的序列（知道上下文是什么，问题是什么）
        sequence_ids = tokenized_example['token_type_ids']        # 一个样本可以有多个span，这是包含此文本span的示例的索引。
        sample_index = tokenized_example['overflow_to_sample']
        tokenized_examples[i]["example_id"] = examples[sample_index]['id']        # 将不属于上下文的偏移量映射设置为None，这样就可以很容易地确定token位置是否属于上下文
        tokenized_examples[i]["offset_mapping"] = [
                (o if sequence_ids[k] == 1 else None)                for k, o in enumerate(tokenized_example["offset_mapping"])
            ]    return tokenized_examples

dev_ds = load_dataset(task_name, splits='dev')print(dev_ds[:2])

dev_ds.map(prepare_validation_features, batched=True)

class BertEmbeddings(Layer):
    def __init__(self,
                 vocab_size,
                 hidden_size=768,
                 hidden_dropout_prob=0.1,
                 max_position_embeddings=512,
                 type_vocab_size=16):
        super(BertEmbeddings, self).__init__()        # Token Embedding
        self.word_embeddings = nn.Embedding(vocab_size, hidden_size)        # position embedding
        self.position_embeddings = nn.Embedding(max_position_embeddings,
                                                hidden_size)        # token_type embedding
        self.token_type_embeddings = nn.Embedding(type_vocab_size, hidden_size)        # 层归一化
        self.layer_norm = nn.LayerNorm(hidden_size)        # dropout层
        self.dropout = nn.Dropout(hidden_dropout_prob)    def forward(self, input_ids, token_type_ids=None, position_ids=None):
        if position_ids is None:
            ones = paddle.ones_like(input_ids, dtype="int64")
            seq_length = paddle.cumsum(ones, axis=-1)

            position_ids = seq_length - ones
            position_ids.stop_gradient = True
        if token_type_ids is None:
            token_type_ids = paddle.zeros_like(input_ids, dtype="int64")        # token embedding
        input_embedings = self.word_embeddings(input_ids)        # position embedding
        position_embeddings = self.position_embeddings(position_ids)        # token_type embedding
        token_type_embeddings = self.token_type_embeddings(token_type_ids)        # token embedding, position embedding和token type embedding进行拼接
        embeddings = input_embedings + position_embeddings + token_type_embeddings        # 层归一化
        embeddings = self.layer_norm(embeddings)        # dropout操作
        embeddings = self.dropout(embeddings)        return embeddings

class BertPooler(Layer):

    def __init__(self, hidden_size):
        super(BertPooler, self).__init__()        # 全连接层
        self.dense = nn.Linear(hidden_size, hidden_size)        # tanh激活函数
        self.activation = nn.Tanh()    def forward(self, hidden_states):
        # 把隐藏状态的第0个token的向量取出来
        first_token_tensor = hidden_states[:, 0]        # 全连接
        pooled_output = self.dense(first_token_tensor)        # tanh激活函数
        pooled_output = self.activation(pooled_output)        return pooled_output

from paddlenlp.transformers import PretrainedModel

class BertPretrainedModel(PretrainedModel):

    model_config_file = "model_config.json"
    pretrained_init_configuration = {        "bert-base-chinese": {            "vocab_size": 21128,            "hidden_size": 768,            "num_hidden_layers": 12,            "num_attention_heads": 12,            "intermediate_size": 3072,            "hidden_act": "gelu",            "hidden_dropout_prob": 0.1,            "attention_probs_dropout_prob": 0.1,            "max_position_embeddings": 512,            "type_vocab_size": 2,            "initializer_range": 0.02,            "pad_token_id": 0,
        },
    }
    resource_files_names = {"model_state": "model_state.pdparams"}
    pretrained_resource_files_map = {        "model_state": {            "bert-base-chinese":            "http://paddlenlp.bj.bcebos.com/models/transformers/bert/bert-base-chinese.pdparams",
       
        }
    }
    base_model_prefix = "bert"

    def init_weights(self, layer):
        """ Initialization hook """
        if isinstance(layer, (nn.Linear, nn.Embedding)):            if isinstance(layer.weight, paddle.Tensor):
                layer.weight.set_value(
                    paddle.tensor.normal(
                        mean=0.0,
                        std=self.initializer_range                        if hasattr(self, "initializer_range") else
                        self.bert.config["initializer_range"],
                        shape=layer.weight.shape))        elif isinstance(layer, nn.LayerNorm):
            layer._epsilon = 1e-12

from paddlenlp.transformers import register_base_model

@register_base_modelclass BertModel(BertPretrainedModel):

    def __init__(self,
                 vocab_size,
                 hidden_size=768,
                 num_hidden_layers=12,
                 num_attention_heads=12,
                 intermediate_size=3072,
                 hidden_act="gelu",
                 hidden_dropout_prob=0.1,
                 attention_probs_dropout_prob=0.1,
                 max_position_embeddings=512,
                 type_vocab_size=16,
                 initializer_range=0.02,
                 pad_token_id=0):
        super(BertModel, self).__init__()
        self.pad_token_id = pad_token_id
        self.initializer_range = initializer_range        # BertEmbeddings层
        self.embeddings = BertEmbeddings(
            vocab_size, hidden_size, hidden_dropout_prob,
            max_position_embeddings, type_vocab_size)        # TransformerEncoderLayer层 12个multi-head
        encoder_layer = nn.TransformerEncoderLayer(
            hidden_size,
            num_attention_heads,
            intermediate_size,
            dropout=hidden_dropout_prob,
            activation=hidden_act,
            attn_dropout=attention_probs_dropout_prob,
            act_dropout=0)        # TransformerEncoder层 12层encoder_layer
        self.encoder = nn.TransformerEncoder(encoder_layer, num_hidden_layers)        # BertPooler层
        self.pooler = BertPooler(hidden_size)        # 模型参数初始化
        self.apply(self.init_weights)    def forward(self,input_ids,token_type_ids=None,position_ids=None,attention_mask=None):
        if attention_mask is None:
            attention_mask = paddle.unsqueeze(
                (input_ids == self.pad_token_id
                 ).astype(self.pooler.dense.weight.dtype) * -1e9,
                axis=[1, 2])        # input_id,position_id,token_type_id embedding
        embedding_output = self.embeddings(
            input_ids=input_ids,
            position_ids=position_ids,
            token_type_ids=token_type_ids)        # 调用TransformerEncoder层 12层encoder_layer
        encoder_outputs = self.encoder(embedding_output, attention_mask)
        sequence_output = encoder_outputs        # 调用BertPooler
        pooled_output = self.pooler(sequence_output)        return sequence_output, pooled_output

class BertForQuestionAnswering(BertPretrainedModel):
    def __init__(self, bert, dropout=None):
        super(BertForQuestionAnswering, self).__init__()        # 加载bert配置
        self.bert = bert  
        # 全连接层
        self.classifier = nn.Linear(self.bert.config["hidden_size"], 2)
        self.apply(self.init_weights)    def forward(self, input_ids, token_type_ids=None):
        # Bert接收输入
        sequence_output, _ = self.bert(
            input_ids,
            token_type_ids=token_type_ids,
            position_ids=None,
            attention_mask=None)        # 分类
        logits = self.classifier(sequence_output)
        logits = paddle.transpose(logits, perm=[2, 0, 1])
        start_logits, end_logits = paddle.unstack(x=logits, axis=0)        return start_logits, end_logits

class CrossEntropyLossForSQuAD(paddle.nn.Layer):
    def __init__(self):
        super(CrossEntropyLossForSQuAD, self).__init__()    def forward(self, y, label):
        # 预测值的开始位置和结束位置
        start_logits, end_logits = y        # ground truth的开始位置和结束位置
        start_position, end_position = label
        start_position = paddle.unsqueeze(start_position, axis=-1)
        end_position = paddle.unsqueeze(end_position, axis=-1)        # 计算start loss
        start_loss = paddle.nn.functional.softmax_with_cross_entropy(
            logits=start_logits, label=start_position, soft_label=False)
        start_loss = paddle.mean(start_loss)        # 计算end loss
        end_loss = paddle.nn.functional.softmax_with_cross_entropy(
            logits=end_logits, label=end_position, soft_label=False)
        end_loss = paddle.mean(end_loss)        # 求最终的损失
        loss = (start_loss + end_loss) / 2
        return loss

# 判断number是否是整型def is_integer(number):
    if sys.version > '3':        return isinstance(number, int)    return isinstance(number, (int, long))class LinearDecayWithWarmup(LambdaDecay):
    def __init__(self,
                 learning_rate,
                 total_steps,
                 warmup,
                 last_epoch=-1,
                 verbose=False):
        warmup_steps = warmup if is_integer(warmup) else int(
            math.floor(warmup * total_steps))        def lr_lambda(current_step):
            # current_step小于warmup_steps的时候，学习率逐渐增加
            if current_step < warmup_steps:                return float(current_step) / float(max(1, warmup_steps))            # 学习率随着current_step的增加而降低
            return max(0.0,                       float(total_steps - current_step) /                       float(max(1, total_steps - warmup_steps)))        super(LinearDecayWithWarmup, self).__init__(learning_rate, lr_lambda,
                                                    last_epoch, verbose)

# 设置GPU模式paddle.set_device("gpu")# 实例化BertForQuestionAnswering模型model = BertForQuestionAnswering.from_pretrained(model_name_or_path)

criterion = CrossEntropyLossForSQuAD()

# batch size的大小batch_size=16 # epoch的大小num_train_epochs=1# warmup的比例warmup_proportion=0.1# 优化器的配置learning_rate=3e-5 weight_decay=0.01adam_epsilon=1e-8

train_batch_sampler = paddle.io.DistributedBatchSampler(train_ds, batch_size=batch_size, shuffle=True)
train_batchify_fn = lambda samples, fn=Dict({            "input_ids": Pad(axis=0, pad_val=tokenizer.pad_token_id),            "token_type_ids": Pad(axis=0, pad_val=tokenizer.pad_token_type_id),            "start_positions": Stack(dtype="int64"),            "end_positions": Stack(dtype="int64")
        }): fn(samples)

train_data_loader = DataLoader(
            dataset=train_ds,
            batch_sampler=train_batch_sampler,
            collate_fn=train_batchify_fn,
            return_list=True)

num_training_steps = len(train_data_loader) * num_train_epochs

lr_scheduler = LinearDecayWithWarmup(
            learning_rate, num_training_steps, warmup_proportion)

decay_params = [
            p.name for n, p in model.named_parameters()            if not any(nd in n for nd in ["bias", "norm"])
        ]# 优化器optimizer = paddle.optimizer.AdamW(
            learning_rate=lr_scheduler,
            epsilon=adam_epsilon,
            parameters=model.parameters(),
            weight_decay=weight_decay,
            apply_decay_param_fun=lambda x: x in decay_params)

global_step = 0logging_steps=100save_steps=1000import time
tic_train = time.time()for epoch in range(num_train_epochs):    # 遍历每个batch的数据
    for step, batch in enumerate(train_data_loader):
        global_step += 1
        input_ids, token_type_ids, start_positions, end_positions = batch        # 调用模型
        logits = model(input_ids=input_ids, token_type_ids=token_type_ids)        # 求损失
        loss = criterion(logits, (start_positions, end_positions))        # 打印日志
        if global_step % logging_steps == 0:            print("global step %d, epoch: %d, batch: %d, loss: %f, speed: %.2f step/s"
                        % (global_step, epoch, step, loss,
                           logging_steps / (time.time() - tic_train)))
            tic_train = time.time()        # 反向传播，更新权重，清除梯度
        loss.backward()
        optimizer.step()
        lr_scheduler.step()
        optimizer.clear_grad()

output_path='./chekpoint/dureader_robust/'# 如果文件夹不存在就创建文件夹if not os.path.exists(output_path):
    os.makedirs(output_path)# 保存模型model.save_pretrained(output_path)# 保存tokenizertokenizer.save_pretrained(output_path)print('Saving checkpoint to:', output_path)

from paddlenlp.metrics.squad import squad_evaluate, compute_prediction

dev_batch_sampler = paddle.io.BatchSampler(
            dev_ds, batch_size=batch_size, shuffle=False)
dev_batchify_fn = lambda samples, fn=Dict({            "input_ids": Pad(axis=0, pad_val=tokenizer.pad_token_id),            "token_type_ids": Pad(axis=0, pad_val=tokenizer.pad_token_type_id)
        }): fn(samples)
dev_data_loader = DataLoader(
            dataset=dev_ds,
            batch_sampler=dev_batch_sampler,
            collate_fn=dev_batchify_fn,
            return_list=True)

@paddle.no_grad()def evaluate(model, data_loader):
    # 切换为预测模式
    model.eval()
    n_best_size=20
    max_answer_length=30

    all_start_logits = []
    all_end_logits = []
    tic_eval = time.time()    # 遍历每个batch数据
    for batch in data_loader:
        input_ids, token_type_ids = batch        # 调用模型，得到模型的输出
        start_logits_tensor, end_logits_tensor = model(input_ids,
                                                       token_type_ids)        # 解析模型的输出
        for idx in range(start_logits_tensor.shape[0]):            if len(all_start_logits) % 1000 == 0 and len(all_start_logits):                print("Processing example: %d" % len(all_start_logits))                print('time per 1000:', time.time() - tic_eval)
                tic_eval = time.time()

            all_start_logits.append(start_logits_tensor.numpy()[idx])
            all_end_logits.append(end_logits_tensor.numpy()[idx])    # 预测
    all_predictions, _, _ = compute_prediction(
        data_loader.dataset.data, data_loader.dataset.new_data,
        (all_start_logits, all_end_logits), False, n_best_size,
        max_answer_length)    # 预测的结果写入到文件
    with open('prediction.json', "w", encoding='utf-8') as writer:
        writer.write(
            json.dumps(
                all_predictions, ensure_ascii=False, indent=4) + "\n")    # 评估结果
    squad_evaluate(
        examples=data_loader.dataset.data,
        preds=all_predictions,
        is_whitespace_splited=False)    # 切换为训练模式
    model.train()

evaluate(model, dev_data_loader)