讯飞机器翻译质量评估挑战赛Baseline(PaddlePaddle)

# 注意请将paddlenlp更新至最新版本!pip install -U paddlenlp

import osimport jsonimport randomimport numpy as npimport pandas as pdfrom tqdm import tqdmfrom sklearn.metrics import accuracy_scorefrom sklearn.model_selection import train_test_splitimport paddleimport paddlenlpimport paddle.nn.functional as Ffrom functools import partialfrom paddlenlp.data import Stack, Dict, Padfrom paddlenlp.datasets import load_datasetfrom paddlenlp.transformers.bert.tokenizer import BertTokenizerimport matplotlib.pyplot as pltimport seaborn as snsfrom scipy.stats import pearsonr

seed = 12345def set_seed(seed):
    paddle.seed(seed)
    random.seed(seed)
    np.random.seed(seed)
set_seed(seed)

# 查看可用的预训练模型BertTokenizer.pretrained_resource_files_map['vocab_file'].keys()

# 超参数MODEL_NAME = 'bert-base-multilingual-uncased'  # bert-base-multilingual-cased# 给出的标签均为文本，创建label_map构建映射关系max_seq_length = 400train_batch_size = 16valid_batch_size = 8test_batch_size = 8# 训练过程中的最大学习率learning_rate = 1e-5# 训练轮次epochs = 5# 学习率预热比例warmup_proportion = 0.1# 权重衰减系数，类似模型正则项策略，避免模型过拟合weight_decay = 0.01max_grad_norm = 1.0# 训练结束后，存储模型参数save_dir_curr = "checkpoint/model"# 记录训练epoch、损失等值loggiing_print = 50loggiing_eval = 50# 提交文件名称sumbit_name = "work/sumbit.csv"

# WMT20-task1数据 使用pandas 读取存在问题，改用open方式读取# 舍弃部分字段如scores,z_scores、z_mean，以平均分作为scoresdef read_wmt20(file_name):
    wmt20_data = []    with open(file_name,"r") as f:        for idx,line in enumerate(f.readlines()):            if idx!=0:
                task_list = line.split("\t")
                scores = int(eval(task_list[4]))                if scores % 10 <4:
                    scores = scores - scores % 10
                elif scores < 7:
                    scores = scores //10 * 10 + 5
                else:
                    scores = (scores //10 + 1) * 10
                wmt20_data.append({                    "index":task_list[0],                    "original":task_list[2],                    "translation":task_list[1],                    "scores":scores / 100 ,
                })    return wmt20_data# 读取讯飞比赛数据，处理为相同格式def read_xfqe(df,istrain=True):
    xfqe_data = []    for idx,rows in df.iterrows():        if istrain:
            xfqe_data.append({                "index":idx,                "original":rows["original"],                "translation":rows["translation"],                "scores":int(rows["scores"]) / 100,
            })        else:
            xfqe_data.append({                "index":idx,                "original":rows["original"],                "translation":rows["translation"],
            })    return xfqe_data

valid = pd.read_csv("data/data154704/valid.txt",sep='\t',names=["original","translation","scores"])
test = pd.read_csv("data/data154704/test.txt",sep='\t',names=["original","translation"])
valid = valid[valid["scores"].notna()]

xfqe_valid = read_xfqe(valid)
xfqe_test = read_xfqe(test,istrain=False)

wmt20_train = read_wmt20("data/data154704/train.enzh.df.short.tsv")
wmt20_dev = read_wmt20("data/data154704/dev.enzh.df.short.tsv")print(len(wmt20_train),len(wmt20_dev))print(len(xfqe_valid),len(xfqe_test))
wmt20_train = wmt20_train + wmt20_devprint("train size: %d \nvalid size %d \ntest size %d" % (len(wmt20_train),len(xfqe_valid),len(xfqe_test)))

# 绘制wmt20的质量分数分布pd.DataFrame(wmt20_train)['scores'].hist()

# 绘制wmt20-task1数据集的文本长度分布states_wmt20 = pd.DataFrame(wmt20_train)
states_wmt20['original_len'] = [len(i) for i in states_wmt20["original"]]
states_wmt20['translation_len'] = [len(i) for i in states_wmt20["translation"]]
states_wmt20['concat_len'] = [i+j for i,j in zip(states_wmt20["original_len"],states_wmt20["translation_len"])]
plt.title("wmt20 text length")
sns.distplot(states_wmt20['original_len'],bins=10,color='r')
sns.distplot(states_wmt20['translation_len'],bins=10,color='g')
sns.distplot(states_wmt20['concat_len'],bins=10,color='b')
plt.show()

# 绘制讯飞数据集的文本长度分布states_valid = valid.copy()
states_valid['original_len'] = [len(i) for i in states_valid["original"]]
states_valid['translation_len'] = [len(i) for i in states_valid["translation"]]
states_valid['concat_len'] = [i+j for i,j in zip(states_valid["original_len"],states_valid["translation_len"])]
plt.title("xf valid text length")
sns.distplot(states_valid['original_len'],bins=10,color='r')
sns.distplot(states_valid['translation_len'],bins=10,color='g')
sns.distplot(states_valid['concat_len'],bins=10,color='b')
plt.show()

# 绘制讯飞数据集的文本长度分布states_test = test.copy()
states_test['original_len'] = [len(i) for i in states_test["original"]]
states_test['translation_len'] = [len(i) for i in states_test["translation"]]
states_test['concat_len'] = [i+j for i,j in zip(states_test["original_len"],states_test["translation_len"])]
plt.title("xf test text length")
sns.distplot(states_test['original_len'],bins=10,color='r')
sns.distplot(states_test['translation_len'],bins=10,color='g')
sns.distplot(states_test['concat_len'],bins=10,color='b')
plt.show()

# 加载tokenizerif MODEL_NAME  == 'bert-base-multilingual-cased':
    tokenizer = BertTokenizer.from_pretrained(MODEL_NAME,do_lower_case=False)else:
    tokenizer = BertTokenizer.from_pretrained(MODEL_NAME)

def read(df,istrain=True):
    if istrain:        for data in df:            yield {                "original":data['original'],                "translation":data['translation'],                "labels":data['scores']
                }    else:        for data in df:            yield {                "original":data['original'],                "translation":data['translation'],
                }# 将生成器传入load_datasettrain_ds = load_dataset(read, df=wmt20_train, lazy=False)
valid_ds = load_dataset(read, df=xfqe_valid, lazy=False)# 查看数据for idx in range(1,3):    print(train_ds[idx])    print("==="*30)for idx in range(1,3):    print(valid_ds[idx])    print("==="*30)

# 编码def convert_example(example, tokenizer, max_seq_len=512, mode='train'):
    # 调用tokenizer的数据处理方法把文本转为id
    tokenized_input = tokenizer(
        example['original'] + '[SEP]' + example['translation'],
        is_split_into_words=True,
        max_seq_len=max_seq_len)    if mode == "test":        return tokenized_input    # 把意图标签转为数字id
    tokenized_input['labels'] = example['labels']    return tokenized_input

train_trans_func = partial(
        convert_example,
        tokenizer=tokenizer,
        mode='train',
        max_seq_len=max_seq_length)

valid_trans_func = partial(
        convert_example,
        tokenizer=tokenizer,
        mode='dev',
        max_seq_len=max_seq_length)

train_ds.map(train_trans_func, lazy=False)
valid_ds.map(valid_trans_func, lazy=False)# 初始化BatchSamplertrain_batch_sampler = paddle.io.BatchSampler(train_ds, batch_size=train_batch_size, shuffle=True)
valid_batch_sampler = paddle.io.BatchSampler(valid_ds, batch_size=valid_batch_size, shuffle=False)# 定义batchify_fnbatchify_fn = lambda samples, fn = Dict({    "input_ids": Pad(axis=0, pad_val=tokenizer.pad_token_id), 
    "token_type_ids": Pad(axis=0, pad_val=tokenizer.pad_token_type_id),    "labels": Stack(dtype="float32"),
}): fn(samples)# 初始化DataLoadertrain_data_loader = paddle.io.DataLoader(
    dataset=train_ds,
    batch_sampler=train_batch_sampler,
    collate_fn=batchify_fn,
    return_list=True)

valid_data_loader = paddle.io.DataLoader(
    dataset=valid_ds,
    batch_sampler=valid_batch_sampler,
    collate_fn=batchify_fn,
    return_list=True)

import paddlefrom paddle import nnfrom paddlenlp.transformers.bert.modeling import BertPretrainedModel# Modelclass MTransQuestErnieM(BertPretrainedModel):
    def __init__(self, bert, labels_dim=1, dropout=None):
        super().__init__()        # 预训练模型
        self.bert = bert
        self.dropout = nn.Dropout(self.bert.config['hidden_dropout_prob'])
        self.classifier = nn.Linear(self.bert.config['hidden_size'],labels_dim)
        self.sigmoid = nn.Sigmoid()
        self.apply(self.init_weights)    def forward(self,
                input_ids,
                token_type_ids=None):
        sequence_output,_ = self.bert(
            input_ids,
            token_type_ids=token_type_ids)
        sequence_output = self.dropout(sequence_output[:,0,:])  
        sequence_output = self.classifier(sequence_output)
        logits = self.sigmoid(sequence_output)        return logits

# 创建MTransQuest modelmodel = MTransQuestErnieM.from_pretrained(MODEL_NAME)

# 训练总步数num_training_steps = len(train_data_loader) * epochs# 学习率衰减策略lr_scheduler = paddlenlp.transformers.LinearDecayWithWarmup(learning_rate, num_training_steps,warmup_proportion)

decay_params = [
    p.name for n, p in model.named_parameters()    if not any(nd in n for nd in ["bias", "norm"])
]# 定义优化器optimizer = paddle.optimizer.AdamW(
    learning_rate=lr_scheduler,
    parameters=model.parameters(),
    weight_decay=weight_decay,
    apply_decay_param_fun=lambda x: x in decay_params,
    grad_clip=paddle.nn.ClipGradByGlobalNorm(max_grad_norm))

# 验证部分@paddle.no_grad()def evaluation(model, data_loader):
    model.eval()
    real_s = []
    pred_s = []    for batch in data_loader:
        input_ids, token_type_ids, labels = batch
        logits = model(input_ids, token_type_ids)
        logits = paddle.reshape(logits,[-1])
        labels = labels
        pred_s.extend(logits.numpy())
        real_s.extend(labels.numpy())
    score =  pearsonr(pred_s,real_s)[0]    return score# 训练阶段def do_train(model,data_loader):
    model_total_epochs = 0
    best_score = 0.
    training_loss = 0
    # 训练
    print("train ...")
    model.train()    for epoch in range(0, epochs):        for step, batch in enumerate(data_loader, start=1):
            input_ids, token_type_ids, labels = batch
            logits = model(input_ids, token_type_ids)
            logits = paddle.reshape(logits,[-1])
            labels = labels
            loss = F.mse_loss(logits,labels)
            
            loss.backward()
            optimizer.step()
            lr_scheduler.step()
            optimizer.clear_grad()
            training_loss  += loss.numpy()
        
            model_total_epochs += 1
            if model_total_epochs % loggiing_print == 0:                print("step: %d / %d, training loss: %.5f" % (model_total_epochs, num_training_steps, training_loss / model_total_epochs))            
            if model_total_epochs % loggiing_eval == 0:
                eval_score = evaluation(model, valid_data_loader)                print("eval pearsonr: %.5f" % eval_score)                if best_score  < eval_score:                    print("pearsonr update %.5f ---> %.5f " % (best_score,eval_score))
                    best_score  = eval_score                    # 保存模型
                    os.makedirs(save_dir_curr,exist_ok=True)
                    save_param_path = os.path.join(save_dir_curr, 'model_best.pdparams')
                    paddle.save(model.state_dict(), save_param_path)                    # 保存tokenizer
                    tokenizer.save_pretrained(save_dir_curr)                else:                    print("but best score %.5f" %  best_score )
                model.train()    return best_score

best_score = do_train(model,train_data_loader)

print("best pearsonr score: %.5f" % best_score)

# 相同方式构造测试集test_ds = load_dataset(read,df=xfqe_test, istrain=False, lazy=False)

test_trans_func = partial(
        convert_example,
        tokenizer=tokenizer,
        mode='test',
        max_seq_len=max_seq_length)

test_ds.map(test_trans_func, lazy=False)

test_batch_sampler = paddle.io.BatchSampler(test_ds, batch_size=test_batch_size, shuffle=False)

test_batchify_fn = lambda samples, fn = Dict({    "input_ids": Pad(axis=0, pad_val=tokenizer.pad_token_id), 
    "token_type_ids": Pad(axis=0, pad_val=tokenizer.pad_token_type_id),
}): fn(samples)

test_data_loader = paddle.io.DataLoader(
    dataset=test_ds,
    batch_sampler=test_batch_sampler,
    collate_fn=test_batchify_fn,
    return_list=True)# 预测阶段def do_sample_predict(model,data_loader):
    model.eval()
    preds = []    for batch in data_loader:
        input_ids, token_type_ids= batch
        logits = model(input_ids, token_type_ids)
        logits = paddle.reshape(logits * 100,[-1]) 
        preds.extend(logits.numpy())    return preds# 读取最佳模型state_dict = paddle.load(os.path.join(save_dir_curr,'model_best.pdparams'))
model.load_dict(state_dict)# 预测print("predict start ...")
pred_score = []
pred_score_continuous = do_sample_predict(model,test_data_loader)# 将连续的预测值 转化 离散的分数（以5作为最小的粒度）for scores in pred_score_continuous:    if scores % 10 <4:
        scores = scores - scores % 10
    elif scores < 7:
        scores = scores //10 * 10 + 5
    else:
        scores = (scores //10 + 1) * 10
    pred_score.append(int(scores))print("predict end ...")

sumbit = pd.DataFrame([],columns=['pred'])
sumbit["pred"] = pred_score
sumbit.to_csv("work/sumbit_{}.csv".format(MODEL_NAME),index=False,header=False)