飞桨常规赛：中文新闻文本标题分类 9月第7名方案

# 进入比赛数据集存放目录%cd /home/aistudio/data/data103654/

# 使用pandas读取数据集import pandas as pd
train = pd.read_table('train.txt', sep='\t',header=None)  # 训练集dev = pd.read_table('dev.txt', sep='\t',header=None)      # 验证集test = pd.read_table('test.txt', sep='\t',header=None)    # 测试集

print(f"train数据集长度： {len(train)}\t dev数据集长度{len(dev)}\t test数据集长度{len(test)}")

# 添加列名便于对数据进行更好处理train.columns = ["text_a",'label']
dev.columns = ["text_a",'label']
test.columns = ["text_a"]

# 拼接训练和验证集，便于统计分析total = pd.concat([train,dev],axis=0)

#创建字体目录fonts%cd ~# !mkdir .fonts# 复制字体文件到该路径!cp data/data61659/simhei.ttf .fonts/

# 总类别标签分布统计print(total['label'].value_counts())
%matplotlib inlineimport matplotlib as mplimport matplotlib.pyplot as plt#指定默认字体mpl.rcParams['font.sans-serif'] = ['SimHei']
mpl.rcParams['font.family']='sans-serif'#解决负号'-'显示为方块的问题mpl.rcParams['axes.unicode_minus'] = Falsetotal['label'].value_counts().plot.bar()
plt.show()

# 最大文本长度max(total['text_a'].str.len())

# 文本长度统计分析,通过分析可以看出文本较短，最长为48total['text_a'].map(len).describe()

# 对测试集的长度统计分析，可以看出在长度上分布与训练数据相近test['text_a'].map(len).describe()

# 保存处理后的数据集文件train.to_csv('train.csv', sep='\t', index=False)  # 保存训练集，格式为text_a,labeldev.to_csv('dev.csv', sep='\t', index=False)      # 保存验证集，格式为text_a,labeltest.to_csv('test.csv', sep='\t', index=False)    # 保存测试集，格式为text_a

# 下载最新版本的paddlenlp# !pip install --upgrade paddlenlp

# 导入所需的第三方库import mathimport numpy as npimport osimport collectionsfrom functools import partialimport randomimport timeimport inspectimport importlibfrom tqdm import tqdmimport paddleimport paddle.nn as nnimport paddle.nn.functional as Ffrom paddle.io import IterableDatasetfrom paddle.utils.download import get_path_from_url# 导入paddlenlp所需的相关包import paddlenlp as ppnlpfrom paddlenlp.data import JiebaTokenizer, Pad, Stack, Tuple, Vocabfrom paddlenlp.datasets import MapDatasetfrom paddle.dataset.common import md5filefrom paddlenlp.datasets import DatasetBuilder

# 定义要进行分类的14个类别label_list=list(train.label.unique())print(label_list)

# id 到 label转换id_label_dict={}for i in range(0,len(label_list)):
    id_label_dict[i]=label_list[i]print(id_label_dict)################################################## label到id转换label_id_dict={}for i in range(0,len(label_list)):
    label_id_dict[label_list[i]]=iprint(label_id_dict)

def read(pd_data):
    for index, item in pd_data.iterrows():       
        yield {'text_a': item['text_a'], 'label': label_id_dict[item['label']]}

# 训练集、测试集from paddle.io import Dataset, Subsetfrom paddlenlp.datasets import MapDatasetfrom paddlenlp.datasets import load_dataset

train_dataset = load_dataset(read, pd_data=train,lazy=False)
dev_dataset = load_dataset(read, pd_data=dev,lazy=False)

for i in range(5):    print(train_dataset[i])

# 此次使用在中文领域效果较优的roberta-wwm-ext-large模型，预训练模型一般“大力出奇迹”，选用大的预训练模型可以取得比base模型更优的效果MODEL_NAME = "roberta-wwm-ext-large"# 只需指定想要使用的模型名称和文本分类的类别数即可完成Fine-tune网络定义，通过在预训练模型后拼接上一个全连接网络（Full Connected）进行分类model = ppnlp.transformers.RobertaForSequenceClassification.from_pretrained(MODEL_NAME, num_classes=14) # 此次分类任务为14分类任务，故num_classes设置为14# 定义模型对应的tokenizer，tokenizer可以把原始输入文本转化成模型model可接受的输入数据格式。需注意tokenizer类要与选择的模型相对应，具体可以查看PaddleNLP相关文档tokenizer = ppnlp.transformers.RobertaTokenizer.from_pretrained(MODEL_NAME)

# 定义数据加载和处理函数def convert_example(example, tokenizer, max_seq_length=128, is_test=False):
    qtconcat = example["text_a"]
    encoded_inputs = tokenizer(text=qtconcat, max_seq_len=max_seq_length)  # tokenizer处理为模型可接受的格式 
    input_ids = encoded_inputs["input_ids"]
    token_type_ids = encoded_inputs["token_type_ids"]    if not is_test:
        label = np.array([example["label"]], dtype="int64")        return input_ids, token_type_ids, label    else:        return input_ids, token_type_ids# 定义数据加载函数dataloaderdef create_dataloader(dataset,
                      mode='train',
                      batch_size=1,
                      batchify_fn=None,
                      trans_fn=None):
    if trans_fn:
        dataset = dataset.map(trans_fn)

    shuffle = True if mode == 'train' else False
    # 训练数据集随机打乱，测试数据集不打乱
    if mode == 'train':
        batch_sampler = paddle.io.DistributedBatchSampler(
            dataset, batch_size=batch_size, shuffle=shuffle)    else:
        batch_sampler = paddle.io.BatchSampler(
            dataset, batch_size=batch_size, shuffle=shuffle)    return paddle.io.DataLoader(
        dataset=dataset,
        batch_sampler=batch_sampler,
        collate_fn=batchify_fn,
        return_list=True)

# 参数设置：# 批处理大小，显存如若不足的话可以适当改小该值  batch_size = 360# 文本序列最大截断长度，需要根据文本具体长度进行确定，最长不超过512。 通过文本长度分析可以看出文本长度最大为48，故此处设置为48max_seq_length = max(total['text_a'].str.len())

# 将数据处理成模型可读入的数据格式trans_func = partial(
    convert_example,
    tokenizer=tokenizer,
    max_seq_length=max_seq_length)

batchify_fn = lambda samples, fn=Tuple(
    Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input_ids
    Pad(axis=0, pad_val=tokenizer.pad_token_type_id),  # token_type_ids
    Stack()  # labels): [data for data in fn(samples)]# 训练集迭代器train_data_loader = create_dataloader(
    train_dataset,
    mode='train',
    batch_size=batch_size,
    batchify_fn=batchify_fn,
    trans_fn=trans_func)# 验证集迭代器dev_data_loader = create_dataloader(
    dev_dataset,
    mode='dev',
    batch_size=batch_size,
    batchify_fn=batchify_fn,
    trans_fn=trans_func)

# 定义超参，loss，优化器等from paddlenlp.transformers import LinearDecayWithWarmup# 定义训练过程中的最大学习率learning_rate = 4e-5# 训练轮次epochs = 32# 学习率预热比例warmup_proportion = 0.1# 权重衰减系数，类似模型正则项策略，避免模型过拟合weight_decay = 0.01num_training_steps = len(train_data_loader) * epochs
lr_scheduler = LinearDecayWithWarmup(learning_rate, num_training_steps, warmup_proportion)# AdamW优化器optimizer = paddle.optimizer.AdamW(
    learning_rate=lr_scheduler,
    parameters=model.parameters(),
    weight_decay=weight_decay,
    apply_decay_param_fun=lambda x: x in [
        p.name for n, p in model.named_parameters()        if not any(nd in n for nd in ["bias", "norm"])
    ])

criterion = paddle.nn.loss.CrossEntropyLoss()  # 交叉熵损失函数metric = paddle.metric.Accuracy()              # accuracy评价指标

# 定义模型训练验证评估函数@paddle.no_grad()def evaluate(model, criterion, metric, data_loader):
    model.eval()
    metric.reset()
    losses = []    for batch in data_loader:
        input_ids, token_type_ids, labels = batch
        logits = model(input_ids, token_type_ids)
        loss = criterion(logits, labels)
        losses.append(loss.numpy())
        correct = metric.compute(logits, labels)
        metric.update(correct)
        accu = metric.accumulate()    print("eval loss: %.8f, accu: %.8f" % (np.mean(losses), accu))  # 输出验证集上评估效果
    model.train()
    metric.reset()    return  np.mean(losses), accu  # 返回准确率

# 固定随机种子便于结果的复现# seed = 1024seed = 512random.seed(seed)
np.random.seed(seed)
paddle.seed(seed)
%cd ~

# 模型训练：import paddle.nn.functional as Ffrom visualdl import LogWriter

save_dir='./'writer = LogWriter("./log")
tic_train = time.time()
global_step = 0best_val_acc=0tic_train = time.time()
accu=0for epoch in range(1, epochs + 1):    for step, batch in enumerate(train_data_loader, start=1):
        input_ids, segment_ids, labels = batch
        logits = model(input_ids, segment_ids)
        loss = criterion(logits, labels)
        probs = F.softmax(logits, axis=1)
        correct = metric.compute(probs, labels)
        metric.update(correct)
        acc = metric.accumulate()
        global_step+=1
        if global_step % 40 == 0:            print(                "global step %d, epoch: %d, batch: %d, loss: %.8f, accu: %.8f, speed: %.2f step/s"
                % (global_step, epoch, step, loss, acc,                    40 / (time.time() - tic_train)))
            tic_train = time.time()

        loss.backward()
        optimizer.step()
        lr_scheduler.step()
        optimizer.clear_grad()        
        # 大于100次再eval，或者再大一点eval，太小没有eval的意义
        if global_step % 200 == 0 and global_step>=3000:            # 评估当前训练的模型
            eval_loss, eval_accu = evaluate(model, criterion, metric, dev_data_loader)            print("eval  on dev  loss: {:.8}, accu: {:.8}".format(eval_loss, eval_accu))            # 加入eval日志显示
            writer.add_scalar(tag="eval/loss", step=global_step, value=eval_loss)
            writer.add_scalar(tag="eval/acc", step=global_step, value=eval_accu)            # 加入train日志显示
            writer.add_scalar(tag="train/loss", step=global_step, value=loss)
            writer.add_scalar(tag="train/acc", step=global_step, value=acc)
            save_dir = "best_checkpoint"
            # 加入保存       
            if eval_accu>best_val_acc:                if not os.path.exists(save_dir):
                    os.mkdir(save_dir)
                best_val_acc=eval_accu                print(f"模型保存在 {global_step} 步， 最佳eval准确度为{best_val_acc:.8f}！")
                save_param_path = os.path.join(save_dir, 'best_model.pdparams')
                paddle.save(model.state_dict(), save_param_path)
                fh = open('best_checkpoint/best_model.txt', 'w', encoding='utf-8')
                fh.write(f"模型保存在 {global_step} 步， 最佳eval准确度为{best_val_acc:.8f}！")
                fh.close()

tokenizer.save_pretrained(save_dir)

save_dir='./'tokenizer.save_pretrained(save_dir)

# 测试最优模型参数在验证集上的分数evaluate(model, criterion, metric, dev_data_loader)

# 导入所需的第三方库import mathimport numpy as npimport osimport collectionsfrom functools import partialimport randomimport timeimport inspectimport importlibfrom tqdm import tqdmimport paddleimport paddle.nn as nnimport paddle.nn.functional as Ffrom paddle.io import IterableDatasetfrom paddle.utils.download import get_path_from_url# 导入paddlenlp所需的相关包import paddlenlp as ppnlpfrom paddlenlp.data import JiebaTokenizer, Pad, Stack, Tuple, Vocabfrom paddlenlp.datasets import MapDatasetfrom paddle.dataset.common import md5filefrom paddlenlp.datasets import DatasetBuilder

# 此次使用在中文领域效果较优的roberta-wwm-ext-large模型，预训练模型一般“大力出奇迹”，选用大的预训练模型可以取得比base模型更优的效果MODEL_NAME = "roberta-wwm-ext-large"# 只需指定想要使用的模型名称和文本分类的类别数即可完成Fine-tune网络定义，通过在预训练模型后拼接上一个全连接网络（Full Connected）进行分类model = ppnlp.transformers.RobertaForSequenceClassification.from_pretrained(MODEL_NAME, num_classes=14) # 此次分类任务为14分类任务，故num_classes设置为14# 定义模型对应的tokenizer，tokenizer可以把原始输入文本转化成模型model可接受的输入数据格式。需注意tokenizer类要与选择的模型相对应，具体可以查看PaddleNLP相关文档tokenizer = ppnlp.transformers.RobertaTokenizer.from_pretrained(MODEL_NAME)

# 加载在验证集上效果最优的一轮的模型参数import osimport paddle

seed = 1024random.seed(seed)
np.random.seed(seed)
paddle.seed(seed)

params_path = '88.73671/best_checkpoint/best_model.pdparams'if params_path and os.path.isfile(params_path):    # 加载模型参数
    state_dict = paddle.load(params_path)
    model.set_dict(state_dict)    print("Loaded parameters from %s" % params_path)

# 读取要进行预测的测试集文件import pandas as pd
test = pd.read_csv('~/data/data103654/test.txt', header=None, names=['text_a'])

print(max(test['text_a'].str.len()))

label_list=['科技', '体育', '时政', '股票', '娱乐', '教育', '家居', '财经', '房产', '社会', '游戏', 'cai票', '星座', '时尚']print(label_list)

# 定义要进行分类的类别id_label_dict={}for i in range(0,len(label_list)):
    id_label_dict[i]=label_list[i]print(id_label_dict)

!head -n5 ~/data/data103654/test.txt

# 定义数据加载和处理函数def convert_example(example, tokenizer, max_seq_length=48, is_test=False):
    qtconcat = example["text_a"]
    encoded_inputs = tokenizer(text=qtconcat, max_seq_len=max_seq_length)  # tokenizer处理为模型可接受的格式 
    input_ids = encoded_inputs["input_ids"]
    token_type_ids = encoded_inputs["token_type_ids"]    if not is_test:
        label = np.array([example["label"]], dtype="int64")        return input_ids, token_type_ids, label    else:        return input_ids, token_type_ids# 定义模型预测函数def predict(model, data, tokenizer, label_map, batch_size=1):
    examples = []    # 将输入数据（list格式）处理为模型可接受的格式
    for text in data:
        input_ids, segment_ids = convert_example(
            text,
            tokenizer,
            max_seq_length=48,
            is_test=True)
        examples.append((input_ids, segment_ids))

    batchify_fn = lambda samples, fn=Tuple(
        Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input id
        Pad(axis=0, pad_val=tokenizer.pad_token_id),  # segment id
    ): fn(samples)    # Seperates data into some batches.
    batches = []
    one_batch = []    for example in examples:
        one_batch.append(example)        if len(one_batch) == batch_size:
            batches.append(one_batch)
            one_batch = []    if one_batch:        # The last batch whose size is less than the config batch_size setting.
        batches.append(one_batch)

    results = []
    model.eval()    for batch in batches:
        input_ids, segment_ids = batchify_fn(batch)
        input_ids = paddle.to_tensor(input_ids)
        segment_ids = paddle.to_tensor(segment_ids)
        logits = model(input_ids, segment_ids)
        probs = F.softmax(logits, axis=1)
        idx = paddle.argmax(probs, axis=1).numpy()
        idx = idx.tolist()
        labels = [label_map[i] for i in idx]
        results.extend(labels)    return results  # 返回预测结果

# 定义对数据的预处理函数,处理为模型输入指定list格式def preprocess_prediction_data(data):
    examples = []    for text_a in data:
        examples.append({"text_a": text_a})    return examples# 对测试集数据进行格式处理data1 = list(test.text_a)
examples = preprocess_prediction_data(data1)

# 对测试集进行预测results = predict(model, examples, tokenizer, id_label_dict, batch_size=128)

print(results)

# 将list格式的预测结果存储为txt文件，提交格式要求：每行一个类别def write_results(labels, file_path):
    with open(file_path, "w", encoding="utf8") as f:
        f.writelines("\n".join(labels))

write_results(results, "./result.txt")

# 因格式要求为zip，故需要将结果文件压缩为submission.zip提交文件!zip 'submission.zip' 'result.txt'

!head result.txt