# 导入必要的库import warnings
warnings.simplefilter('ignore')import numpy as npimport pandas as pd

%matplotlibimport matplotlib.pyplot as pltimport seaborn as sns

# 读取数据集train = pd.read_excel('data/data104082/train.xlsx')
test = pd.read_excel('data/data104082/test.xlsx')print(train.shape, test.shape)

train.head()

test.head()

# label 分布train['label'].value_counts()

# 空值填充for col in ['diseaseName', 'conditionDesc', 'title', 'hopeHelp']:
    train[col].fillna('', inplace=True)
    test[col].fillna('', inplace=True)

# 文本长度for col in ['diseaseName', 'conditionDesc', 'title', 'hopeHelp']:    print(train[col].apply(len).describe())    print(test[col].apply(len).describe())

# 清理一些换行字符，不然会对创建数据集有影响def clean_str(x):
    return x.replace('\r', '').replace('\t', ' ').replace('\n', ' ')for col in ['diseaseName', 'conditionDesc', 'title', 'hopeHelp']:
    train[col] = train[col].apply(lambda x: clean_str(x))
    test[col] = test[col].apply(lambda x: clean_str(x))

# 拼接的长度train['text'] = train['diseaseName'].astype(str) + " " + \
                train['conditionDesc'].astype(str) + " " + \
                train['title'].astype(str) + " " + \
                train['hopeHelp'].astype(str)

test['text'] =  test['diseaseName'].astype(str) + " " + \
                test['conditionDesc'].astype(str) + " " + \
                test['title'].astype(str) + " " + \
                test['hopeHelp'].astype(str)print(train['text'].apply(len).describe())print(test['text'].apply(len).describe())

train = train[['text', 'label']].copy()
test = test[['text']].copy()

train = train.sample(frac=1, random_state=42)  # 随机打乱train_size = int(0.9 * len(train))
train_df = train[:train_size]
valid_df = train[train_size:]
test_df = test.copy()print(train_df.shape, valid_df.shape, test_df.shape)

# 保存为文本文件train_df[['label', 'text']].to_csv('train.txt', index=False, header=False, sep='\t')
valid_df[['label', 'text']].to_csv('valid.txt', index=False, header=False, sep='\t')

# 更新 paddlehub!pip install -q --upgrade paddlehub -i https://pypi.tuna.tsinghua.edu.cn/simple

from typing import Dict, List, Optional, Union, Tupleimport paddleimport paddlehub as hubfrom paddlehub.datasets.base_nlp_dataset import TextClassificationDatasetfrom paddlehub.text.bert_tokenizer import BertTokenizerfrom paddlehub.text.tokenizer import CustomTokenizerclass DemoDataset(TextClassificationDataset):
    def __init__(self, tokenizer: Union[BertTokenizer, CustomTokenizer], max_seq_len: int = 256, mode: str = 'train'):
        base_path = './'
        if mode == 'train':
            data_file = 'train.txt'
        elif mode == 'test':
            data_file = 'test.txt'
        else:
            data_file = 'valid.txt'
        super().__init__(
            base_path=base_path,
            tokenizer=tokenizer,
            max_seq_len=max_seq_len,
            mode=mode,
            data_file=data_file,
            label_list=["0", "1", "2", "3", "4",                        "5", "6", "7", "8", "9"],
            is_file_with_header=False)

model = hub.Module(name='ernie_tiny', version='2.0.1', task='seq-cls', num_classes=10)

# 生成数据集train_dataset = DemoDataset(tokenizer=model.get_tokenizer(), mode='train')
valid_dataset = DemoDataset(tokenizer=model.get_tokenizer(), mode='valid')

optimizer = paddle.optimizer.AdamW(learning_rate=5e-5, parameters=model.parameters())
trainer = hub.Trainer(model, optimizer, checkpoint_dir='./checkpoint', use_gpu=True)

trainer.train(
    train_dataset,
    epochs=5,
    batch_size=32,
    eval_dataset=valid_dataset,
    save_interval=5,
)

model = hub.Module(
    name='ernie_tiny',
    version='2.0.1',
    task='seq-cls',
    load_checkpoint='./checkpoint/best_model/model.pdparams',
    label_map={i:str(i) for i in range(10)}
)

data = [[i] for i in test_df['text'].values]
results = model.predict(data, max_seq_len=256, batch_size=1, use_gpu=False)

sub = pd.DataFrame({'id':[i+1 for i in range(len(test_df))], 'label': results})
sub.head()

sub.to_csv('ernie_tiny_baseline.csv', index=False)

<br/>