主流互联网游戏评论情感态势分析

!tree

#引入需要的库import paddlenlp as ppnlpimport paddlefrom paddlenlp.datasets import load_datasetfrom paddlenlp.datasets import MapDatasetfrom paddlenlp.data import Stack, Pad, Tupleimport paddle.nn.functional as Fimport numpy as npimport pandas as pdfrom functools import partialimport os

#读取数据集并划分训练集与验证集df=pd.read_csv('data/data109273/taptap_review_ready.csv')#读取数据集的评论内容review = df.review.tolist()#读取数据集的情感指数sentiment = df.sentiment.tolist()

full_data=[]for i in range(len(review)):    #将每条评论内容与情感指数形成字典并添加到full_data列表
    dic = {'text':review[i],'label':sentiment[i]}
    full_data.append(dic)print(len(full_data))#数据集切分train_ds = MapDataset(full_data[:4400])
dev_ds = MapDataset(full_data[4400:4880])
label_list = ['0', '1']print(len(train_ds))print(train_ds[3])

from paddlenlp.transformers import SkepForSequenceClassification, SkepTokenizer# 加载skep模型model = SkepForSequenceClassification.from_pretrained(pretrained_model_name_or_path="skep_ernie_1.0_large_ch", num_classes=len(label_list))# 加载模型对应的Tokenizer，用于数据预处理tokenizer = SkepTokenizer.from_pretrained(pretrained_model_name_or_path="skep_ernie_1.0_large_ch")

#数据预处理def convert_example(example,tokenizer,label_list,max_seq_length=128,is_test=False):
    if is_test:
        text = example['text']    else:
        text = example['text']
        label = example['label']    #tokenizer.encode方法实现切分token，映射token ID以及拼接特殊token
    encoded_inputs = tokenizer.encode(text=text, max_seq_len=max_seq_length)

    input_ids = encoded_inputs["input_ids"]
    token_type_ids = encoded_inputs["token_type_ids"]    if not is_test:
        label_map = {}        for (i, l) in enumerate(label_list):
            label_map[l] = i        # label：情感极性类别    
        label = np.array([label], dtype="int64")        return input_ids, token_type_ids, label    else:        return input_ids, token_type_ids#数据迭代器def create_dataloader(dataset, trans_fn=None, mode='train', batch_size=1, use_gpu=False, pad_token_id=0, batchify_fn=None):
    if trans_fn:
        dataset = dataset.map(trans_fn, lazy=True)    if mode == 'train' and use_gpu:
        sampler = paddle.io.DistributedBatchSampler(dataset=dataset, batch_size=batch_size, shuffle=True)    else:
        shuffle = True if mode == 'train' else False 
        sampler = paddle.io.BatchSampler(dataset=dataset, batch_size=batch_size, shuffle=shuffle) #生成一个取样器
    dataloader = paddle.io.DataLoader(dataset, batch_sampler=sampler, return_list=True, collate_fn=batchify_fn)    return dataloader#将数据处理成模型可读入的数据格式trans_fn = partial(convert_example, tokenizer=tokenizer, label_list=label_list, max_seq_length=128, is_test=False)# 将数据组成批量式数据batchify_fn = lambda samples, fn=Tuple(Pad(axis=0,pad_val=tokenizer.pad_token_id), Pad(axis=0, pad_val=tokenizer.pad_token_id), Stack(dtype="int64")):[data for data in fn(samples)]#batch_size批量数据大小batch_size=64#训练集迭代器train_loader = create_dataloader(train_ds, mode='train', batch_size=batch_size, batchify_fn=batchify_fn, trans_fn=trans_fn)#验证集迭代器dev_loader = create_dataloader(dev_ds, mode='dev', batch_size=batch_size, batchify_fn=batchify_fn, trans_fn=trans_fn)

#设置训练参数import timefrom utils import evaluate# 训练轮次epochs = 3# 训练过程中保存模型参数的文件夹ckpt_dir = "skep_ckpt"# len(train_loader)一轮训练所需要的step数num_training_steps = len(train_loader) * epochs# Adam优化器optimizer = paddle.optimizer.AdamW(
    learning_rate=2e-5,
    parameters=model.parameters())# 交叉熵损失函数criterion = paddle.nn.loss.CrossEntropyLoss()# accuracy评价指标metric = paddle.metric.Accuracy()

#模型训练global_step = 0tic_train = time.time()for epoch in range(1, epochs + 1):    for step, batch in enumerate(train_loader, start=1):
        input_ids, token_type_ids, labels = batch        # 喂数据给model
        logits = model(input_ids, token_type_ids)        # 计算损失函数值
        loss = criterion(logits, labels)        # 预测分类概率值
        probs = F.softmax(logits, axis=1)        # 计算acc
        correct = metric.compute(probs, labels)
        metric.update(correct)
        acc = metric.accumulate()

        global_step += 1
        if global_step % 10 == 0:            print(                "global step %d, epoch: %d, batch: %d, loss: %.5f, accu: %.5f, speed: %.2f step/s"
                % (global_step, epoch, step, loss, acc,                    10 / (time.time() - tic_train)))
            tic_train = time.time()        
        # 反向梯度回传，更新参数
        loss.backward()
        optimizer.step()
        optimizer.clear_grad()        if global_step % 100 == 0:
            save_dir = os.path.join(ckpt_dir, "model_%d" % global_step)            if not os.path.exists(save_dir):
                os.makedirs(save_dir)            # 评估当前训练的模型
            evaluate(model, criterion, metric, dev_loader)            # 保存当前模型参数等
            model.save_pretrained(save_dir)            # 保存tokenizer的词表等
            tokenizer.save_pretrained(save_dir)

# 加载训练好的模型参数params_path = save_dir+'/model_state.pdparams'if params_path and os.path.isfile(params_path):    # 加载模型参数
    state_dict = paddle.load(params_path)
    model.set_dict(state_dict)    print("Loaded parameters from %s" % params_path)

#定义测试集数据的处理函数def convert_example(example,tokenizer,label_list,max_seq_length=512,is_test=False):
    encoded_inputs = tokenizer(text=example, max_seq_len=max_seq_length)
    input_ids = np.array(encoded_inputs["input_ids"], dtype="int64")
    token_type_ids = np.array(encoded_inputs["token_type_ids"], dtype="int64")    return input_ids, token_type_ids

#定义预测函数def predict(model, data, tokenizer, label_map, batch_size=1):
    examples = []    for text in data:
        input_ids, token_type_ids = convert_example(text,tokenizer,label_list=label_map.values(),max_seq_length=512,is_test=True)
        examples.append((input_ids, token_type_ids))    #划分数据
    batches = [
        examples[idx:idx + batch_size]        for idx in range(0, len(examples), batch_size)
    ]
    batchify_fn = lambda samples, fn=Tuple(
        Pad(axis=0, pad_val=tokenizer.pad_token_id),  # input ids
        Pad(axis=0, pad_val=tokenizer.pad_token_type_id),  # token type ids
    ): [data for data in fn(samples)]    
    #返回结果集
    results = []

    model.eval()    for batch in batches:
        input_ids, token_type_ids = batchify_fn(batch)
        input_ids = paddle.to_tensor(input_ids)
        token_type_ids = paddle.to_tensor(token_type_ids)
        logits = model(input_ids, token_type_ids)
        probs = F.softmax(logits, axis=1)
        idx = paddle.argmax(probs, axis=1).numpy()
        idx = idx.tolist()
        labels = [label_map[i] for i in idx]
        results.extend(labels)    return results

#加载测试集import csvwith open('spider/data/bilibili_yuanshen.csv','r',encoding='gbk',errors='ignore')as f:
    cs =list(csv.reader(f))#定义存储评论列表comments=[]#csv文件含有表头，故i初始化为1#评论文本在csv文件第三列，故选择cs[i][2]for i in range(1,len(cs)):
    comments.append(cs[i][2])print(len(comments))print(comments[1])

#对测试集数据情感分析label_map = {0: '0', 1: '1'}
results = predict(model,comments,tokenizer,label_map,batch_size=1)#统计积极评论数量count = 0for result in results:    if result == '1':
        count =count + 1print(count)#计算积极评论比例positive_ratio = count / len(results)print(positive_ratio)