【NLP】常规赛：中文新闻文本标题分类

import paddleimport numpy as npimport jiebaimport matplotlib.pyplot as pltfrom tqdm import tqdm

train_data_path="data/data118554/train.txt"val_data_path='data/data118554/dev.txt'test_data_path='data/data118554/test.txt'

def openfile(path):
    with open(path,'r',encoding='utf-8') as source:
        lines=source.readlines()    return lines

train_lines=openfile(train_data_path)
val_lines=openfile(val_data_path)
test_lines=openfile(test_data_path)

print(len(train_lines))print(train_lines[0])print(len(val_lines))print(val_lines[0])print(len(test_lines))print(test_lines[0])

def data_process(datalines,test=False):
    datalist=[]
    labellist=[]    for datas in datalines:        #data,label=datas.strip().split()
        data=datas.strip().split()        #print(data)
        if test==False:
            labellist.append(data[-1])            if len(data[:-1])>1:                for i in range(1,len(data[:-1])):
                    data[0]+=","+data[i]        else:            if len(data)>1:                for i in range(1,len(data)):
                    data[0]+=","+data[i]

        
        datalist.append(data[0])        
    return datalist,labellist

train_data,train_label=data_process(train_lines)
val_data,val_label=data_process(val_lines)
test_data,_=data_process(test_lines,test=True)

for i in range(5):    print(train_data[i],'   ',train_label[i])print("***********")for i in range(5):    print(val_data[i],'   ',val_label[i])    
print("***********")for i in range(5):    print(test_data[i])

def jieba_process(datalist):
    data=[]    for datas in tqdm(datalist):
        data.append(jieba.lcut(datas))    
    return data

train_data=jieba_process(train_data)
val_data=jieba_process(val_data)
test_data=jieba_process(test_data)

print(train_data[0],train_label[0])print(val_data[0],val_label[0])print(test_data[0])

label_set=set()for label in tqdm(train_label):
    label_set.add(label)

print(label_set)

label_dict=dict()
dict_label=dict()for label in label_set:
    label_dict[label]=len(label_dict)
    dict_label[len(label_dict)-1]=label

print(label_dict)print(dict_label)

alllen_dict=dict()for data in train_data:
    length=len(data)    if length not in alllen_dict:
        alllen_dict[length]=0
    alllen_dict[length]+=1

alllen_dict = sorted(alllen_dict.items(), key = lambda x:x[0], reverse = False)print(alllen_dict)

x=[l[0] for l in alllen_dict]
y=[l[1] for l in alllen_dict]
plt.bar(x, y)  
plt.xlabel('length')
plt.ylabel('nums')
plt.legend(loc='lower right')
plt.show()

def build_cropus(data):
    crpous=[]    for i in range(len(data)):
        crpous.extend(data[i])    return crpous

allcrpous=build_cropus(train_data+val_data+test_data)print(len(allcrpous))

# 构造词典，统计每个词的频率，并根据频率将每个词转换为一个整数iddef build_dict(corpus,frequency):
    # 首先统计每个不同词的频率（出现的次数），使用一个词典记录
    word_freq_dict = dict()    for word in corpus:        if word not in word_freq_dict:
            word_freq_dict[word] = 0
        word_freq_dict[word] += 1

    # 将这个词典中的词，按照出现次数排序，出现次数越高，排序越靠前
    word_freq_dict = sorted(word_freq_dict.items(), key = lambda x:x[1], reverse = True)    
    # 构造3个不同的词典，分别存储，
    # 每个词到id的映射关系：word2id_dict
    
    # 每个id到词的映射关系：id2word_dict
    word2id_dict = {'<pad>':0,'<unk>':1}
   
    id2word_dict = {0:'<pad>',1:'<unk>'}    # 按照频率，从高到低，开始遍历每个单词，并为这个单词构造一个独一无二的id
    for word, freq in word_freq_dict:        if freq>frequency:
            curr_id = len(word2id_dict)
            word2id_dict[word] = curr_id
            id2word_dict[curr_id] = word        else:
            word2id_dict[word]=1
    return word2id_dict, id2word_dict,word_freq_dict

word_fre=1word2id_dict,id2word_dict,word_counts=build_dict(allcrpous,word_fre)print(len(word2id_dict))print(len(id2word_dict))

vocab_maxlen=len(word2id_dict)print('有',len(word2id_dict),'个字被映射到',len(id2word_dict),'个id上') # 字：id

counts_word_dict=dict()for word,counts in word_counts:    if counts not in counts_word_dict:
        counts_word_dict[counts]=0
    counts_word_dict[counts]+=1counts_word_dict = sorted(counts_word_dict.items(), key = lambda x:x[0], reverse = False)#print(counts_word_dict)x=[l[0] for l in counts_word_dict]
y=[l[1] for l in counts_word_dict]

plt.bar(x[:10], y[:10])  
plt.xlabel('frequency')
plt.ylabel('nums')
plt.legend(loc='lower right')
plt.show()

tensor_maxlen=15  # 根据统计到的标题长度分布设定vocab_size=len(id2word_dict)  # 词汇量

def build_tensor(data,dicta,maxlen):
    tensor=[]    for i in range(len(data)):
        subtensor=[]
        lista=data[i]        for j in range(len(lista)):
            index=dicta.get(lista[j])
            subtensor.append(index)        
        # 长度限定，不足补0 ；超过则截断   
        if len(subtensor) < maxlen:  
            subtensor+=[0]*(maxlen-len(subtensor))        else:
            subtensor=subtensor[:maxlen]

        tensor.append(subtensor)    return tensor

train_tensor=paddle.to_tensor(np.array(build_tensor(train_data,word2id_dict,tensor_maxlen)))
val_tensor=paddle.to_tensor(np.array(build_tensor(val_data,word2id_dict,tensor_maxlen)))
test_tensor=np.array(build_tensor(test_data,word2id_dict,tensor_maxlen))

print(train_tensor.shape)print(train_tensor[0])print(type(train_tensor))print(val_tensor.shape)print(val_tensor[0])print(type(val_tensor))print(test_tensor.shape)print(test_tensor[0])print(type(test_tensor))

print(train_label[0])print(val_label[0])print(label_dict)def get_label_tensor(dict,label):
    tensor=[]    for d in label:
        tensor.append(dict[d])    return tensor# def test_lable_tensor(lena):#     tensor=[]#     for i in range(lena):#         tensor.append(0)#     return tensor

train_label_tensor=np.array(get_label_tensor(label_dict,train_label))
val_label_tensor=np.array(get_label_tensor(label_dict,val_label))# test_lable_tensor=np.array(test_label_tesnor(len(test_tensor)))

print(train_label_tensor[0])

numclass=len(label_set)
train_label_tensor=paddle.to_tensor(train_label_tensor,dtype='int64')
val_label_tensor=paddle.to_tensor(val_label_tensor,dtype='int64')# test_label_tensor=paddle.to_tensor(test_label_tensor,dtype='int64')# train_label_tensor=paddle.nn.functional.one_hot(paddle.to_tensor(train_label_tensor,dtype='int32'),numclass)# val_label_tensor=paddle.nn.functional.one_hot(paddle.to_tensor(val_label_tensor,dtype='int32'),numclass)

print(type(train_label_tensor))print(train_label_tensor[0])print(train_label_tensor.shape)print(val_label_tensor.shape)# print(test_label_tensor.shape)

class MyDataset(paddle.io.Dataset):
    """
    步骤一：继承paddle.io.Dataset类
    """
    def __init__(self, title,lable):
        """
        步骤二：实现构造函数，定义数据集大小
        """
        super(MyDataset, self).__init__()
        self.title = title
        self.lable=lable    def __getitem__(self, index):
        """
        步骤三：实现__getitem__方法，定义指定index时如何获取数据，并返回单条数据（训练数据，对应的标签）
        """
        # if self.lable==None:
        #     return self.title[index]
        # else:
        return self.title[index], self.lable[index]    def __len__(self):
        """
        步骤四：实现__len__方法，返回数据集总数目
        """
        return self.title.shape[0]

BATCH_SIZE=128embed_dim=256hidden_size=128train_batch_num=train_tensor.shape[0]//BATCH_SIZE #3482val_batch_num=val_tensor.shape[0]//BATCH_SIZE #156print(train_batch_num)print(val_batch_num)

# 定义数据集train_dataset = MyDataset(train_tensor,train_label_tensor)
train_loader = paddle.io.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True,drop_last=True)

val_dataset=MyDataset(val_tensor,val_label_tensor)
val_loader=paddle.io.DataLoader(val_dataset,batch_size=BATCH_SIZE,shuffle=True,drop_last=True)# test_dataset=MyDataset(test_tensor,train_label_tensor)# test_loader=paddle.io.DataLoader(val_dataset,batch_size=BATCH_SIZE)

class Mynet(paddle.nn.Layer):
    def __init__(self,vocab_size,embed_dim,hidden_size,data_maxlen,numclass):
        super(Mynet,self).__init__()
        self.numclass=numclass
        self.data_maxlen=data_maxlen
        self.vocab_size=vocab_size
        self.embed_dim=embed_dim

        self.emb=paddle.nn.Embedding(vocab_size,embed_dim)
        self.gru=paddle.nn.GRU(embed_dim,hidden_size,2)

        self.l1=paddle.nn.Linear(hidden_size,64)
        self.l2=paddle.nn.Linear(64,32)
        self.l3=paddle.nn.Linear(32,self.numclass)

        self.drop=paddle.nn.Dropout(0.5)    
    def forward(self,x):
        x=self.emb(x)
        x,states=self.gru(x)
        x=paddle.mean(x,axis=1)

        x=self.drop(x)

        out=paddle.nn.functional.relu(self.l1(x))
        out=self.drop(out)
        out=paddle.nn.functional.relu(self.l2(out))
        out=self.l3(out)
        out=paddle.nn.functional.softmax(out,axis=-1)        return out

mynet=Mynet(vocab_size,embed_dim,hidden_size,tensor_maxlen,numclass)

paddle.summary(mynet,(128,15),dtypes='int64')

epochs = 20lr=0.001log_freq=1000model_path='./model/train_model'

model=paddle.Model(mynet)# 为模型训练做准备，设置优化器，损失函数和精度计算方式model.prepare(optimizer=paddle.optimizer.Adam(learning_rate=lr,parameters=model.parameters()),
              loss=paddle.nn.CrossEntropyLoss(),
              metrics=paddle.metric.Accuracy())

model.fit(train_data=train_loader,
           eval_data=val_loader,
           epochs=epochs,
           eval_freq=1,
           save_freq=5,
           save_dir=model_path,
           
           verbose=1,
           callbacks=[paddle.callbacks.VisualDL('./log')])

model.save('./model/infer')

infer_model=paddle.Model(Mynet(vocab_size,embed_dim,hidden_size,tensor_maxlen,numclass))
infer_model.load('./model/infer')

with open('result.txt','w',encoding="utf-8") as res:    for title in test_tensor:
        re = infer_model.predict_batch([[title]])        #print(re)
        index=paddle.argmax(paddle.to_tensor(re))
        index=int(index[0])        #print(type(index))
        #print(dict_label[index])
        res.write(dict_label[index]+'\n')print('_____________over__________over______________')