[网络安全篇4] 全连接网络分类实现

# 对数据进行一个copy!cp data/data157200/syscallsbinders_frequency_5_Cat.csv work/

import pandas as pd# 按文件名读取整个文件data_471 = pd.read_csv("work/syscallsbinders_frequency_5_Cat.csv") 
data_471.head(2)     # 展示前几行数据，通常为展示前5行

import pandas as pd# 按文件名读取整个文件data_471 = pd.read_csv("work/syscallsbinders_frequency_5_Cat.csv") 
df = pd.DataFrame(data_471)len(df.columns)# 记录每一列求和小于阈值的列threshold = 0de = []# 记录样本数据中的最大值de_max = 0de_va = []for i in range(0,470):
    name = df.columns.values[i]    if(df[name].sum() < threshold):
        de.append(name)    if(df[name].max() >= de_max):
        de_max = df[name].max()# 删除小于阈值的列for j in de:
    df.drop(j,axis=1,inplace=True)

len_new = len(df.columns)

len_new

de_max

# 新数据写入csv 虽然没有变化，新存一下吧，养成一个良好的文件备份习惯df.to_csv("new_date_"+str(len_new)+".csv",index=False,sep=',')

import numpy as npimport pandas as pd

mydata = pd.read_csv("new_date_"+str(len_new)+".csv")
row_list = mydata.values.tolist()# 一个数据展示print(row_list[0])

len(row_list)

# 获取最后一维的数据，也就是分类类别 1-5 这个是一行的分类的展示row_list[0][-1]

import numpy as np# 获取类别分布 所有行的最后一列为labelcls_ = np.asarray(mydata.iloc[:-2,-1])print('Labels:', np.unique(cls_))

_records = {
    i: sum([1 for j in cls_ if j==i]) for i in range(1,6)
}print('dist:', _records)

import random
random.shuffle(row_list)

# 验证一下是不是真的打乱了 因为最原始的数据前面全是1分类，所以下面的输出只要不是全1 就是打乱的了for i in range(4):    print(row_list[i][-1])

# 划分数据train_rate = 0.8train_list = []
eval_list = []

_cls_records = {i:0 for i in cls_}for data in row_list:
    _data_str, _label_int= data[:-1],data[-1:]
    label = int(_label_int[0])    # 控制每个类别的比例
    if _cls_records[label] < int(_records[label]*train_rate):
        train_list.append([_data_str,_label_int])    else:
        eval_list.append([_data_str,_label_int])
    _cls_records[label] += 1print(len(train_list))print(len(eval_list))print(len(train_list)+len(eval_list))

# 解析数据样本train_data = []
train_label = []for data_label in train_list:
    data,label = data_label[0],data_label[1]
    train_data.append(data)
    train_label.append(label)

val_data = []
val_label = []for data_label in eval_list:
    data,label = data_label[0],data_label[1]
    val_data.append(data)
    val_label.append(label)

print("Train Dataset Length:", len(train_data))print("Eval Dataset Length:", len(val_data))print("Example Sample:", train_data[0], '------', train_label[0])

from paddle.io import Datasetfrom PIL import Imageclass MyDataset(Dataset):
    def __init__(self, mode = 'train'):
        # 训练样本数量
        self.training_data, self.training_label, self.test_data, self.test_label = train_data, train_label, val_data, val_label        if mode  == 'train':
            self.num_samples = len(train_data)        else:
            self.num_samples = len(val_data)
        self.mode = mode    def __getitem__(self, idx):
        if self.mode == 'train':
            data = self.training_data[idx]
            label = self.training_label[idx]        else:
            data = self.test_data[idx]
            label = self.test_label[idx]
        
        data = np.array(data).astype('float32')        
        return data, np.array(label, dtype='int64')    def __len__(self):
        # 返回样本总数量
        return self.num_samples

# 训练的数据提供器train_dataset = MyDataset(mode='train')# 测试的数据提供器eval_dataset = MyDataset(mode='val')# 查看训练和测试数据的大小print('train大小：', train_dataset.__len__())print('eval大小：', eval_dataset.__len__())# 查看图片数据、大小及标签for data, label in train_dataset:    print(np.array(data).shape)    print(np.array(label).shape)    break

import paddleimport paddle.nn as nn# 实现MLP模块class Mlp(nn.Layer):
    # self代表类的实例自身
    def __init__(self):
        # 初始化父类中的一些参数
        super(Mlp, self).__init__()
        
        self.fc1 = paddle.nn.Linear(in_features=470, out_features=1024)
        self.bn1 = paddle.nn.BatchNorm1D(1024)
        self.relu1 = paddle.nn.ReLU()
        self.fc2 = paddle.nn.Linear(in_features=1024, out_features=512)
        self.bn2 = paddle.nn.BatchNorm1D(512)
        self.relu2 = paddle.nn.ReLU()
        self.fc3 = paddle.nn.Linear(in_features=512, out_features=64)
        self.bn3 = paddle.nn.BatchNorm1D(64)
        self.relu3 = paddle.nn.ReLU()
        self.fc4 = paddle.nn.Linear(in_features=64, out_features=1)    
    # 网络的前向计算
    def forward(self, inputs):
        x = self.fc1(inputs)
        x = self.bn1(x)
        x = self.relu1(x)
        x = self.fc2(x)
        x = self.bn2(x)
        x = self.relu2(x)
        x = self.fc3(x)
        x = self.bn3(x)
        x = self.relu3(x)
        x = self.fc4(x)        return x

# 测试mlp = Mlp()# 模型结构展示paddle.summary(mlp, (256, 470))

import paddlefrom paddle.io import DataLoader

BATCH_SIZE = 256  # 设置batch大小# 训练的数据提供器train_dataloader = DataLoader(
        dataset = train_dataset,
        batch_size = BATCH_SIZE,
        shuffle = True,
        drop_last=True
        )# 测试的数据提供器eval_dataloader = DataLoader(
        dataset = eval_dataset,
        batch_size = BATCH_SIZE,
        shuffle =False,
        drop_last=True
        )

import os

EPOCH_NUM = 1500   # 设置外层循环次数save_dir = 'output_model/'# 模型封装model = Mlp()

learning_rate = 5e-3opt = paddle.optimizer.SGD(learning_rate=learning_rate, parameters=model.parameters())def train_mlp():
    # 定义外层循环
    train_loss = []
    val_loss = []    for epoch_id in range(EPOCH_NUM):        # 开启模型训练模式
        model.train()        for iter_id, mini_batch in enumerate(train_dataloader()):            # 清空梯度变量，以备下一轮计算
            opt.clear_grad()

            x = np.array(mini_batch[:][0])
            y = np.array(mini_batch[:][1])            # 将numpy数据转为飞桨动态图tensor的格式
            features = paddle.to_tensor(x,dtype='float32')
            y = paddle.to_tensor(y,dtype='float32')            # 前向计算
            predicts = model(features)            # 计算损失
            loss = paddle.nn.functional.l1_loss(predicts, label=y)
            avg_loss = paddle.mean(loss)
            train_loss.append(avg_loss.numpy())            # 反向传播，计算每层参数的梯度值
            avg_loss.backward()            # 更新参数，根据设置好的学习率迭代一步
            opt.step()        
        # 开启验证模式
        model.eval()        for iter_id, mini_batch in enumerate(eval_dataloader()):

            x = np.array(mini_batch[:][0])
            y = np.array(mini_batch[:][1])

            features = paddle.to_tensor(x,dtype='float32')
            y = paddle.to_tensor(y,dtype='float32')

            predicts = model(features)
            
            loss = paddle.nn.functional.l1_loss(predicts, label=y)
            avg_loss = paddle.mean(loss)
            val_loss.append(avg_loss.numpy())        
        print(f'Epoch {epoch_id}, train loss is  {np.mean(train_loss)}, val loss is {np.mean(val_loss)}')        if (epoch_id % 5 == 0):            if not os.path.isdir(save_dir):
                os.makedirs(save_dir)
            save_path = os.path.join(save_dir, '{0}'.format(epoch_id))
            paddle.save(model.state_dict(), save_path + '.pdparams')
            paddle.save(opt.state_dict(), save_path + '.pdopt')    return train_loss,val_loss

tloss,vloss = train_mlp()

import matplotlib.pyplot as plt
%matplotlib inline

x = [i for i in range(len(tloss))]
y = [i for i in range(len(vloss))]def draw(x,tloss,label):
    plt.title(label,fontsize = 24)
    plt.xlabel('iter_id',fontsize = 20)
    plt.ylabel('loss',fontsize = 20)
    plt.plot(x,tloss,color = 'green',label = label)
    plt.legend()
    plt.grid()
    plt.show()

draw(x,tloss,'train loss')

draw(y,vloss,'val loss')

paddle.save(model.state_dict(), 'out/final.pdparams')  # 保存模型

print(val_data[10])print(val_label[10][0])

import paddle.nn.functional as F

label_txt = ['','Adware','Banking','SMS malware','Riskware','Benign']

model = Mlp()

para_dict = paddle.load('out/final.pdparams')
model.load_dict(para_dict)

model.eval()
x = paddle.reshape(paddle.to_tensor(val_data[10],dtype='float32'),[-1,470])
predicts = model(x)

pre = paddle.round(predicts)print("实际标签为:",label_txt[val_label[10][0]])print("预测标签为:",label_txt[pre[0]])