模型压缩之剪枝（MLP）

np.percentile(a, q, axis=None, out=None, overwrite_input=False, interpolation='linear', keepdims=False)
 
a : array，用来算分位数的对象，可以是多维的数组
q : 介于0-100的float，用来计算是几分位的参数，如四分之一位就是25，如要算两个位置的数就(25,75)
axis : 坐标轴的方向，一维的就不用考虑了，多维的就用这个调整计算的维度方向，取值范围0/1
out : 输出数据的存放对象，参数要与预期输出有相同的形状和缓冲区长度
overwrite_input : bool，默认False，为True时及计算直接在数组内存计算，计算后原数组无法保存
interpolation : 取值范围{'linear', 'lower', 'higher', 'midpoint', 'nearest'}
            默认liner，比如取中位数，但是中位数有两个数字6和7，选不同参数来调整输出
keepdims : bool,默认False,为真时取中位数的那个轴将保留在结果中

# 作用：找到一组数的分位数值，如二分位数等(具体什么位置根据自己定义)# 方便我们之后设定剪枝的阈值import numpy as np
a = np.array([[1,2,3,4,5,6,7,8,9]])
np.percentile(a, 50)

# 导入所需包import paddleimport paddle.nn as nnimport paddle.nn.functional as Fimport paddle.utilsimport numpy as npimport mathfrom copy import deepcopyfrom matplotlib import pyplot as pltfrom paddle.io import Datasetfrom paddle.io import DataLoaderfrom paddle.vision import datasetsfrom paddle.vision import transforms

# 搭建基础线性层class MaskedLinear(nn.Linear):
    def __init__(self, in_features, out_features, bias=True):
        super(MaskedLinear, self).__init__(in_features, out_features, bias)
        self.mask_flag = False
        self.mask = None

    def set_mask(self, mask):
        self.mask = mask
        self.weight.set_value(self.weight * self.mask)
        self.mask_flag = True

    def get_mask(self):
        print(self.mask_flag)        return self.mask    def forward(self, x):
        if self.mask_flag:
            weight = self.weight * self.mask            return F.linear(x, weight, self.bias)        else:            return F.linear(x, self.weight, self.bias)

# 搭建MLP网络class MLP(nn.Layer):
    def __init__(self):
        super(MLP, self).__init__()
        self.linear1 = MaskedLinear(28 * 28 * 3, 200)
        self.relu1 = nn.ReLU()
        self.linear2 = MaskedLinear(200, 200)
        self.relu2 = nn.ReLU()
        self.linear3 = MaskedLinear(200, 10)    def forward(self, x):
        out = paddle.reshape(x, (x.shape[0], -1))
        out = self.relu1(self.linear1(out))
        out = self.relu2(self.linear2(out))
        out = self.linear3(out)        return out    def set_masks(self, masks):
        # Should be a less manual way to set masks
        # Leave it for the future
        self.linear1.set_mask(masks[0])
        self.linear2.set_mask(masks[1])
        self.linear3.set_mask(masks[2])

# 打印输出网络结构mlp_Net = MLP()
paddle.summary(mlp_Net,(1, 3, 28, 28))

# 图像转tensor操作，也可以加一些数据增强的方式，例如旋转、模糊等等# 数据增强的方式要加在Compose([  ])中def get_transforms(mode='train'):
    if mode == 'train':
        data_transforms = transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize(mean=[0.4914, 0.4822, 0.4465], std=[0.2023, 0.1994, 0.2010])])    else:
        data_transforms = transforms.Compose([
            transforms.ToTensor(),
            transforms.Normalize(mean=[0.4914, 0.4822, 0.4465], std=[0.2023, 0.1994, 0.2010])])    return data_transforms# 获取官方MNIST数据集def get_dataset(name='MNIST', mode='train'):
    if name == 'MNIST':
        dataset = datasets.MNIST(mode=mode, transform=get_transforms(mode))    return dataset# 定义数据加载到模型形式def get_dataloader(dataset, batch_size=128, mode='train'):
    dataloader = DataLoader(dataset, batch_size=batch_size, num_workers=2, shuffle=(mode == 'train'))    return dataloader

# 初始化函数，用于模型初始化class AverageMeter():
    """ Meter for monitoring losses"""
    def __init__(self):
        self.avg = 0
        self.sum = 0
        self.cnt = 0
        self.reset()    def reset(self):
        """reset all values to zeros"""
        self.avg = 0
        self.sum = 0
        self.cnt = 0

    def update(self, val, n=1):
        """update avg by val and n, where val is the avg of n values"""
        self.sum += val * n
        self.cnt += n
        self.avg = self.sum / self.cnt

# mlp网络训练def mlp_train_one_epoch(model, dataloader, criterion, optimizer, epoch, total_epoch, report_freq=20):
    print(f'----- Training Epoch [{epoch}/{total_epoch}]:')
    loss_meter = AverageMeter()
    acc_meter = AverageMeter()
    model.train()    for batch_idx, data in enumerate(dataloader):
        image = data[0]
        label = data[1]

        out = model(image)
        loss = criterion(out, label)

        loss.backward()
        optimizer.step()
        optimizer.clear_grad()

        pred = nn.functional.softmax(out, axis=1)
        acc1 = paddle.metric.accuracy(pred, label)

        batch_size = image.shape[0]
        loss_meter.update(loss.cpu().numpy()[0], batch_size)
        acc_meter.update(acc1.cpu().numpy()[0], batch_size)        if batch_idx > 0 and batch_idx % report_freq == 0:            print(f'----- Batch[{batch_idx}/{len(dataloader)}], Loss: {loss_meter.avg:.5}, Acc@1: {acc_meter.avg:.4}')    print(f'----- Epoch[{epoch}/{total_epoch}], Loss: {loss_meter.avg:.5}, Acc@1: {acc_meter.avg:.4}')

# mlp网络预测def mlp_validate(model, dataloader, criterion, report_freq=10):
    print('----- Validation')
    loss_meter = AverageMeter()
    acc_meter = AverageMeter()
    model.eval()    for batch_idx, data in enumerate(dataloader):
        image = data[0]
        label = data[1]

        out = model(image)
        loss = criterion(out, label)

        pred = paddle.nn.functional.softmax(out, axis=1)
        acc1 = paddle.metric.accuracy(pred, label)
        batch_size = image.shape[0]
        loss_meter.update(loss.cpu().numpy()[0], batch_size)
        acc_meter.update(acc1.cpu().numpy()[0], batch_size)        if batch_idx > 0 and batch_idx % report_freq == 0:            print(f'----- Batch [{batch_idx}/{len(dataloader)}], Loss: {loss_meter.avg:.5}, Acc@1: {acc_meter.avg:.4}')    print(f'----- Validation Loss: {loss_meter.avg:.5}, Acc@1: {acc_meter.avg:.4}')

def weight_prune(model, pruning_perc):
    '''
    Prune pruning_perc % weights layer-wise
    '''
    threshold_list = []    for p in model.parameters():        if len(p.shape) != 1: # bias
            weight = p.abs().numpy().flatten()  # 将权重参数拉伸为1维
            threshold = np.percentile(weight, pruning_perc)   # 根据阈值对权重参数进行筛选
            threshold_list.append(threshold)    # generate mask
    masks = []
    idx = 0
    for p in model.parameters():        if len(p.shape) != 1:
            pruned_inds = p.abs() > threshold_list[idx]         # 返回bool矩阵
            pruned_inds = paddle.cast(pruned_inds, 'float32')   # paddle.cast将bool->float
            masks.append(pruned_inds)
            idx += 1
    return masks

# mlp网络主函数def mlp_main():
    total_epoch = 1
    batch_size = 256

    model = MLP()
    train_dataset = get_dataset(mode='train')
    train_dataloader = get_dataloader(train_dataset, batch_size, mode='train')
    val_dataset = get_dataset(mode='test')
    val_dataloader = get_dataloader(val_dataset, batch_size, mode='test')
    criterion = nn.CrossEntropyLoss()
    scheduler = paddle.optimizer.lr.CosineAnnealingDecay(0.02, total_epoch)
    optimizer = paddle.optimizer.Momentum(learning_rate=scheduler,
                                          parameters=model.parameters(),
                                          momentum=0.9,
                                          weight_decay=5e-4)

    eval_mode = False
    if eval_mode:
        state_dict = paddle.load('./mlp_ep2.pdparams')
        model.set_state_dict(state_dict)
        mlp_validate(model, val_dataloader, criterion)        return

    save_freq = 5
    test_freq = 1
    for epoch in range(1, total_epoch+1):
        mlp_train_one_epoch(model, train_dataloader, criterion, optimizer, epoch, total_epoch)
        scheduler.step()        if epoch % test_freq == 0 or epoch == total_epoch:
            mlp_validate(model, val_dataloader, criterion)        if epoch % save_freq == 0 or epoch == total_epoch:
            paddle.save(model.state_dict(), f'./mlp_ep{epoch}.pdparams')
            paddle.save(optimizer.state_dict(), f'./mlp_ep{epoch}.pdopts')    # 剪枝后的效果
    print("\n=====Pruning 60%=======\n")
    pruned_model = deepcopy(model)
    mask = weight_prune(pruned_model, 60)
    pruned_model.set_masks(mask)
    mlp_validate(pruned_model, val_dataloader, criterion)    return model,pruned_model

# 返回值是剪枝前后网络模型mlp_model, mlp_pruned_model = mlp_main()

# 定义模型权重展示函数def plot_weights(model):
    modules = [module for module in model.sublayers()]
    num_sub_plot = 0
    for i, layer in enumerate(modules):        if hasattr(layer, 'weight'):
            plt.subplot(131+num_sub_plot)
            w = layer.weight
            w_one_dim = w.cpu().numpy().flatten()
            plt.hist(w_one_dim[w_one_dim!=0], bins=50)
            num_sub_plot += 1
    plt.show()

# 剪枝前的权重plot_weights(mlp_model)

# 剪枝后的权重plot_weights(mlp_pruned_model)

# 找出特定元素的位置# 筛选出True值对应位置的数据np.random.seed(7) #相同的种子可确保随机数按序生成时是相同的，结果可重现b = np.random.randint(40, 100, size=(6,6)) 	 # 生成40到100,6x6个随机数print('b={}\nb中小于70的元素为\n\n{}'.format(b,b<70))  
ind = np.where(b>60,b,0)  # 返回的是一个tuple 类型print("np.where(b>60,b,0)=\n{}".format(ind))