基于MobileNetV2的柠檬外观分类实践

!cd data $$/
!unzip -oq /home/aistudio/data/data73045/lemon_homework.zip !unzip -oq /home/aistudio/lemon_homework/lemon_lesson.zip -d /home/aistudio/lemon_homework/
!unzip -oq /home/aistudio/lemon_homework/lemon_lesson/test_images.zip -d /home/aistudio/lemon_homework/
!unzip -oq /home/aistudio/lemon_homework/lemon_lesson/train_images.zip -d /home/aistudio/lemon_homework/

# 导入所需要的库from sklearn.utils import shuffleimport osimport pandas as pdimport numpy as npfrom PIL import Imageimport paddleimport paddle.nn as nnfrom paddle.io import Datasetimport paddle.vision.transforms as Timport paddle.nn.functional as Ffrom paddle.metric import Accuracy 
#在python中运行代码经常会遇到的情况是——代码可以正常运行但是会提示警告，有时特别讨厌。#那么如何来控制警告输出呢？其实很简单，python通过调用warnings模块中定义的warn()函数来发出警告。我们可以通过警告过滤器进行控制是否发出警告消息。import warnings
warnings.filterwarnings("ignore")# 读取数据train_images = pd.read_csv('lemon_homework/train.csv', usecols=['id','class_num'])# labelshuffling 自定义标签打乱def labelShuffling(dataFrame, groupByName = 'class_num'):

    groupDataFrame = dataFrame.groupby(by=[groupByName])
    labels = groupDataFrame.size()    print("length of label is ", len(labels))
    maxNum = max(labels)
    lst = pd.DataFrame()    for i in range(len(labels)):        print("Processing label  :", i)
        tmpGroupBy = groupDataFrame.get_group(i)
        createdShuffleLabels = np.random.permutation(np.array(range(maxNum))) % labels[i]        print("Num of the label is : ", labels[i])
        lst=lst.append(tmpGroupBy.iloc[createdShuffleLabels], ignore_index=True)        print("Done" )    # lst.to_csv('test1.csv', index=False)
    return lst# 划分训练集和校验集all_size = len(train_images)# print(all_size)train_size = int(all_size * 0.8)
train_image_list = train_images[:train_size]
val_image_list = train_images[train_size:]

df = labelShuffling(train_image_list)
df = shuffle(df)

train_image_path_list = df['id'].values
label_list = df['class_num'].values
label_list = paddle.to_tensor(label_list, dtype='int64')
train_label_list = paddle.nn.functional.one_hot(label_list, num_classes=4)

val_image_path_list = val_image_list['id'].values
val_label_list = val_image_list['class_num'].values
val_label_list = paddle.to_tensor(val_label_list, dtype='int64')
val_label_list = paddle.nn.functional.one_hot(val_label_list, num_classes=4)# 定义数据预处理data_transforms = T.Compose([
    T.Resize(size=(224, 224)),
    T.RandomHorizontalFlip(224),
    T.RandomVerticalFlip(224),
    T.RandomRotation(90),
    T.Transpose(),    # HWC -> CHW
    T.Normalize(
        mean=[0.31169346, 0.25506335, 0.12432463],      #归一化  
        std=[0.34042713, 0.29819837, 0.1375536],
        to_rgb=True)    
        #计算过程：output[channel] = (input[channel] - mean[channel]) / std[channel]])

# 构建Datasetclass MyDataset(paddle.io.Dataset):
    """
    步骤一：继承paddle.io.Dataset类
    """
    def __init__(self, train_img_list, val_img_list,train_label_list,val_label_list, mode='train'):
        """
        步骤二：实现构造函数，定义数据读取方式，划分训练和测试数据集
        """
        super(MyDataset, self).__init__()
        self.img = []
        self.label = []        # 借助pandas读csv的库
        self.train_images = train_img_list
        self.test_images = val_img_list
        self.train_label = train_label_list
        self.test_label = val_label_list        if mode == 'train':            # 读train_images的数据
            for img,la in zip(self.train_images, self.train_label):
                self.img.append('lemon_homework/train_images/'+img)
                self.label.append(la)        else:            # 读test_images的数据
            for img,la in zip(self.test_images, self.test_label):
                self.img.append('lemon_homework/train_images/'+img)
                self.label.append(la)    def load_img(self, image_path):
        # 实际使用时使用Pillow相关库进行图片读取即可，这里我们对数据先做个模拟
        image = Image.open(image_path).convert('RGB')        return image    def __getitem__(self, index):
        """
        步骤三：实现__getitem__方法，定义指定index时如何获取数据，并返回单条数据（训练数据，对应的标签）
        """
        image = self.load_img(self.img[index])
        label = self.label[index]        # label = paddle.to_tensor(label)
        
        return data_transforms(image).astype("float32"), paddle.nn.functional.label_smooth(label)    def __len__(self):
        """
        步骤四：实现__len__方法，返回数据集总数目
        """
        return len(self.img)#train_loadertrain_dataset = MyDataset(train_img_list=train_image_path_list, val_img_list=val_image_path_list, train_label_list=train_label_list, val_label_list=val_label_list, mode='train')
train_loader = paddle.io.DataLoader(train_dataset, places=paddle.CPUPlace(), batch_size=32, shuffle=True, num_workers=0)#val_loaderval_dataset = MyDataset(train_img_list=train_image_path_list, val_img_list=val_image_path_list, train_label_list=train_label_list, val_label_list=val_label_list, mode='test')
val_loader = paddle.io.DataLoader(train_dataset, places=paddle.CPUPlace(), batch_size=32, shuffle=True, num_workers=0)

!rm vdl/vdlrecords.model.logfrom visualdl import LogReader, LogWriter

args={    'logdir':'./vdl',    'file_name':'vdlrecords.model.log',    'iters':0,
}

write = LogWriter(logdir=args['logdir'], file_name=args['file_name'])#iters 初始化为0iters = args['iters'] 

#自定义Callbackclass Callbk(paddle.callbacks.Callback):
    def __init__(self, write, iters=0):
        self.write = write
        self.iters = iters    def on_train_batch_end(self, step, logs):
        self.iters += 1
        #记录loss
        self.write.add_scalar(tag="loss",step=self.iters,value=logs['loss'][0])        #记录 accuracy
        self.write.add_scalar(tag="acc",step=self.iters,value=logs['acc'])

from work.mobilenet import MobileNetV2#定义输入input_define = paddle.static.InputSpec(shape=[-1,3,224,224], dtype="float32", name="img")
label_define = paddle.static.InputSpec(shape=[-1,1], dtype="int64", name="label")# 模型封装model_res = MobileNetV2(class_dim=4)

model = paddle.Model(model_res,inputs=input_define,labels=label_define)# 定义优化器scheduler = paddle.optimizer.lr.LinearWarmup(
        learning_rate=0.1, warmup_steps=20, start_lr=0, end_lr=0.1, verbose=True)
optim = paddle.optimizer.SGD(learning_rate=scheduler, parameters=model.parameters())# optim = paddle.optimizer.Adam(learning_rate=0.001, parameters=model.parameters())# 配置模型model.prepare(
    optim,
    paddle.nn.CrossEntropyLoss(soft_label=True),
    Accuracy()
    )# 模型训练与评估model.fit(
        train_loader,
        val_loader,
        epochs=50,
        callbacks=Callbk(write=write, iters=iters),
        verbose=1,
        batch_size=64,
        save_dir="/home/aistudio/iterhui/" #把模型参数、优化器参数保存至自定义的文件夹
        )

#模型保存model.save('Hapi_MyCNN', False)  # save for inferenceresult = model.evaluate(val_loader,batch_size=64,log_freq=100, verbose=1, num_workers=0, callbacks=Callbk(write=write, iters=iters))print(result)