柠檬分类竞赛：飞桨图像分类套件PaddleClas竞赛实战

cd path_to_clone_PaddleClas

以下二者任选其一
git clone https://github.com/PaddlePaddle/PaddleClas.git
git clone https://gitee.com/paddlepaddle/PaddleClas.git

pip install --upgrade -r requirements.txt

export PYTHONPATH=path_to_PaddleClas:$PYTHONPATH

!git clone https://gitee.com/paddlepaddle/PaddleClas.git

!pip install -r PaddleClas/requirements.txt

ILSVRC2012_val_00000001.JPEG 65...

TRAIN: # 训练配置
    batch_size: 32 # 训练的batch size
    num_workers: 4 # 每个trainer(1块GPU上可以视为1个trainer)的进程数量
    file_list: "./dataset/flowers102/train_list.txt" # 训练集标签文件，每一行由"image_name label"组成
    data_dir: "./dataset/flowers102/" # 训练集的图像数据路径
    shuffle_seed: 0 # 数据打散的种子
    transforms: # 训练图像的数据预处理
        - DecodeImage: # 解码
            to_rgb: True
            to_np: False
            channel_first: False
        - RandCropImage: # 随机裁剪
            size: 224
        - RandFlipImage: # 随机水平翻转
            flip_code: 1
        - NormalizeImage: # 归一化
            scale: 1./255.
            mean: [0.485, 0.456, 0.406]            std: [0.229, 0.224, 0.225]            order: ''
        - ToCHWImage: # 通道转换

!unzip data/data71799/lemon_lesson.zip

!unzip lemon_lesson/train_images.zip -d lemon_lesson/

!unzip lemon_lesson/test_images.zip -d lemon_lesson/

import pandas as pdimport codecsimport osfrom PIL import Image

df = pd.read_csv('lemon_lesson/train_images.csv')

all_file_dir = 'lemon_lesson'train_file = codecs.open(os.path.join(all_file_dir, "train_list.txt"), 'w')
eval_file = codecs.open(os.path.join(all_file_dir, "eval_list.txt"), 'w')

image_path_list = df['id'].values
label_list = df['class_num'].values# 划分训练集和校验集all_size = len(image_path_list)
train_size = int(all_size * 0.8)
train_image_path_list = image_path_list[:train_size]
train_label_list = label_list[:train_size]
val_image_path_list = image_path_list[train_size:]
val_label_list = label_list[train_size:]

image_path_pre = 'lemon_lesson/train_images'for file,label_id in zip(train_image_path_list, train_label_list):    # print(file)
    # print(label_id)
    try:
        img = Image.open(os.path.join(image_path_pre, file))        
        # train_file.write("{0}\0{1}\n".format(os.path.join(image_path_pre, file), label_id))
        train_file.write("{0}{1}{2}\n".format(os.path.join(image_path_pre, file),' ', label_id))        # eval_file.write("{0}\t{1}\n".format(os.path.join(image_path_pre, file), label_id))
    except Exception as e:        pass
        # 存在一些文件打不开，此处需要稍作清洗
        # print('error!')for file,label_id in zip(val_image_path_list, val_label_list):    # print(file)
    # print(label_id)
    try:
        img = Image.open(os.path.join(image_path_pre, file))        # train_file.write("{0}\t{1}\n".format(os.path.join(image_path_pre, file), label_id))
        eval_file.write("{0}{1}{2}\n".format(os.path.join(image_path_pre, file),' ', label_id))    except Exception as e:        # pass
        # 存在一些文件打不开，此处需要稍作清洗
        print('error!')

train_file.close()
eval_file.close()

!cp -r lemon_lesson  MyDataset

!pip install paddlex

import pandas as pdimport codecsimport osfrom PIL import Image

df = pd.read_csv('MyDataset/train_images.csv')
image_path_list = df['id'].values
label_list = df['class_num'].values

df.class_num.value_counts()

!mkdir MyDataset/0!mkdir MyDataset/1!mkdir MyDataset/2!mkdir MyDataset/3

class_0 = df[df.class_num==0]
class_0_list = class_0['id'].values

import shutilfor i in class_0_list:    try:
        shutil.copy(os.path.join('MyDataset/train_images', i), os.path.join('MyDataset/0', i))    except Exception as e:        pass

class_1 = df[df.class_num==1]
class_1_list = class_1['id'].values

import shutilfor i in class_1_list:    try:
        shutil.copy(os.path.join('MyDataset/train_images', i), os.path.join('MyDataset/1', i))    except Exception as e:        pass

class_2 = df[df.class_num==2]
class_2_list = class_2['id'].valuesimport shutilfor i in class_2_list:    try:
        shutil.copy(os.path.join('MyDataset/train_images', i), os.path.join('MyDataset/2', i))    except Exception as e:        pass

class_3 = df[df.class_num==3]
class_3_list = class_3['id'].valuesimport shutilfor i in class_3_list:    try:
        shutil.copy(os.path.join('MyDataset/train_images', i), os.path.join('MyDataset/3', i))    except Exception as e:        pass

!ls MyDataset/0 -l |grep "^-"|wc -l
!ls MyDataset/1 -l |grep "^-"|wc -l
!ls MyDataset/2 -l |grep "^-"|wc -l
!ls MyDataset/3 -l |grep "^-"|wc -l

# 使用PaddleX数据切分API要删除多余目录和list文件，否则会出现异常!rm -r MyDataset/train_images
!rm -r MyDataset/test_images
!rm MyDataset/*.txt

!paddlex --split_dataset --format ImageNet --dataset_dir MyDataset --val_value 0.2 --test_value 0.1

python tools/download.py -a MobileNetV3_small_x1_0 -p ./pretrained -d True

python tools/train.py \
    -c configs/quick_start/MobileNetV3_large_x1_0_finetune.yaml \
    -o pretrained_model="" \
    -o use_gpu=True

%cd PaddleClas/

!ls -ll

!python tools/download.py -a MobileNetV3_large_x1_0 -p ./pretrained -d True

    # 在`./logdir`路径下建立日志文件
    with LogWriter(logdir="./logdir") as writer:        for epoch_id in range(last_epoch_id + 1, config.epochs):
            net.train()            # 1. train with train dataset
            program.run(train_dataloader, config, net, optimizer, lr_scheduler,
                        epoch_id, 'train')            # 2. validate with validate dataset
            if config.validate and epoch_id % config.valid_interval == 0:
                net.eval()                with paddle.no_grad():
                    top1_acc = program.run(valid_dataloader, config, net, None,                                        None, epoch_id, 'valid')                if top1_acc > best_top1_acc:
                    best_top1_acc = top1_acc
                    best_top1_epoch = epoch_id                    if epoch_id % config.save_interval == 0:
                        model_path = os.path.join(config.model_save_dir,
                                                config.ARCHITECTURE["name"])
                        save_model(net, optimizer, model_path, "best_model")
                message = "The best top1 acc {:.5f}, in epoch: {:d}".format(
                    best_top1_acc, best_top1_epoch)
                logger.info("{:s}".format(logger.coloring(message, "RED")))                # 使用scalar组件记录一个标量数据
                writer.add_scalar(tag="val_acc", step=epoch_id, value=top1_acc)

!cp ../train.py tools/train.py

# 开始训练!python tools/train.py -c ../MobileNetV3_large_x1_0_finetune.yaml

!python tools/eval.py \
    -c ../MobileNetV3_large_x1_0_finetune.yaml \
    -o pretrained_model="./output/MobileNetV3_large_x1_0/best_model/ppcls"\
    -o load_static_weights=False

TRAIN:
    batch_size: 32
    num_workers: 0
    file_list: "../lemon_lesson/train_list.txt"
    data_dir: "../"
    shuffle_seed: 2021
    transforms:
        - DecodeImage:
            to_rgb: True
            to_np: False
            channel_first: False
        - RandCropImage:
            size: 224
        - RandFlipImage:
            flip_code: -1
        - RandomErasing: # 使用RandomErasing方法进行数据增广
            EPSILON: 0.5
        - NormalizeImage:
            scale: 1./255.
            mean: [0.485, 0.456, 0.406]            std: [0.229, 0.224, 0.225]            order: ''
        - ToCHWImage:

!cp ../random_erasing.py ppcls/data/imaug/random_erasing.py

!python tools/train.py -c ../MobileNetV3_large_x1_0_finetune.yaml

!cp ../img_aug.py ./

!mkdir ../img_aug

!python img_aug.py

python tools/export_model.py \    --model=模型名字 \    --pretrained_model=预训练模型路径 \    --output_path=预测模型保存路径

python tools/infer/predict.py \
    -m model文件路径 \    -p params文件路径 \    -i 图片路径 \    --use_gpu=1 \    --use_tensorrt=True

# 注意要写入类别数!python tools/export_model.py \
    --model=MobileNetV3_large_x1_0 \
    --pretrained_model=output/MobileNetV3_large_x1_0/best_model/ppcls \
    --output_path=inference \
    --class_dim 4

!ls ../lemon_lesson/test_images -l |grep "^-"|wc -l

# 可以预测整个目录!python tools/infer/predict.py \
    --model_file inference/inference.pdmodel \
    --params_file inference/inference.pdiparams \
    --image_file ../lemon_lesson/test_images \
    --use_gpu=True

!cp ../predict.py tools/infer/submit.py

# 可以预测整个目录!python tools/infer/submit.py \
    --model_file inference/inference.pdmodel \
    --params_file inference/inference.pdiparams \
    --image_file ../lemon_lesson/test_images \
    --use_gpu=True

%cd ~

# 环境变量配置，用于控制是否使用GPU# 说明文档：https://paddlex.readthedocs.io/zh_CN/develop/appendix/parameters.html#gpuimport os
os.environ['CUDA_VISIBLE_DEVICES'] = '0'from paddlex.cls import transformsimport paddlex as pdx# 定义训练和验证时的transforms# API说明https://paddlex.readthedocs.io/zh_CN/develop/apis/transforms/cls_transforms.htmltrain_transforms = transforms.Compose([
    transforms.RandomCrop(crop_size=224), transforms.RandomHorizontalFlip(),
    transforms.Normalize()
])
eval_transforms = transforms.Compose([
    transforms.ResizeByShort(short_size=256),
    transforms.CenterCrop(crop_size=224), transforms.Normalize()
])# 定义训练和验证所用的数据集# API说明：https://paddlex.readthedocs.io/zh_CN/develop/apis/datasets.html#paddlex-datasets-imagenet# 主要就是把数据集目录配置进去train_dataset = pdx.datasets.ImageNet(
    data_dir='MyDataset',
    file_list='MyDataset/train_list.txt',
    label_list='MyDataset/labels.txt',
    transforms=train_transforms,
    shuffle=True)
eval_dataset = pdx.datasets.ImageNet(
    data_dir='MyDataset',
    file_list='MyDataset/val_list.txt',
    label_list='MyDataset/labels.txt',
    transforms=eval_transforms)# 初始化模型，并进行训练# 可使用VisualDL查看训练指标，参考https://paddlex.readthedocs.io/zh_CN/develop/train/visualdl.htmlmodel = pdx.cls.MobileNetV3_small_ssld(num_classes=len(train_dataset.labels))# API说明：https://paddlex.readthedocs.io/zh_CN/develop/apis/models/classification.html#train# 各参数介绍与调整说明：https://paddlex.readthedocs.io/zh_CN/develop/appendix/parameters.htmlmodel.train(
    num_epochs=10,
    train_dataset=train_dataset,
    train_batch_size=32,
    eval_dataset=eval_dataset,
    lr_decay_epochs=[4, 6, 8],
    learning_rate=0.025,
    save_dir='output/MobileNetV3_small_ssld',
    use_vdl=True)