基于用户画像的商品推荐挑战赛Baseline,初赛0.65线上

from multiprocessing import cpu_countimport shutilimport osimport paddleimport paddle.fluid as fluidimport pandas as pd
paddle.enable_static()
train = pd.read_csv('./训练集/train.txt', header=None)
test = pd.read_csv('./测试集/apply_new.txt', header=None)
train.columns = ['pid', 'label', 'gender', 'age', 'tagid', 'time', 'province', 'city', 'model', 'make']
test.columns = ['pid', 'gender', 'age', 'tagid', 'time', 'province', 'city', 'model', 'make']
data = pd.concat([train, test])
data['tagid'] = data['tagid'].apply(lambda x: ' '.join(map(str, eval(x))))print(train.shape,test.shape)

# 生成数据字典dict_set = {}
ik = 1for tagid in data['tagid'].values:    for s in tagid.split(' '):        if s in dict_set:            pass
        else:
            dict_set[s] = ik
            ik = ik + 1dict_dim = len(dict_set) + 1# 根据数据字典编码tagiddata['input_data'] = data['tagid'].apply(lambda x: ','.join([str(dict_set[i]) for i in x.split(' ')]))print(data.shape,dict_dim)

# 定义网络def bilstm_net(data, dict_dim, class_dim, emb_dim=128, hid_dim=128, hid_dim2=96, emb_lr=30.0):
    # embedding layer
    emb = fluid.layers.embedding(input=data,
                                 size=[dict_dim, emb_dim],
                                 param_attr=fluid.ParamAttr(learning_rate=emb_lr))    # bi-lstm layer
    fc0 = fluid.layers.fc(input=emb, size=hid_dim * 4)

    rfc0 = fluid.layers.fc(input=emb, size=hid_dim * 4)

    lstm_h, c = fluid.layers.dynamic_lstm(input=fc0, size=hid_dim * 4, is_reverse=False)

    rlstm_h, c = fluid.layers.dynamic_lstm(input=rfc0, size=hid_dim * 4, is_reverse=True)    # extract last layer
    lstm_last = fluid.layers.sequence_last_step(input=lstm_h)
    rlstm_last = fluid.layers.sequence_last_step(input=rlstm_h)    # concat layer
    lstm_concat = fluid.layers.concat(input=[lstm_last, rlstm_last], axis=1)    # full connect layer
    fc1 = fluid.layers.fc(input=lstm_concat, size=hid_dim2, act='tanh')    # softmax layer
    prediction = fluid.layers.fc(input=fc1, size=class_dim, act='softmax')    return prediction

train = data[:train.shape[0]]
test = data[train.shape[0]:]

train.head(5)

<br/>

# 训练数据的预处理def train_mapper(sample):
    data, label = sample
    data = [int(data) for data in data.split(',')]    return data, int(label)# 训练数据的readerdef train_reader():
    def reader():
        for line in train[['input_data', 'label']].values:
            data, label = line[0], line[1]            yield data, label    return paddle.reader.xmap_readers(train_mapper, reader, cpu_count(), 128)

# 定义输入数据， lod_level不为0指定输入数据为序列数据words = fluid.layers.data(name='input_data', shape=[1], dtype='int64', lod_level=1)
label = fluid.layers.data(name='label', shape=[1], dtype='int64')# 输入，字典，输出model = bilstm_net(words, dict_dim, 2)# 获取损失函数和准确率cost = fluid.layers.cross_entropy(input=model, label=label)
avg_cost = fluid.layers.mean(cost)
acc = fluid.layers.accuracy(input=model, label=label)# 获取预测程序test_program = fluid.default_main_program().clone(for_test=True)# 定义优化方法optimizer = fluid.optimizer.AdagradOptimizer(learning_rate=0.002)
opt = optimizer.minimize(avg_cost)# 创建一个执行器place = fluid.CUDAPlace(0)
exe = fluid.Executor(place)# 进行参数初始化exe.run(fluid.default_startup_program())

train_reader = paddle.batch(reader=train_reader(), batch_size=2048)# 定义输入数据的维度feeder = fluid.DataFeeder(place=place, feed_list=[words, label])# 开始训练for pass_id in range(5):    # 进行训练
    bst = 0
    for batch_id, data in enumerate(train_reader()):
        train_cost, train_acc = exe.run(program=fluid.default_main_program(),
                                        feed=feeder.feed(data),
                                        fetch_list=[avg_cost, acc])        if batch_id % 25 == 0:            print('Pass:%d, Batch:%d, Cost:%0.5f, Acc:%0.5f' % (pass_id, batch_id, train_cost[0], train_acc[0]))            # # 进行测试
            # test_costs = []
            # test_accs = []
            # for batch_id, data in enumerate(train_reader()):
            #     test_cost, test_acc = exe.run(program=test_program,
            #                                   feed=feeder.feed(data),
            #                                   fetch_list=[avg_cost, acc])
            #     test_costs.append(test_cost[0])
            #     test_accs.append(test_acc[0])
            # # 计算平均预测损失在和准确率
            # test_cost = (sum(test_costs) / len(test_costs))
            # test_acc = (sum(test_accs) / len(test_accs))
            # print('Test:%d, Cost:%0.5f, ACC:%0.5f' % (pass_id, test_cost, test_acc))
    if train_acc[0] > bst:
        bst = train_acc        # 保存预测模型
        save_path = './tmp_123'
        # 删除旧的模型文件
        shutil.rmtree(save_path, ignore_errors=True)        # 创建保持模型文件目录
        os.makedirs(save_path)        # 保存预测模型
        fluid.io.save_inference_model(save_path, feeded_var_names=[words.name], target_vars=[model], executor=exe)

# 保存预测模型路径save_path = './tmp_123'# 从模型中获取预测程序、输入数据名称列表、分类器[infer_program, feeded_var_names, target_var] = fluid.io.load_inference_model(dirname=save_path, executor=exe)

test_data = []for input_data in test['input_data'].values:
    tmp = []    for input_data_1 in input_data.split(','):
        tmp.append(int(input_data_1))
    test_data.append(tmp)

result = []
n = 4096for b in [test_data[i:i + n] for i in range(0, len(test_data), n)]:
    tensor_words = fluid.create_lod_tensor(b, [[len(x) for x in b]], place)
    t_result = exe.run(program=infer_program,
                 feed={feeded_var_names[0]: tensor_words},
                 fetch_list=target_var)
    result.extend(list(t_result[0][:,1]))

submit = test[['pid']]
submit['tmp_train'] = result
submit.columns = ['user_id', 'tmp_train']

submit['rank'] = submit['tmp_train'].rank()
submit['category_id'] = 1submit.loc[submit['rank'] <= int(submit.shape[0] * 0.5), 'category_id'] = 0

print(submit['category_id'].mean())
submit[['user_id', 'category_id']].to_csv('paddle.csv', index=False)