Tic-Tac-Toe：井字游戏（井字棋）

import numpy as npimport paddlefrom Model import Modelfrom VictoryRule import Rulefrom QLearning import QLearningfrom visualdl import LogWriter
log_writer = LogWriter(logdir="./log")


Max_Epoch = 200           #最大迭代次数xsize = 3                 #多少行ysize = 3                 #多少列winnum = 3                #连珠数，多少个连珠则获胜learning_rate = 1e-3      #学习率decay_rate = 0.6          #每步衰减率player=1                  #玩家是数字，非0，非负computer=2                #电脑的数字，非0，非负remain = []               #地图中剩余可下棋子位置rule = Rule(xsize,ysize,winnum) #规则Qchart = QLearning(xsize * ysize,decay_rate)#Q表格player_model = Model(xsize * ysize,xsize * ysize)
player_model.train()
computer_model = Model(xsize * ysize,xsize * ysize)
computer_model.train()
player_optimizer = paddle.optimizer.SGD(parameters=player_model.parameters(),
                                  learning_rate=learning_rate)
computer_optimizer = paddle.optimizer.SGD(parameters=computer_model.parameters(),
                                  learning_rate=learning_rate)def restart():
    "重启环境"
    Qchart.clear()
    remain.clear()
    rule.map = np.zeros(xsize * ysize,dtype=int)    for i in range(xsize * ysize):
        remain.append(i)def modelupdate(player_loss,computer_loss):
    "模型更新"
    log_writer.add_scalar(tag="player/loss", step=epoch, value=player_loss.numpy())
    log_writer.add_scalar(tag="computer/loss", step=epoch, value=computer_loss.numpy())    # 梯度更新
    player_loss.backward()
    computer_loss.backward()
    player_optimizer.step()
    player_optimizer.clear_grad()
    computer_optimizer.step()
    computer_optimizer.clear_grad()
    paddle.save(player_model.state_dict(),'player_model')
    paddle.save(computer_model.state_dict(),'computer_model')    

for i in range(xsize * ysize):
    remain.append(i)for epoch in range(Max_Epoch):    while True:
        player_predict = player_model(paddle.to_tensor(rule.map, dtype='float32',stop_gradient=False))#玩家方预测
        for pred in np.argsort(-player_predict.numpy()):            if pred in remain:
                remain.remove(pred)                break
        rule.map[pred] = player
        Qchart.update(pred,'player')        print('player down at {}'.format(pred))
        overcode=rule.checkover(pred,player)        if overcode == player:            "获胜方为玩家"
            player_loss = paddle.nn.functional.mse_loss(player_predict, paddle.to_tensor(Qchart.playerstep, dtype='float32', stop_gradient=False))
            computer_loss = paddle.nn.functional.mse_loss(computer_predict, paddle.to_tensor(-1 * Qchart.computerstep, dtype='float32', stop_gradient=False))#损失计算中，失败方的label为每步的负数
            print("Player Victory!")            print(rule.map.reshape(xsize,ysize))            #print("epoch:{}\tplayer loss:{}\tcomputer loss:{}".format(epoch,player_loss.numpy()[0],computer_loss.numpy()[0]))
            modelupdate(player_loss,computer_loss)
            restart()            break
        elif overcode == 0:
            player_loss = paddle.nn.functional.mse_loss(player_predict, paddle.to_tensor(Qchart.playerstep, dtype='float32', stop_gradient=False))
            computer_loss = paddle.nn.functional.mse_loss(computer_predict, paddle.to_tensor(Qchart.computerstep, dtype='float32', stop_gradient=False))            print("Draw!")            print(rule.map.reshape(xsize,ysize))            #print("epoch:{}\tplayer loss:{}\tcomputer loss:{}".format(epoch,player_loss.numpy()[0],computer_loss.numpy()[0]))
            modelupdate(player_loss,computer_loss)
            restart()            break
        computer_predict = computer_model(paddle.to_tensor(rule.map, dtype='float32',stop_gradient=False))#电脑方预测
        for pred in np.argsort(-computer_predict.numpy()):            if pred in remain:
                remain.remove(pred)                break
        rule.map[pred] = computer
        Qchart.update(pred,'computer')        print('computer down at {}'.format(pred))
        overcode=rule.checkover(pred, computer)        if overcode == computer:
            player_loss = paddle.nn.functional.mse_loss(player_predict, paddle.to_tensor(-1 * Qchart.playerstep, dtype='float32', stop_gradient=False))
            computer_loss = paddle.nn.functional.mse_loss(computer_predict, paddle.to_tensor(Qchart.computerstep, dtype='float32', stop_gradient=False))            print("Computer Victory!")            print(rule.map.reshape(xsize,ysize))            #print("epoch:{}\tplayer loss:{}\tcomputer loss:{}".format(epoch,player_loss.numpy()[0],computer_loss.numpy()[0]))
            modelupdate(player_loss,computer_loss)
            restart()            break
        elif overcode == 0:
            player_loss = paddle.nn.functional.mse_loss(player_predict, paddle.to_tensor(Qchart.playerstep, dtype='float32', stop_gradient=False))
            computer_loss = paddle.nn.functional.mse_loss(computer_predict, paddle.to_tensor(Qchart.computerstep, dtype='float32', stop_gradient=False))            print("Draw!")            print(rule.map.reshape(xsize,ysize))            #print("epoch:{}\tplayer loss:{}\tcomputer loss:{}".format(epoch,player_loss.numpy()[0],computer_loss.numpy()[0]))
            modelupdate(player_loss,computer_loss)
            restart()            break

player down at 7computer down at 3player down at 1computer down at 8player down at 6computer down at 2player down at 0computer down at 5Computer Victory!
[[1 1 2]
 [2 0 2]
 [1 1 2]]