本来在做Berkeley的cs294的hw2，但是由于对gym环境，比如observation和action的数据形式，和对TensorFlow的不熟悉，所以针对gym的CartPole-v0环境做一个演员-评论家的demo。这样可以清楚的观察到所做的努力，在没有加入优势函数的时候，网络训练的效果较差，几乎没有，加入后有较大的改善，但是还是没有破百，加入后破百了但是没有上两百，在正则化之后分数可以随意飙高。
明确如下：
- CartPole-v0的环境下，目标是向一个方向连续滑动且保持杆子的平衡
- 对优势函数做正则化可以有效防止过拟合，使得最后获得的reward分数更高
- 该环境的数据是离散的
#!/usr/bin/env python
# -*- coding: utf8 -*-
import gym
import numpy as np
import tensorflow as tf
import cPickle as pickle
import matplotlib.pyplot as plt
import math
def buildNet(input_layer, output_shape=[None,2], scope='test', layer_size=4, size=10, output_activation=None):
layer = input_layer
with tf.variable_scope(scope):
for i in range(0, layer_size):
layer = tf.layers.dense(layer, size, activation=tf.tanh)
output_layer = tf.layers.dense(layer, output_shape, activation=output_activation, name="ac_logits")
return output_layer
env = gym.make('CartPole-v0')
discrete = isinstance(env.action_space, gym.spaces.Discrete)
obs_dim = env.observation_space.shape[0]
ac_dim = env.action_space.n if discrete else env.action_space.shape[0]
input_layer = tf.placeholder(tf.float32, [None, obs_dim], name="observation")
ac_logits = buildNet(input_layer, ac_dim, 'ac_test')
ac = tf.placeholder(tf.int32, [None], name="action")
ac_sample = tf.reshape(tf.multinomial(ac_logits,1),[-1]) #multinomial->从多项式分布中抽取样本，抽取的样本数是1;[-1]表示shape是缺省值，这样可以从中选出该选择哪类动作
logprob = -tf.nn.sparse_softmax_cross_entropy_with_logits(labels=ac,logits=ac_logits) #把概率分布转换成softmax形式，则如此所有概率之和为1
baseline = tf.squeeze(buildNet(input_layer, 1, 'baseline'))
base_target = tf.placeholder(tf.float32, [None], name='base_target')
base_loss = tf.nn.l2_loss(baseline-base_target) #对baseline的损失函数只要是普通的l2范数即可
base_train = tf.train.AdamOptimizer(0.001).minimize(base_loss)
adv = tf.placeholder(tf.float32, [None], name='adv')
loss = tf.reduce_mean(-logprob*adv)
train = tf.train.AdamOptimizer(0.001).minimize(loss)
# buildNet()
step = 200
i = 0
batch_size = 50
min_timesteps_per_batch = 500
gamma = 0.99
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
# print(sess.run(ac_logits, feed_dict={input_layer: ob[None]})[0])
for i in range(i, step):
paths = []
batch = 0
reward = 0
while True:
obs, acs, rews = [],[],[]
ob = env.reset()
while True:
# env.render()
obs.append(ob)
action = sess.run(ac_sample, {input_layer: ob[None]})
# print(action)
action = action[0]
acs.append(action)
# ob, rew, done, _ = env.step()
ob, rew, done, _ = env.step(action)
reward += rew
rews.append(rew)
if done:
break
path = {
'observation': obs,
'action': acs,
'reward': rews
}
# print(np.sum(rews))
paths.append(path)
batch+=len(path['reward'])
if batch >= min_timesteps_per_batch:
break
ob_no = np.concatenate([path["observation"] for path in paths])
ac_na = np.concatenate([path["action"] for path in paths])
q_n = []
for path in paths:
r = path['reward']
max_len = len(r)
q = [np.sum(np.power(gamma, np.arange(max_len-t)) * r[t:]) for t in range(max_len)]
q_n.extend(q)
b_n = sess.run(baseline, {input_layer:ob_no})
b_n = (b_n - np.mean(q_n)) / (np.std(q_n))
# q_n = q_n - b_n
adv_n = q_n - b_n
adv_n = (adv_n - np.mean(adv_n))/(np.std(adv_n))
print("step:",i,"reward:",reward/len(paths))
print(sess.run(train, {input_layer:ob_no, ac:ac_na, adv:adv_n}))
q_n = (q_n - np.mean(q_n)) / (np.std(q_n))
sess.run(base_train, {input_layer: ob_no, base_target:q_n})
#最终想要达到的效果，应该是能稳定平滑的向一边移动
