强化学习-cs294-hw2-demo 强化学习-cs294-hw2-demo

【强化学习-cs294-hw2-demo】本来在做Berkeley的cs294的hw2，但是由于对gym环境，比如observation和action的数据形式，和对TensorFlow的不熟悉，所以针对gym的CartPole-v0环境做一个演员-评论家的demo。这样可以清楚的观察到所做的努力，在没有加入优势函数的时候，网络训练的效果较差，几乎没有，加入后有较大的改善，但是还是没有破百，加入后破百了但是没有上两百，在正则化之后分数可以随意飙高。
明确如下：

CartPole-v0的环境下，目标是向一个方向连续滑动且保持杆子的平衡
对优势函数做正则化可以有效防止过拟合，使得最后获得的reward分数更高
该环境的数据是离散的

#!/usr/bin/env python # -*- coding: utf8 -*-import gym import numpy as np import tensorflow as tf import cPickle as pickle import matplotlib.pyplot as plt import mathdef buildNet(input_layer, output_shape=[None,2], scope='test', layer_size=4, size=10, output_activation=None): layer = input_layer with tf.variable_scope(scope): for i in range(0, layer_size): layer = tf.layers.dense(layer, size, activation=tf.tanh) output_layer = tf.layers.dense(layer, output_shape, activation=output_activation, name="ac_logits") return output_layerenv = gym.make('CartPole-v0') discrete = isinstance(env.action_space, gym.spaces.Discrete) obs_dim = env.observation_space.shape[0] ac_dim = env.action_space.n if discrete else env.action_space.shape[0]input_layer = tf.placeholder(tf.float32, [None, obs_dim], name="observation") ac_logits = buildNet(input_layer, ac_dim, 'ac_test') ac = tf.placeholder(tf.int32, [None], name="action") ac_sample = tf.reshape(tf.multinomial(ac_logits,1),[-1]) #multinomial->从多项式分布中抽取样本，抽取的样本数是1; [-1]表示shape是缺省值，这样可以从中选出该选择哪类动作 logprob = -tf.nn.sparse_softmax_cross_entropy_with_logits(labels=ac,logits=ac_logits) #把概率分布转换成softmax形式，则如此所有概率之和为1baseline = tf.squeeze(buildNet(input_layer, 1, 'baseline')) base_target = tf.placeholder(tf.float32, [None], name='base_target') base_loss = tf.nn.l2_loss(baseline-base_target) #对baseline的损失函数只要是普通的l2范数即可 base_train = tf.train.AdamOptimizer(0.001).minimize(base_loss)adv = tf.placeholder(tf.float32, [None], name='adv')loss = tf.reduce_mean(-logprob*adv) train = tf.train.AdamOptimizer(0.001).minimize(loss)# buildNet() step = 200i = 0 batch_size = 50 min_timesteps_per_batch = 500 gamma = 0.99init = tf.global_variables_initializer()with tf.Session() as sess: sess.run(init) # print(sess.run(ac_logits, feed_dict={input_layer: ob[None]})[0])for i in range(i, step): paths = [] batch = 0 reward = 0 while True: obs, acs, rews = [],[],[] ob = env.reset() while True: # env.render() obs.append(ob) action = sess.run(ac_sample, {input_layer: ob[None]}) # print(action) action = action[0] acs.append(action) # ob, rew, done, _ = env.step() ob, rew, done, _ = env.step(action) reward += rew rews.append(rew) if done: breakpath = { 'observation': obs, 'action': acs, 'reward': rews } # print(np.sum(rews)) paths.append(path) batch+=len(path['reward']) if batch >=min_timesteps_per_batch: break ob_no = np.concatenate([path["observation"] for path in paths]) ac_na = np.concatenate([path["action"] for path in paths]) q_n = [] for path in paths: r = path['reward'] max_len = len(r) q = [np.sum(np.power(gamma, np.arange(max_len-t)) * r[t:]) for t in range(max_len)] q_n.extend(q) b_n = sess.run(baseline, {input_layer:ob_no}) b_n = (b_n - np.mean(q_n)) / (np.std(q_n)) # q_n = q_n - b_n adv_n = q_n - b_n adv_n = (adv_n - np.mean(adv_n))/(np.std(adv_n)) print("step:",i,"reward:",reward/len(paths)) print(sess.run(train, {input_layer:ob_no, ac:ac_na, adv:adv_n})) q_n = (q_n - np.mean(q_n)) / (np.std(q_n)) sess.run(base_train, {input_layer: ob_no, base_target:q_n})#最终想要达到的效果，应该是能稳定平滑的向一边移动