import numpy as np
import paddle.fluid as fluid
import gym


class Critic(object):
    def __init__(self, exe, gamma, state_dim, lr):
        # self.exe = fluid.Executor(fluid.CPUPlace())
        self.exe = exe
        self.state_dim = state_dim
        self.gamma = gamma
        self.lr = lr
        self._built_net()

    def _built_net(self):
        self.critic_program = fluid.Program()
        with fluid.program_guard(self.critic_program):
            state = fluid.data(name='state', shape=[None, self.state_dim], dtype='float32')
            reward = fluid.data(name='reward', shape=[None, 1], dtype='float32')
            next_reward = fluid.data(name='next_reward', shape=[None, 1], dtype='float32')
            # 创建网络
            h1 = fluid.layers.fc(input=state, size=24, act='relu')
            self.value = fluid.layers.fc(input=h1, size=1)
            self.td_error = reward + self.gamma * next_reward - self.value
            cost = fluid.layers.reduce_mean(fluid.layers.square(self.td_error))

            self.critic_program_test = self.critic_program.clone(for_test=True)
            fluid.optimizer.AdamOptimizer(learning_rate=self.lr).minimize(cost)

    def train(self, state, reward, next_state):
        next_reward = self.exe.run(self.critic_program_test,
                                   feed={'state': next_state},
                                   fetch_list=[self.value])[0]
        print(next_reward.shape)
        td_error = self.exe.run(self.critic_program,
                                feed={'state': state,
                                      'reward': reward,
                                      'next_reward': next_reward},
                                fetch_list=[self.td_error])[0]


exe = fluid.Executor(fluid.CPUPlace())
env = gym.make('CartPole-v1')
critic = Critic(exe, gamma=0.9, state_dim=4, lr=0.005)
exe.run(fluid.default_startup_program())

state = env.reset()
next_state, reward, done, info = env.step(0)


def dat(data):
    return np.expand_dims(data, axis=0).astype(np.float32)


critic.train(dat(state), dat(reward), dat(next_state))

有没有大神帮忙看一下。为什么会出现这种错误呢。

很好奇都没人碰到这种问题吗？还是大家都是用动态图写的？或者碰到这种问题的人被paddle劝退了？

paddlepaddle版本换到1.6.3试试

楼主解决这个问题了么....我也碰上参数未初始化了，欲哭无泪

看起来代码好像没什么问题

这个要手动调整你喂数据给网络的顺序，感觉有点蛋疼。

推荐使用动态图，静态图做强化学习我已经放弃了，强化学习你绕不过两个算法DQN， DDPG。

好像无关paddle的版本的问题。这个是paddle内部的一个机制。paddle静态图用起来真的要万分小心。

没有遇到过。