How to fix AttributeError: 'tuple' object has no attribute 'rank' creating a DqnAgent with Tensorflow?

Question

我正在尝试使用自己的环境创建 DqnAgent。但是我收到以下错误

AttributeError: 'tuple' object has no attribute 'rank' In call to configurable 'DqnAgent' (<class 'tf_agents.agents.dqn.dqn_agent.DqnAgent'>)

下面是我的环境的代码myEnv

import numpy as np
import tensorflow as tf

from tf_agents.environments import py_environment
from tf_agents.specs import array_spec
from tf_agents.utils import common
from tf_agents.networks import q_network
from tf_agents.agents.dqn import dqn_agent

class myEnv(py_environment.PyEnvironment):
    def __init__(self):
        self._action_spec = array_spec.BoundedArraySpec(
            shape=(), dtype=np.int32, minimum=0, maximum=2, name='action')
        self._observation_spec = array_spec.BoundedArraySpec(
            shape=(2,), dtype=np.int32, minimum=0, name='observation')
        self._state = np.zeros(shape=(2,),dtype=np.int32)


    def action_spec(self):
        return self._action_spec


    def observation_spec(self):
        return self._observation_spec

    def _reset(self):
        self._state = np.zeros(shape=(8,),dtype=np.int32)
        self._c_reward = 0
        return ts.restart(self._state)


    def _step(self, action):
      self._state[action] = self._state[action] + 1
      return ts.transition(self._state, reward=0.0, discount=1.0)

我可以创建 myEnv 的实例而不会出现任何错误。

train_env = myEnv()

但是如果我尝试使用以下代码创建 agent

q_net = q_network.QNetwork(
    train_env.observation_spec(),
    train_env.action_spec(),
    fc_layer_params=(10,))

optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)
train_step_counter = tf.Variable(0)

agent = dqn_agent.DqnAgent(
    train_env.time_step_spec(),
    train_env.action_spec(),
    q_network=q_net,
    optimizer=optimizer,
    td_errors_loss_fn=common.element_wise_squared_loss,
    train_step_counter=train_step_counter)

我收到了这个 Traceback

---------------------------------------------------------------------------
AttributeError                            Traceback (most recent call last)
<ipython-input-3-a730748f947f> in <module>()
     15     optimizer=optimizer,
     16     td_errors_loss_fn=common.element_wise_squared_loss,
---> 17     train_step_counter=train_step_counter)

4 frames
/usr/local/lib/python3.7/dist-packages/tf_agents/agents/dqn/dqn_agent.py in _check_action_spec(self, action_spec)
    293 
    294     # TODO(oars): Get DQN working with more than one dim in the actions.
--> 295     if len(flat_action_spec) > 1 or flat_action_spec[0].shape.rank > 0:
    296       raise ValueError(
    297           'Only scalar actions are supported now, but action spec is: {}'

AttributeError: 'tuple' object has no attribute 'rank'
  In call to configurable 'DqnAgent' (<class 'tf_agents.agents.dqn.dqn_agent.DqnAgent'>)

如何解决这个错误？

Answer 1

DqnAgent 期望 TFPyEnvironment 但您将环境实施为 PyEnvironment。

要修复此错误，您应该在创建代理之前将环境转换为 TensorFlow 实现。

你可以这样做：

from tf_agents.environments import tf_py_environment

train_env = myEnv()

train_env_tf = tf_py_environment.TFPyEnvironment(train_envt)

q_net = q_network.QNetwork(
    train_env_tf.observation_spec(),
    train_env_tf.action_spec(),
    fc_layer_params=(10,))

optimizer = tf.keras.optimizers.Adam(learning_rate=1e-3)
train_step_counter = tf.Variable(0)

agent = dqn_agent.DqnAgent(
    train_env_tf.time_step_spec(),
    train_env_tf.action_spec(),
    q_network=q_net,
    optimizer=optimizer,
    td_errors_loss_fn=common.element_wise_squared_loss,
    train_step_counter=train_step_counter)

How to fix AttributeError: 'tuple' object has no attribute 'rank' creating a DqnAgent with Tensorflow?

How to fix AttributeError: 'tuple' object has no attribute 'rank' creating a DqnAgent with Tensorflow?

tensorflow2.0