Код: Выделить всё
Traceback (most recent call last):
File "/home/perry/Documents/Programming/decathlon_training_simulation_tf/train_network.py", line 308, in
).run(env_train_py.reset())
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/perry/Documents/Programming/decathlon_training_simulation_tf/.venv/lib/python3.11/site-packages/tf_agents/drivers/py_driver.py", line 119, in run
action_step = self.policy.action(time_step, policy_state)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/perry/Documents/Programming/decathlon_training_simulation_tf/.venv/lib/python3.11/site-packages/tf_agents/policies/py_policy.py", line 169, in action
return self._action(time_step, policy_state)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/perry/Documents/Programming/decathlon_training_simulation_tf/.venv/lib/python3.11/site-packages/tf_agents/policies/py_tf_eager_policy.py", line 107, in _action
policy_step = self._policy_action_fn(time_step, policy_state)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/perry/Documents/Programming/decathlon_training_simulation_tf/.venv/lib/python3.11/site-packages/tensorflow/python/util/traceback_utils.py", line 153, in error_handler
raise e.with_traceback(filtered_tb) from None
File "/home/perry/Documents/Programming/decathlon_training_simulation_tf/.venv/lib/python3.11/site-packages/tf_agents/policies/tf_policy.py", line 317, in action
nest_utils.assert_same_structure(
File "/home/perry/Documents/Programming/decathlon_training_simulation_tf/.venv/lib/python3.11/site-packages/tf_agents/utils/nest_utils.py", line 131, in assert_same_structure
raise exception(
ValueError: time_step and time_step_spec structures do not match:
TimeStep(
{'step_type': .,
'reward': .,
'discount': .,
'observation': {'dev_strength': {'r': ., 'best': .},
'dev_speed': {'r': ., 'best': .},
'dev_jumping': {'r': ., 'best': .},
'dev_endurance': {'r': ., 'best': .},
'dev_100_tech': {'r': ., 'best': .},
'dev_lj_tech': {'r': ., 'best': .},
'dev_sp_tech': {'r': ., 'best': .},
'dev_hj_tech': {'r': ., 'best': .},
'dev_400_tech': {'r': ., 'best': .},
'dev_110h_tech': {'r': ., 'best': .},
'dev_dt_tech': {'r': ., 'best': .},
'dev_pv_tech': {'r': ., 'best': .},
'dev_jt_tech': {'r': ., 'best': .},
'dev_1500_tech': {'r': ., 'best': .},
'dev_100': {'start': ., 'best': ., 'shift': ., 'k': .},
'dev_lj': {'start': ., 'best': ., 'shift': ., 'k': .},
'dev_sp': {'start': ., 'best': ., 'shift': ., 'k': .},
'dev_hj': {'start': ., 'best': ., 'shift': ., 'k': .},
'dev_400': {'start': ., 'best': ., 'shift': ., 'k': .},
'dev_110h': {'start': ., 'best': ., 'shift': ., 'k': .},
'dev_dt': {'start': ., 'best': ., 'shift': ., 'k': .},
'dev_pv': {'start': ., 'best': ., 'shift': ., 'k': .},
'dev_jt': {'start': ., 'best': ., 'shift': ., 'k': .},
'dev_1500': {'start': ., 'best': ., 'shift': ., 'k': .},
'100_i': .,
'lj_i': .,
'sp_i': .,
'hj_i': .,
'400_i': .,
'110h_i': .,
'dt_i': .,
'pv_i': .,
'jt_i': .,
'1500_i': .,
'100_o': .,
'lj_o': .,
'sp_o': .,
'hj_o': .,
'400_o': .,
'110h_o': .,
'dt_o': .,
'pv_o': .,
'jt_o': .,
'1500_o': .,
'inv_speed': .,
'inv_strength': .,
'inv_jumping': .,
'inv_endurance': .,
'inv_100_tech': .,
'inv_lj_tech': .,
'inv_sp_tech': .,
'inv_hj_tech': .,
'inv_400_tech': .,
'inv_110h_tech': .,
'inv_dt_tech': .,
'inv_pv_tech': .,
'inv_jt_tech': .,
'inv_1500_tech': .,
't': .}})
vs.
TimeStep(
{'step_type': ., 'reward': ., 'discount': ., 'observation': .})
Values:
TimeStep(
{'step_type': ,
'reward': ,
'discount': ,
'observation': {'dev_strength': {'r': ,
'best': },
'dev_speed': {'r': ,
'best': },
'dev_jumping': {'r': ,
'best': },
'dev_endurance': {'r': ,
'best': },
'dev_100_tech': {'r': ,
'best': },
'dev_lj_tech': {'r': ,
'best': },
'dev_sp_tech': {'r': ,
'best': },
'dev_hj_tech': {'r': ,
'best': },
'dev_400_tech': {'r': ,
'best': },
'dev_110h_tech': {'r': ,
'best': },
'dev_dt_tech': {'r': ,
'best': },
'dev_pv_tech': {'r': ,
'best': },
'dev_jt_tech': {'r': ,
'best': },
'dev_1500_tech': {'r': ,
'best': },
'dev_100': {'start': ,
'best': ,
'shift': ,
'k': },
'dev_lj': {'start': ,
'best': ,
'shift': ,
'k': },
'dev_sp': {'start': ,
'best': ,
'shift': ,
'k': },
'dev_hj': {'start': ,
'best': ,
'shift': ,
'k': },
'dev_400': {'start': ,
'best': ,
'shift': ,
'k': },
'dev_110h': {'start': ,
'best': ,
'shift': ,
'k': },
'dev_dt': {'start': ,
'best': ,
'shift': ,
'k': },
'dev_pv': {'start': ,
'best': ,
'shift': ,
'k': },
'dev_jt': {'start': ,
'best': ,
'shift': ,
'k': },
'dev_1500': {'start': ,
'best': ,
'shift': ,
'k': },
'100_i': ,
'lj_i': ,
'sp_i': ,
'hj_i': ,
'400_i': ,
'110h_i': ,
'dt_i': ,
'pv_i': ,
'jt_i': ,
'1500_i': ,
'100_o': ,
'lj_o': ,
'sp_o': ,
'hj_o': ,
'400_o': ,
'110h_o': ,
'dt_o': ,
'pv_o': ,
'jt_o': ,
'1500_o': ,
'inv_speed': ,
'inv_strength': ,
'inv_jumping': ,
'inv_endurance': ,
'inv_100_tech': ,
'inv_lj_tech': ,
'inv_sp_tech': ,
'inv_hj_tech': ,
'inv_400_tech': ,
'inv_110h_tech': ,
'inv_dt_tech': ,
'inv_pv_tech': ,
'inv_jt_tech': ,
'inv_1500_tech': ,
't': }})
vs.
TimeStep(
{'step_type': TensorSpec(shape=(), dtype=tf.int32, name='step_type'),
'reward': TensorSpec(shape=(), dtype=tf.float32, name='reward'),
'discount': BoundedTensorSpec(shape=(), dtype=tf.float32, name='discount', minimum=array(0., dtype=float32), maximum=array(1., dtype=float32)),
'observation': BoundedTensorSpec(shape=(), dtype=tf.float32, name='observation_spec', minimum=array(0., dtype=float32), maximum=array(1000., dtype=float32))}).
Код: Выделить всё
# MDP specs.
self._action_spec = array_spec.BoundedArraySpec(
shape=(), dtype=np.int32,
minimum=0, maximum=len(self.latent_abilities) + len(self.events) - 1,
name='action'
)
self._observation_spec = array_spec.BoundedArraySpec(
shape=(), dtype=np.float32,
minimum=0, maximum=1000,
name='observation_spec'
)
# self._observation_spec = array_spec.BoundedArraySpec(
# shape=(, len(self.latent_abilities) + len(self.events)*4 + len(self.latent_abilities) + len(self.events) + 1), dtype=np.float32,
# minimum=0, maximum=1000,
# name='observation_spec',
# )
# self._observation_spec = {
# # Latent ability development.
# **{
# f'dev_{ability}': {
# 'r': array_spec.BoundedArraySpec(
# shape=(), dtype=np.float32,
# minimum=0, maximum=1000,
# ),
# 'best': array_spec.BoundedArraySpec(
# shape=(), dtype=np.float32,
# minimum=0, maximum=1000,
# )
# }
# for ability in self.latent_abilities
# },
# # Event technical development.
# **{
# f'dev_{event}_tech': {
# 'r': array_spec.BoundedArraySpec(
# shape=(), dtype=np.float32,
# minimum=0, maximum=1000,
# ),
# 'best': array_spec.BoundedArraySpec(
# shape=(), dtype=np.float32,
# minimum=0, maximum=1000,
# )
# }
# for event in self.events
# },
# # Event development.
# **{
# f'dev_{event}': {
# 'r': array_spec.BoundedArraySpec(
# shape=(), dtype=np.float32,
# minimum=0, maximum=1000,
# ),
# 'best': array_spec.BoundedArraySpec(
# shape=(), dtype=np.float32,
# minimum=0, maximum=1000,
# )
# }
# for event in self.events
# },
# # Initial marks.
# **{
# f'{event}_i': array_spec.BoundedArraySpec(
# shape=(), dtype=np.float32,
# minimum=0, maximum=1000,
# )
# for event in self.events
# },
# # Optimal marks.
# **{
# f'{event}_o': array_spec.BoundedArraySpec(
# shape=(), dtype=np.float32,
# minimum=0, maximum=1000,
# )
# for event in self.events
# },
# # Investment in latent abilities.
# **{
# f'inv_{ability}': array_spec.BoundedArraySpec(
# shape=(), dtype=np.int32,
# minimum=0, maximum=1000,
# )
# for ability in self.latent_abilities
# },
# # Investment in technical abilities.
# **{
# f'inv_{event}_tech': array_spec.BoundedArraySpec(
# shape=(), dtype=np.int32,
# minimum=0, maximum=1000,
# )
# for event in self.events
# },
# # Time.
# 't': array_spec.BoundedArraySpec(
# shape=(),
# dtype=np.float32,
# minimum=0,
# maximum=self.time_bounds[1],
# )
# }
Код: Выделить всё
# Training environment.
from training_env import CombinedEventEnvironment
# Utilities.
from util import (
preprocess_marks,
observation_from_state,
construct_state,
athlete_profile,
score_performances,
calc_score_combined_event,
simulated_competition,
)
# Reverb.
import reverb
# TensorFlow.
import tensorflow as tf
# TensorFlow Agents.
from tf_agents.agents.dqn import dqn_agent
from tf_agents.drivers import py_driver
from tf_agents.environments import tf_py_environment
from tf_agents.eval import metric_utils
from tf_agents.networks import sequential
from tf_agents.policies import (
py_tf_eager_policy,
random_tf_policy,
)
from tf_agents.replay_buffers import (
reverb_replay_buffer,
reverb_utils,
)
from tf_agents.trajectories import trajectory
from tf_agents.specs import tensor_spec
from tf_agents.utils import common
# Data processing.
import numpy as np
import pandas as pd
# General utilities.
import argparse
import json
# Arguments.
parser = argparse.ArgumentParser()
parser.add_argument('--events', '-e', default='assets/events_dec.json')
parser.add_argument(
'--event_improvement_directions',
'-eid',
default='assets/events_improvement_directions_dec.json'
)
parser.add_argument(
'--latent_abilities',
'-l',
default='assets/latent_abilities_dec.json'
)
parser.add_argument('--factor_loadings', '-fl', default='assets/factor_loadings_dec.json')
parser.add_argument('--coef_scoring', '-cs', default='assets/coef_scoring_dec.json')
parser.add_argument(
'--coef_dev_latent',
'-cdl',
default='assets/coef_dev_latent_dec.json'
)
parser.add_argument('--coef_dev_tech', '-cdt', default='assets/coef_dev_tech_dec.json')
parser.add_argument('--coef_dev_event', '-cde', default='assets/coef_dev_event_dec.json')
parser.add_argument('--marks', '-m', default='data/DecData.csv')
parser.add_argument('--profiles', '-prof', default=None)
args = parser.parse_args()
# Load assets.
with open(args.events) as _: events = json.load(_)
with open(args.event_improvement_directions) as _:
event_improvement_directions = json.load(_)
with open(args.latent_abilities) as _: latent_abilities = json.load(_)
with open(args.factor_loadings) as _: factor_loadings = json.load(_)
with open(args.coef_scoring) as _: coef_scoring = json.load(_)
with open(args.coef_dev_latent) as _: coef_dev_latent = json.load(_)
with open(args.coef_dev_tech) as _: coef_dev_tech = json.load(_)
with open(args.coef_dev_event) as _: coef_dev_event = json.load(_)
# Learning parameters.
num_iterations = 20_000
initial_collect_steps = 100
collect_steps_per_iteration = 1
replay_buffer_max_length = 1000 # 100_000
batch_size = 64
learning_rate = 1e-3
log_interval = 200
num_eval_episodes = 10
eval_interval = 1000
# Load data.
df = pd.read_csv(args.marks)
df_ath = pd.read_csv(args.profiles) if args.profiles is not None else None
df = preprocess_marks(df, events=events)
# Build environment.
env = CombinedEventEnvironment(
# Events.
events=events,
event_improvement_directions=event_improvement_directions,
# Factor structure.
latent_abilities=latent_abilities,
factor_loadings=factor_loadings,
# Scoring coefficients.
coef_scoring=coef_scoring,
# Development curve parameters.
coef_dev_latent=coef_dev_latent,
coef_dev_tech=coef_dev_tech,
coef_dev_event=coef_dev_event,
# Time bounds.
time_bounds=(5, 100),
# Discount factor.
gamma_=0.99,
# Data.
df=df,
df_ath=df_ath,
save_athlete_profiles=True,
)
env_train_py = CombinedEventEnvironment(
# Events.
events=events,
event_improvement_directions=event_improvement_directions,
# Factor structure.
latent_abilities=latent_abilities,
factor_loadings=factor_loadings,
# Scoring coefficients.
coef_scoring=coef_scoring,
# Development curve parameters.
coef_dev_latent=coef_dev_latent,
coef_dev_tech=coef_dev_tech,
coef_dev_event=coef_dev_event,
# Time bounds.
time_bounds=(5, 100),
# Discount factor.
gamma_=0.99,
# Data.
df=df,
df_ath=df_ath,
save_athlete_profiles=True,
)
env_eval_py = CombinedEventEnvironment(
# Events.
events=events,
event_improvement_directions=event_improvement_directions,
# Factor structure.
latent_abilities=latent_abilities,
factor_loadings=factor_loadings,
# Scoring coefficients.
coef_scoring=coef_scoring,
# Development curve parameters.
coef_dev_latent=coef_dev_latent,
coef_dev_tech=coef_dev_tech,
coef_dev_event=coef_dev_event,
# Time bounds.
time_bounds=(5, 100),
# Discount factor.
gamma_=0.99,
# Data.
df=df,
df_ath=df_ath,
save_athlete_profiles=True,
)
print(env_train_py.reset())
print(env_train_py.step(action=2))
train_env = tf_py_environment.TFPyEnvironment(env_train_py)
eval_env = tf_py_environment.TFPyEnvironment(env_eval_py)
# Build network.
fc_layer_params = (100, 50)
action_tensor_spec = tensor_spec.from_spec(env_train_py.action_spec())
num_actions = action_tensor_spec.maximum - action_tensor_spec.minimum + 1
print("Specs:")
print(action_tensor_spec)
print(env_train_py.time_step_spec())
def dense_layer(num_units):
return tf.keras.layers.Dense(
num_units,
activation=tf.keras.activations.relu,
kernel_initializer=tf.keras.initializers.VarianceScaling(
scale=2.0, mode='fan_in', distribution='truncated_normal',
)
)
dense_layers = [dense_layer(num_units) for num_units in fc_layer_params]
q_values_layer = tf.keras.layers.Dense(
num_actions,
activation=None,
kernel_initializer=tf.keras.initializers.RandomUniform(
minval=-0.03, maxval=0.03,
),
bias_initializer=tf.keras.initializers.Constant(-0.2)
)
q_net = sequential.Sequential(
layers=[
*dense_layers,
q_values_layer
],
input_spec=tf.TensorSpec(shape=(1,), dtype=tf.int32)
)
# Optimizer.
optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)
train_step_counter = tf.Variable(0)
# Instantiate DQN agent.
agent = dqn_agent.DqnAgent(
train_env.time_step_spec(),
train_env.action_spec(),
q_network=q_net,
optimizer=optimizer,
td_errors_loss_fn=common.element_wise_squared_loss,
train_step_counter=train_step_counter,
)
agent.initialize()
# Set up policies for data collection and learning.
policy_collect = agent.collect_policy
policy_eval = agent.policy
policy_random = random_tf_policy.RandomTFPolicy(
train_env.time_step_spec(),
train_env.action_spec()
)
time_step = train_env.reset()
print("Random Policy Action:", policy_random.action(time_step))
# Average return.
def compute_average_return(env, policy, n_episodes=10):
"""
Computes the average return across all episodes.
"""
def episode():
time_step = env.reset()
episode_return = 0.0
while not time_step.is_last():
action_step = policy.action(time_step)
time_step = env.step(action_step.action)
episode_return += time_step.reward
return episode_return
total_return = sum([
episode()
for i in range(n_episodes)
])
return total_return/n_episodes
# Test.
print("Avg. return test:", compute_average_return(eval_env, policy_random, num_eval_episodes))
# Replay buffer.
table_name = 'uniform_table'
replay_buffer_signature = tensor_spec.from_spec(
agent.collect_data_spec
)
replay_buffer_signature = tensor_spec.add_outer_dim(
replay_buffer_signature
)
table = reverb.Table(
table_name,
max_size=replay_buffer_max_length,
sampler=reverb.selectors.Uniform(),
remover=reverb.selectors.Fifo(),
rate_limiter=reverb.rate_limiters.MinSize(1),
signature=replay_buffer_signature
)
# Create a server to collect data in our table.
reverb_server = reverb.Server([table])
# Create a replay buffer that lives on our server.
replay_buffer = reverb_replay_buffer.ReverbReplayBuffer(
agent.collect_data_spec,
table_name=table_name,
sequence_length=2,
local_server=reverb_server,
)
# Create an observer that drops data into that replay buffer.
rb_observer = reverb_utils.ReverbAddTrajectoryObserver(
replay_buffer.py_client,
table_name,
sequence_length=2,
)
# Collect data for training by exploring using the random policy.
py_driver.PyDriver(
env,
py_tf_eager_policy.PyTFEagerPolicy(
policy_random, use_tf_function=True
),
[rb_observer],
max_steps=initial_collect_steps
).run(env_train_py.reset())
# Out of curiosity:
print(iter(replay_buffer.as_dataset()).next())
Буду очень признателен за любую помощь!
