Мое состояние представляет собой список времени обучения для каждого клиента за последнее h число раундов (сохранение исторической информации за последние h раундов). Таким образом, в основном измерение состояния равно num_clients*h.
Мои действия — это интенсивность обучения для каждого клиента. Таким образом, измерение действия равно num_clients.
Мои сети актеров-критиков задаются следующим образом:
Код: Выделить всё
num_clients = 5
h = 4
state_dimension = num_clients*h
action_dimension = num_clients
class PolicyNetwork(nn.Module):
def __init__(self, state_dimension, action_dimension):
super(PolicyNetwork, self).__init__()
self.input_layer = nn.Linear(state_dimension, 128)
self.output_layer = nn.Linear(128, action_dimension)
def forward(self, x):
x = F.relu(self.input_layer(x))
action_probs = F.softmax(self.output_layer(x), dim=1)
return action_probs
class StateValueNetwork(nn.Module):
def __init__(self, state_dimension):
super(StateValueNetwork, self).__init__()
self.input_layer = nn.Linear(state_dimension, 128)
self.output_layer = nn.Linear(128, 1)
def forward(self, x):
x = F.relu(self.input_layer(x))
state_value = self.output_layer(x)
return state_value
Итак, моя награда =- (Максимальная разница во времени обучения)
Обучение актера и критика осуществляется с помощью буфер повтора.
Изначально для каждого эпизода состояние пустое и для каждого раунда оно обновляется. По истечении часа раундов актер-критик начинает прогнозировать политику.
Для реализации я использовал 5 клиентов.
Основная логика и обучение актера-критика таковы: дано следующим образом -
Код: Выделить всё
#Defining parameters
num_episodes = 3
num_rounds=15
total_training_intensity=15
BATCH_SIZE = 10
cpu_cycles = torch.rand(5) * (30 - 10) + 10
size_data_sample = 4
cpu_freq = torch.rand(5) * (1.0 - 0.1) + 0.1
#Define replay buffer
class ReplayBuffer:
def __init__(self, capacity):
self.buffer = deque(maxlen=capacity)
def push(self, state, action, reward, next_state):
self.buffer.append(Experience(state, action, reward, next_state))
def sample(self, batch_size):
return random.sample(self.buffer, batch_size)
#MAIN LOGIC
for episode in range(num_episodes):
episode_reward = 0
for rnd in range(num_rounds):
if rnd==0:
#Initially set state to all zero
state = deque()
flattened_state = [item for sublist in state for item in sublist]
state_tensor = torch.tensor(flattened_state, dtype=torch.float32)
#Normalize state
state_tensor = (state_tensor - state_tensor.mean()) / (state_tensor.std() + 1e-10)
state_tensor = state_tensor.view(-1)
if(len(state)!=h):
#Initiate same training intensity for clients for the first 3 rounds
intensities = [round(float(total_training_intensity)/float(num_clients)) for _ in range(num_clients)]
else:
#Get training intensities from policy network for remaining rounds
actions,_ = select_action(policy_network, state_tensor)
intensities = [round(i * total_training_intensity) if round(i * total_training_intensity) != 0 else 1 for i in actions]
print('Local training intensities: ',intensities)
client_training_time = []
for i in range(num_clients):
client_training_time.append((intensities[i]*cpu_cycles[i]*size_data_sample)/(cpu_freq[i]*1e3)) #Get client training time
#Define new state
if(len(state)==h):
state.popleft()
state.append(client_training_time)
#Normalize new state to add in the replay buffer
flattened_new_state = [item for sublist in state for item in sublist]
new_state_tensor = torch.tensor(flattened_new_state, dtype=torch.float32)
new_state_tensor = (new_state_tensor - new_state_tensor.mean()) / (new_state_tensor.std() + 1e-10)
new_state_tensor = new_state_tensor.view(-1)
#Get reward
normal_list = [float(tensor_item) for tensor_item in client_training_time]
reward = -(max(normal_list) - min(normal_list))
episode_reward += reward
# Store transition tuple into experience replay buffer
if(rnd>=h):
#Add to buffer
replay_buffer.push(state_tensor.tolist(),actions,reward,new_state_tensor.tolist())
# Train on a batch of experiences
if len(replay_buffer) >= BATCH_SIZE:
for i in range(3):
experiences = replay_buffer.sample(BATCH_SIZE)
batch = Experience(*zip(*experiences))
for i in range(3):
state_batch = torch.FloatTensor(batch.state).to(DEVICE)
action_batch = torch.LongTensor(batch.action).unsqueeze(1).to(DEVICE)
reward_batch = torch.FloatTensor(batch.reward).unsqueeze(1).to(DEVICE)
next_state_batch = torch.FloatTensor(batch.next_state).to(DEVICE)
# Compute value loss
current_values = stateval_network(state_batch)
next_values = stateval_network(next_state_batch)
expected_values = reward_batch + DISCOUNT_FACTOR * next_values
value_loss = F.mse_loss(current_values, expected_values.detach())
# Compute policy loss
advantage = expected_values.detach() - current_values.detach()
action_probs = policy_network(state_batch)
policy_loss = -(torch.log(action_probs) * advantage).mean()
# Update networks
stateval_optimizer.zero_grad()
value_loss.backward()
stateval_optimizer.step()
policy_optimizer.zero_grad()
policy_loss.backward()
policy_optimizer.step()
Подробнее здесь: https://stackoverflow.com/questions/787 ... nvironment
