I am having a big memory issue where when I save and append the output of my model it increases the memory usage by a lot on every pass. I have no idea if this is intended or if this is a bug or something stupid I am doing. I modeled it off this example: https://github.com/pytorch/examples/blob/master/reinforcement_learning/actor_critic.py 2
Here is where I am appending the values to backprop on after the episode:

  
      github.com
  
  
    shriuken/starcraft2ai/blob/master/src/QAgent.py#L66 1

    if USE_CUDA and torch.cuda.is_available():
        probs, state_value, x, y = self.forward(Variable(state).cuda())
    else:
        probs, state_value, x, y = self.forward(Variable(state))
    m = Categorical(probs)
    x_probs = Categorical(x)
    y_probs = Categorical(y)
    x_coord = x_probs.sample()
    y_coord = y_probs.sample()
    action = m.sample()
    self.saved_actions.append([m.log_prob(action), state_value, x_probs.log_prob(x_coord), y_probs.log_prob(y_coord)])
    return action.data[0], x_coord, y_coord


def forward(self, x):
    x = convolution_layer(x, layer * num_layers)
    x = self.mp(residual_layer(x, 256, 78))
    x = F.relu(self.lin1(x.view(x.size(0), -1)))
    action_scores = self.action_head(x)
    state_values = self.value_head(x)
    x_coord = self.x(x)
    y_coord = self.y(x)