When I do a backprop, don’t I need that computation graph? I got around this by taking the float values from the Variables but then my network never actually learned 
EDIT: I just tested this by changing the pytorch RL example and saving the m.log_prob(action).data. It ended up not learning on the backprop and was stuck at an average length of 20-21. This leads me to believe I need the computation graph for backprop

  
      github.com
  
  
    pytorch/examples/blob/master/reinforcement_learning/actor_critic.py#L61 1



model = Policy()
optimizer = optim.Adam(model.parameters(), lr=3e-2)




def select_action(state):
state = torch.from_numpy(state).float()
probs, state_value = model(Variable(state))
m = Categorical(probs)
action = m.sample()
model.saved_actions.append(SavedAction(m.log_prob(action), state_value))
return action.data[0]




def finish_episode():
R = 0
saved_actions = model.saved_actions
policy_losses = []
value_losses = []
rewards = []
for r in model.rewards[::-1]: