Tensors saved for the backward pass from forward ARE actually freed automatically during the corresponding backward!
If you are curious you can check in the autograd engine’s evaluate function, we call fn.release_variables if keep_graph (i.e., the retain_graph parameter) is set to False.
  

      github.com
  

  
    pytorch/pytorch/blob/master/torch/csrc/autograd/engine.cpp#L769



      
      
          at::ThreadLocalStateGuard tls_guard(graph_task->thread_locals_);
          
          
// Switches to a function's CUDA stream (if applicable) before calling it
          const auto opt_parent_stream = (*func).stream(c10::DeviceType::CUDA);
          c10::OptionalStreamGuard parent_stream_guard{opt_parent_stream};
          
          
auto outputs = call_function(graph_task, func, inputs);
          
          
auto& fn = *func;
          if (!graph_task->keep_graph_) {
            fn.release_variables();
          }
          
          
int num_outputs = outputs.size();
          if (num_outputs == 0) { // Note: doesn't acquire the mutex
            // Records leaf stream (if applicable)
            // See note "Streaming backwards"
            if (opt_parent_stream) {
              std::lock_guard<std::mutex> lock(graph_task->mutex_);
              graph_task->leaf_streams.emplace(*opt_parent_stream);
            }