Ok, I put together a quick toy network.
Its a simple model, with the forward function fully expanded for the discussion.
#include <torch/torch.h>

struct ToyImpl : torch::nn::Module {
	ToyImpl(int64_t input_size, int64_t output_size) : 
		linear1(register_module("linear1", torch::nn::Linear(input_size,64))),
		linear2(register_module("linear2", torch::nn::Linear(64, output_size))) {}

	torch::Tensor forward(torch::Tensor input){
		torch::Tensor p = linear1(input);
		torch::Tensor q = torch::relu(p);
		torch::Tensor r = linear2(q);
		return r;
	}
	torch::nn::Linear linear1, linear2;
};

TORCH_MODULE(Toy);

const int64_t input_size  = 16;
const int64_t output_size = 2;
const int64_t N  = 256; // batch size

int main()
{
	Toy toyNet(input_size,output_size);
	toyNet->to(torch::kCUDA);
	// input tensor & desired output
	torch::Tensor input        = torch::rand({N, input_size }).to(torch::kCUDA);
	torch::Tensor output_truth = torch::rand({N, output_size}).to(torch::kCUDA);

	float learning_rate = 1e-4;
 	 torch::optim::SGD optimizer(
      toyNet->parameters(), torch::optim::SGDOptions(learning_rate));

 	// a single epoch
	optimizer.zero_grad();
	auto output_pred = toyNet->forward(input);
	auto loss = torch::mse_loss(output_pred, output_truth.detach());
	loss.backward();
	optimizer.step();

	return 0;
}

So, during toyNet->forward(input) I want to track when tensors p, q, r are accessed and when their computation is finished. It would also be useful to know which of them are saved for backwards.
Then, during loss.backwards() I want to again track when the saved tensors (amoungst p, q ,r) are accessed and when their computation is complete.