I had a similar issue in a specific part of my model. I do not declare any custom class that needs its own autograd.
I was having trouble finding the source of the error, so I just started blindly printing my tensors until a print statement of a certain variable (that was printed once before and was not changed since then) gave me this error.
def forward(self, x: dict, node_type: torch.Tensor):
      print(node_type)
      X = torch.zeros(node_type.size(0), self.out_features, device=self.device)
      # X = torch.empty(node_type.size(0), self.out_features, device=self.device)
      print(node_type)
      print('X', X, X.size())

      for nt in node_type.unique().tolist():
         print(nt)
         print(node_type)
         node_mask = (node_type == nt)
         print(node_mask)
         print(node_type)
         enc = self.apply_net(x[nt], nt)

         print('X', X) # this is where I got the error
         print('enc', enc)

         X[node_mask] = enc

      print(node_type)
      return X

This the output:
tensor([0, 0, 0], device='cuda:0')
tensor([0, 0, 0], device='cuda:0')
tensor([0, 0, 0], device='cuda:0')
tensor([0, 0, 0], device='cuda:0')
X tensor([[0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0.],
        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
         0., 0., 0., 0., 0., 0., 0., 0.]], device='cuda:0') torch.Size([3, 128])
0
tensor([0, 0, 0], device='cuda:0')
tensor([True, True, True], device='cuda:0')
tensor([0, 0, 0], device='cuda:0')
X Something happened, saving checkpoint...
Training step: 1
	Saving checkpoint...
THCudaCheck FAIL file=/pytorch/torch/csrc/generic/serialization.cpp line=31 error=700 : an illegal memory access was encountered
Traceback (most recent call last):
  File "/home/user/Code/my_agent/my_agent/training/run.py", line 514, in <module>
    runner.run()
  File "/home/user/Code/my_agent/my_agent/training/run.py", line 238, in run
    q_vals = self.model.policy_net(current_state)
  File "/home/user/.anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/user/Code/my_agent/my_agent/nn/nets.py", line 156, in forward
    x = self.encoding_layer(input_by_class, batch_node_types)
  File "/home/user/.anaconda3/lib/python3.8/site-packages/torch/nn/modules/module.py", line 722, in _call_impl
    result = self.forward(*input, **kwargs)
  File "/home/user/Code/my_agent/my_agent/nn/modules/encoding.py", line 101, in forward
    print('X', X)
  File "/home/user/.anaconda3/lib/python3.8/site-packages/torch/tensor.py", line 153, in __repr__
    return torch._tensor_str._str(self)
  File "/home/user/.anaconda3/lib/python3.8/site-packages/torch/_tensor_str.py", line 371, in _str
    return _str_intern(self)
  File "/home/user/.anaconda3/lib/python3.8/site-packages/torch/_tensor_str.py", line 351, in _str_intern
    tensor_str = _tensor_str(self, indent)
  File "/home/user/.anaconda3/lib/python3.8/site-packages/torch/_tensor_str.py", line 241, in _tensor_str
    formatter = _Formatter(get_summarized_data(self) if summarize else self)
  File "/home/user/.anaconda3/lib/python3.8/site-packages/torch/_tensor_str.py", line 89, in __init__
    nonzero_finite_vals = torch.masked_select(tensor_view, torch.isfinite(tensor_view) & tensor_view.ne(0))
RuntimeError: copy_if failed to synchronize: cudaErrorIllegalAddress: an illegal memory access was encountered

As you can see, I printed variable X before, did not get a CUDA error, but then printed it once again and got an error. My problem started ocurring when I updated PyTorch to 1.6.0. Downgrading back to 1.5.0 solved it for me.
My System is Manjaro Linux, Python 3.8.3 on Anaconda, CUDA 10.1, NVIDIA drivers 435.21, GeForce GTX 1070.