Sorry for the late reply.

  
      github.com
  
  
    pytorch/pytorch/blob/59b139dabd8689163ec71fbcc987c88f3cc7e5ae/torch/legacy/nn/Module.py#L240 107



    # 7. fix up the parameter tensors to point at the flattened parameters
    for param, meta in zip(parameters, parameterMeta):
        param.set_(flatParameters.storage(),
                   meta['storage_offset'],
                   meta['size'],
                   meta['stride'])


    return flatParameters


def flattenParameters(self):
    _params = self.parameters()
    if _params is None:
        return
    parameters, gradParameters = _params
    p, g = self._flatten(parameters), self._flatten(gradParameters)


    assert p.nelement() == g.nelement()
    if parameters:
        for param, grad in zip(parameters, gradParameters):
            assert param.storage_offset() == grad.storage_offset()



  
  
    
    
  
  


This is the code for flatten paramters. It makes the memory the parameters and its grads occupied contiguous.
However, eventually I solved this issue by deleting all the intermediate variables, such as h_n, c_n etc (for LSTM).