What about this.
import torch

from torch import Tensor
from torch.nn.parameter import Parameter
from torch.nn import Module
from torch.autograd import Variable

class Blah(Module):
    def __init__(self, dim):
        super(Blah, self).__init__()
        self.register_buffer('s', torch.rand(dim))
        self.t = Parameter(torch.rand(dim))

blah = Blah(10)

blah.zero_grad()

print('s', type(blah.s))
print('t', type(blah.t.data))

if torch.cuda.is_available():
    blah.cuda()
    print('s', type(blah.s))
    print('t', type(blah.t.data))

Which outputs the following.
/home/atcold/anaconda3/bin/python /home/atcold/Work/buffer.py
s <class 'torch.FloatTensor'>
t <class 'torch.FloatTensor'>
s <class 'torch.cuda.FloatTensor'>
t <class 'torch.cuda.FloatTensor'>

Process finished with exit code 0

Here is a ref. 8 to the documentation of the register_buffer() method.