import torch.nn as nn
import torch

class MyModule(nn.Module):
    def __init__(self):
        super(MyModule, self).__init__()
        self.fc1 = nn.Linear(1, 1)

    def forward(self, x):
        n = torch.range(0,5)
        print(n.type())
        #x = x.mm(n)
        x = self.fc1(x)
        return x

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(10, 1)
        self.module1 = MyModule()

    def forward(self, x):
        x = self.fc1(x)
        x = self.module1(x)
        return x

model = Net()
model = model.to('cuda:0')
model(torch.zeros(1, 10).to('cuda:0'))
print(model.module1.fc1.weight.type())
print(model.fc1.weight.type())

Output:

torch.FloatTensor
torch.cuda.FloatTensor
torch.cuda.FloatTensor

The new tensor I define in my custom layer is not on cuda. As a result, I need to pass in device so that I can move them to device.
I would expect the default to be that new tensor should also be on cuda once your model is moved to cuda.