Thanks, guys, I create the optimizer before I put the network in GPU and this problem solved.
But, I may have two new problems now. The train speed of GPU is way slower than CPU.  When the network runs on CPU, it takes around 19 seconds per 2000 steps. when it runs on GPU, it takes around 31 seconds per 2000 steps.
And the loss of the model is very low from the beginning and it just not decreasing. Did I do something wrong?
Here is the network configuration
class Net(nn.Module):
    
    def __init__(self):
        super(Net, self).__init__()        
        # self.conv1 = nn.Conv2d(1, 6, 5) # for single-channel images
        self.conv1 = nn.Conv2d(3, 6, 5) # for RGB images
        # self.conv3 = nn.Conv2d(6, 16, 5)        
        self.conv3_0 = nn.Conv2d(3, 1, 5)
        self.conv3_1 = nn.Conv2d(4, 1, 5)
        self.conv3_2 = nn.Conv2d(6, 1, 5)        
        self.conv5 = nn.Conv2d(16, 120, 5)        
        self.fc6 = nn.Linear(120, 84)
        self.fc7 = nn.Linear(84, 10)
    
    def forward(self, x):
        x = self.conv1(x)
        x = F.max_pool2d(x, 2)
        x_chunks = torch.chunk(x, 6, dim=1)        
        chunk_batchs = []
        chunk_batchs.append(torch.cat(x_chunks[0: 3], dim=1))
        chunk_batchs.append(torch.cat(x_chunks[1: 4], dim=1))
        chunk_batchs.append(torch.cat(x_chunks[2: 5], dim=1))
        chunk_batchs.append(torch.cat(x_chunks[3: ], dim=1))
        chunk_batchs.append(torch.cat((x_chunks[0], x_chunks[4], x_chunks[5]), dim=1))
        chunk_batchs.append(torch.cat((x_chunks[0], x_chunks[1], x_chunks[5]), dim=1))        
        chunk_batchs.append(torch.cat(x_chunks[0: 4], dim=1))
        chunk_batchs.append(torch.cat(x_chunks[1: 5], dim=1))
        chunk_batchs.append(torch.cat(x_chunks[2: ], dim=1))
        chunk_batchs.append(torch.cat((x_chunks[0], x_chunks[3], x_chunks[4], x_chunks[5]), dim=1))
        chunk_batchs.append(torch.cat((x_chunks[0], x_chunks[1], x_chunks[4], x_chunks[5]), dim=1))
        chunk_batchs.append(torch.cat((x_chunks[0], x_chunks[1], x_chunks[2], x_chunks[5]), dim=1))
        chunk_batchs.append(torch.cat((x_chunks[0], x_chunks[1], x_chunks[3], x_chunks[4]), dim=1))
        chunk_batchs.append(torch.cat((x_chunks[1], x_chunks[2], x_chunks[4], x_chunks[5]), dim=1))
        chunk_batchs.append(torch.cat((x_chunks[1], x_chunks[2], x_chunks[3], x_chunks[5]), dim=1))

        out = []        
        for t_batch in chunk_batchs[0: 6]:
            out.append(self.conv3_0(t_batch))
        for f_batch in chunk_batchs[6: ]:
            out.append(self.conv3_1(f_batch))        
        out.append(self.conv3_2(x))
        x = torch.cat(out, dim=1)
      
        x = F.max_pool2d(x, 2)        
        x = self.conv5(x)        
        x = x.view(-1, 120)
        x = F.tanh(self.fc6(x))        
        x = self.fc7(x)

        return x 
  
net = Net()

And Here is my train code
for epoch in range(2):
    running_loss = 0.0
    s_time = time.clock()
    for i, data in enumerate(trainloader, 0):
        inputs, labels = data
        labels = labels.view(bs, 1)
        onehot_label = torch.FloatTensor(bs, 10)
        onehot_label.zero_()
        onehot_label.scatter_(1, labels, 1)
        
        # will be remove on cpu version
        inputs = inputs.to(device)
        onehot_label = onehot_label.to(device)
        
        optimizer.zero_grad()
        outputs = net(inputs)
        loss = MSE(outputs, onehot_label)
        loss.backward()
        optimizer.step()        
        running_loss += loss.item()
        if i % 2000 == 1999:
            r_time = time.clock() - s_time
            print('[%d, %5d] loss: %.3f time: %.3f'%
                  (epoch + 1, i + 1, running_loss / 2000, r_time))
            s_time = time.clock()
            running_loss = 0.0

print('Finished Training')

Here is train output on CPU
[1,  2000] loss: 0.088 time: 20.982
[1,  4000] loss: 0.085 time: 20.645
[1,  6000] loss: 0.083 time: 20.652
[1,  8000] loss: 0.081 time: 19.481
[1, 10000] loss: 0.080 time: 19.409
[1, 12000] loss: 0.080 time: 19.292
[2,  2000] loss: 0.079 time: 19.401
[2,  4000] loss: 0.079 time: 19.402
[2,  6000] loss: 0.078 time: 19.422
[2,  8000] loss: 0.078 time: 19.493
[2, 10000] loss: 0.077 time: 20.072
[2, 12000] loss: 0.077 time: 19.498
Finished Training

Here is GPU train output
[1,  2000] loss: 0.076 time: 30.982
[1,  4000] loss: 0.077 time: 36.717
[1,  6000] loss: 0.077 time: 31.032
[1,  8000] loss: 0.077 time: 31.602
[1, 10000] loss: 0.076 time: 48.607
[1, 12000] loss: 0.076 time: 38.124
[2,  2000] loss: 0.075 time: 30.924
[2,  4000] loss: 0.075 time: 34.561
[2,  6000] loss: 0.075 time: 30.950
[2,  8000] loss: 0.075 time: 30.780
[2, 10000] loss: 0.074 time: 30.815
[2, 12000] loss: 0.074 time: 30.876
Finished Training

Here is GPU status when training