Thanks for your reply.
This is the code structure I used. There are two GPUs but I indeed specify to use one of them.
Note: There are embedding layers in the model.
class cnnTextNetwork(Configurable):
  def __init__(self, option, model, *args, **cargs):
    # Other Operations

    if self.use_gpu:
      self.model = model(self.args).cuda()
    else:
      self.model = model(self.args)
    return

  def train(self):
    # Other Operations
    while True:
      for batch in self.train_minibatch():
        self.model.train()
        feature, target = batch['text'], batch['label']
        if self.use_gpu:
          feature = Variable(torch.from_numpy(feature).cuda())
        else:
          feature = Variable(torch.from_numpy(feature))
        optimizer.zero_grad() 
        logit = self.model(feature)
        # Other Operations