I’ve tried hard coding cpu call throughout the process flow and still get the same error (even when the same tensor is moved to the cpu). Only a single tensor is involved in this process:
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
<ipython-input-32-f94813b879a4> in <module>
     11     start_time = time.time()
     12 
---> 13     train_loss = train(model, train_iterator, optimizer, criterion, CLIP)
     14     valid_loss = evaluate(model, valid_iterator, criterion)
     15 

<ipython-input-28-f807d4f617eb> in train(model, iterator, optimizer, criterion, clip)
     11         optimizer.optimizer.zero_grad()
     12 
---> 13         output = model(src, trg[:,:-1])
     14         #output = [batch size, trg sent len - 1, output dim]
     15         #trg = [batch size, trg sent len]

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

<ipython-input-19-15e3ff9eb197> in forward(self, src, trg)
     32         src_mask, trg_mask = self.make_masks(src, trg)
     33 
---> 34         src_inject = self.embedder.get_embeddings(src,src.size()[1])
     35         src_inject = torch.stack(src_inject).cuda()
     36         enc_src = self.encoder(src, src_inject, src_mask)

~/project/Embedder.py in get_embeddings(self, primary_sequences, d_embed, alphabet, pack_only, compile_only)
     28             return fully_packed_sequences
     29         fully_packed_sequences.to('cpu') #the input is moved to the CPU
---> 30         compiled_sequences = self.encoder.embedding(fully_packed_sequences)
     31         if compile_only == True:
     32             return compiled_sequences

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/project/src/models/embedding.py in forward(self, x)
    119 
    120         if self.lm:
--> 121             h = self.embed(x)
    122         else:
    123             if type(x) is PackedSequence:

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/project/src/models/embedding.py in forward(self, x)
     23     def forward(self, x):
     24         packed = type(x) is PackedSequence
---> 25         h_lm = self.lm.encode(x)
     26 
     27         # embed and unpack if packed

~/project/src/models/sequence.py in encode(self, x)
    182     def encode(self, x):
    183         x.to('cpu') #again
--> 184         z_fwd,z_rvs = self.embed_and_split(x, pad=True)
    185         h_fwd_layers,h_rvs_layers = self.transform(z_fwd, z_rvs)
    186 

~/project/src/models/sequence.py in embed_and_split(self, x, pad)
    150         # now, encode x as distributed vectors
    151         x.to('cpu') # a third time
--> 152         z = self.embed(x)
    153 
    154         # to pass to transform, we discard the last element for z_fwd and the first element for z_rvs

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/sparse.py in forward(self, input)
    116         return F.embedding(
    117             input, self.weight, self.padding_idx, self.max_norm,
--> 118             self.norm_type, self.scale_grad_by_freq, self.sparse)
    119 
    120     def extra_repr(self):

/opt/conda/lib/python3.6/site-packages/torch/nn/functional.py in embedding(input, weight, padding_idx, max_norm, norm_type, scale_grad_by_freq, sparse)
   1452         # remove once script supports set_grad_enabled
   1453         _no_grad_embedding_renorm_(weight, input, max_norm, norm_type)
-> 1454     return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
   1455 
   1456 

RuntimeError: Expected object of backend CPU but got backend CUDA for argument #3 'index'