I move a pretrained embedding model onto the GPU in an effort to make my model faster and I keep getting this device assert trigger, im running the code with CUDA_LAUNCH_BLOCKING= 1 and this is the (rather unhelpful) stacktrace I received, any ideas?
RuntimeError                              Traceback (most recent call last)
<ipython-input-32-f94813b879a4> in <module>
     11     start_time = time.time()
     12 
---> 13     train_loss = train(model, train_iterator, optimizer, criterion, CLIP)
     14     valid_loss = evaluate(model, valid_iterator, criterion)
     15 

<ipython-input-28-f807d4f617eb> in train(model, iterator, optimizer, criterion, clip)
     11         optimizer.optimizer.zero_grad()
     12 
---> 13         output = model(src, trg[:,:-1])
     14         #output = [batch size, trg sent len - 1, output dim]
     15         #trg = [batch size, trg sent len]

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

<ipython-input-19-ee8c0f2d26d1> in forward(self, src, trg)
     31         #trg = [batch size, trg sent len]
     32         src_mask, trg_mask = self.make_masks(src, trg)
---> 33         src_inject = self.embedder.get_embeddings(src,src.size()[1])
     34         src_inject = torch.stack(src_inject).cuda()
     35         enc_src = self.encoder(src, src_inject, src_mask)

~/project/Embedder.py in get_embeddings(self, primary_sequences, d_embed, alphabet, pack_only, compile_only)
     27         if pack_only == True:
     28             return fully_packed_sequences
---> 29         compiled_sequences = self.encoder.embedding(fully_packed_sequences)
     30         if compile_only == True:
     31             return compiled_sequences

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/project/src/models/embedding.py in forward(self, x)
    119 
    120         if self.lm:
--> 121             h = self.embed(x)
    122         else:
    123             if type(x) is PackedSequence:

/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    487             result = self._slow_forward(*input, **kwargs)
    488         else:
--> 489             result = self.forward(*input, **kwargs)
    490         for hook in self._forward_hooks.values():
    491             hook_result = hook(self, input, result)

~/project/src/models/embedding.py in forward(self, x)
     23     def forward(self, x):
     24         packed = type(x) is PackedSequence
---> 25         h_lm = self.lm.encode(x)
     26 
     27         # embed and unpack if packed

~/project/src/models/sequence.py in encode(self, x)
    180 
    181     def encode(self, x):
--> 182         z_fwd,z_rvs = self.embed_and_split(x, pad=True)
    183         h_fwd_layers,h_rvs_layers = self.transform(z_fwd, z_rvs)
    184 

~/project/src/models/sequence.py in embed_and_split(self, x, pad)
    156         if packed:
    157             lengths = [s-1 for s in batch_sizes]
--> 158             z_fwd = pack_padded_sequence(z_fwd, lengths, batch_first=True)
    159             z_rvs = pack_padded_sequence(z_rvs, lengths, batch_first=True)
    160         # reverse z_rvs

/opt/conda/lib/python3.6/site-packages/torch/nn/utils/rnn.py in pack_padded_sequence(input, lengths, batch_first)
    146                       category=torch.jit.TracerWarning, stacklevel=2)
    147     lengths = torch.as_tensor(lengths, dtype=torch.int64)
--> 148     return PackedSequence(torch._C._VariableFunctions._pack_padded_sequence(input, lengths, batch_first))
    149 
    150 

RuntimeError: CUDA error: device-side assert triggered