Traceback (most recent call last):
File “run_techqa3.py”, line 629, in 
main()
File “run_techqa3.py”, line 623, in main
model = train(args, train_dataset, model, optimizer, tokenizer, model_evaluator)
File “run_techqa3.py”, line 222, in train
outputs = model(**inputs)
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 532, in call
result = self.forward(*input, **kwargs)
File “/dccstor/sahban21_2905/Chunk_Extraction/TechQA-Base/techqa-master/model_techqa3.py”, line 88, in forward
outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 532, in call
result = self.forward(*input, **kwargs)
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/transformers/modeling_bert.py”, line 790, in forward
encoder_attention_mask=encoder_extended_attention_mask,
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 532, in call
result = self.forward(*input, **kwargs)
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/transformers/modeling_bert.py”, line 407, in forward
hidden_states, attention_mask, head_mask[i], encoder_hidden_states, encoder_attention_mask
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 532, in call
result = self.forward(*input, **kwargs)
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/transformers/modeling_bert.py”, line 368, in forward
self_attention_outputs = self.attention(hidden_states, attention_mask, head_mask)
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 532, in call
result = self.forward(*input, **kwargs)
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/transformers/modeling_bert.py”, line 314, in forward
hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 532, in call
result = self.forward(*input, **kwargs)
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/transformers/modeling_bert.py”, line 251, in forward
context_layer = torch.matmul(attention_probs, value_layer)
RuntimeError: CUDA error: device-side assert triggered