With CUDA_LAUNCH_BLOCKING =1
Traceback (most recent call last):
File “run_techqa2.py”, line 629, in 
main()
File “run_techqa2.py”, line 623, in main
model = train(args, train_dataset, model, optimizer, tokenizer, model_evaluator)
File “run_techqa2.py”, line 222, in train
outputs = model(**inputs)
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 532, in call
result = self.forward(*input, **kwargs)
File “/dccstor/sahban21_2905/Chunk_Extraction/TechQA-Base/techqa-master/model_techqa1.py”, line 88, in forward
outputs = self.bert(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 532, in call
result = self.forward(*input, **kwargs)
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/transformers/modeling_bert.py”, line 783, in forward
input_ids=input_ids, position_ids=position_ids, token_type_ids=token_type_ids, inputs_embeds=inputs_embeds
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 532, in call
result = self.forward(*input, **kwargs)
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/transformers/modeling_bert.py”, line 174, in forward
position_embeddings = self.position_embeddings(position_ids)
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/torch/nn/modules/module.py”, line 532, in call
result = self.forward(*input, **kwargs)
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/torch/nn/modules/sparse.py”, line 114, in forward
self.norm_type, self.scale_grad_by_freq, self.sparse)
File “/dccstor/sahban21_2905/miniconda3/envs/chunk_ext/lib/python3.7/site-packages/torch/nn/functional.py”, line 1484, in embedding
return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)
RuntimeError: CUDA error: device-side assert triggered