Hi all
I am currently facing RuntimeError: CUDA error: device-side assert triggered when I am trying to sample. So, the code is,
softmax_tmpl_distribution = self.softmax(tmpl_distribution)
tmpl_token_id = softmax_tmpl_distribution.multinomial(num_samples=self.agent_tmpl_n)

tmpl_token_id_random_size = torch.ones([self.batch_size, self.agent_tmpl_n])
tmpl_token_id_random = F.softmax(tmpl_token_id_random_size, dim=-1).multinomial(num_samples=1).to(device)
tmpl_token_id = tmpl_token_id.gather(-1, tmpl_token_id_random)
# 1st error
# tmpl_token_id_random = torch.randint(self.agent_tmpl_n, (self.batch_size, 1))
# tmpl_token_id_random = tmpl_token_id_random.to(device) # error
# tmpl_token_id = tmpl_token_id.gather(-1, tmpl_token_id_random)
# 2nd error
# tmpl_token_id_random = torch.randint(self.agent_tmpl_n, (self.batch_size, 1))
# tmpl_token_id = tmpl_token_id.cpu() # error
# tmpl_token_id = tmpl_token_id.gather(-1, tmpl_token_id_random)
# tmpl_token_id = tmpl_token_id.to(device)
# 3rd error
# tmpl_token_id_random = torch.randint(self.agent_tmpl_n, (self.batch_size,))
# tmpl_token_id_random = tmpl_token_id_random.unsqueeze(-1).to(device) # error
# tmpl_token_id = tmpl_token_id.gather(-1, tmpl_token_id_random)
# errors are: /pytorch/aten/src/ATen/native/cuda/MultinomialKernel.cu:87: int at::native::<unnamed>::bina
# rySearchForMultinomial(scalar_t *, scalar_t *, int, scalar_t) [with scalar_t = float]: block: [2,0,0],
# thread: [0,0,0] Assertion `cumdist[size - 1] > static_cast<scalar_t>(0)` failed.

where tmpl_distribution is just output from network. I apply softmax to it and sample it through multinomial to get tmpl_token_id. This works. However, when I try to uniformly sample from tmpl_token_id using all the above method, including random.sample (I did not include this above), it just throws me an error message.
/pytorch/aten/src/ATen/native/cuda/MultinomialKernel.cu:87: int at::native::<unnamed>::binarySearchForMultinomial(scalar_t *, scalar_t *, int, scalar_t) [with scalar_t = float]: block: [6,0,0], thread: [0,0,0] Assertion `cumdist[size - 1] > static_cast<scalar_t>(0)` failed.
Traceback (most recent call last):
  File "run.py", line 98, in <module>
    trainer.train(params['steps'])
  File "/projects/da33/dkRyu/main/phd-main-github/trainer/trainer.py", line 224, in train
    obs_reps, scores, graph_state_reps, graph_mask_tt
  File "/home/dryu0002/da33/dkRyu/conda-envs/main/lib/python3.7/site-packages/torch/nn/modules/module.py", line 532, in __call__
    result = self.forward(*input, **kwargs)
  File "/projects/da33/dkRyu/main/phd-main-github/models/models.py", line 340, in forward
    tmpl_token_id = self.random_selecting_tmpl(tmpl_distribution)
  File "/projects/da33/dkRyu/main/phd-main-github/models/models.py", line 511, in random_selecting_tmpl
    tmpl_token_id_random = F.softmax(tmpl_token_id_random_size, dim=-1).multinomial(num_samples=1).to(device)
RuntimeError: CUDA error: device-side assert triggered

Thank you in advance