The nans are not generated in by softmax. The model itself is generating the nan because of the exploding gradients due to the learning rate.
I tried to check my network weights. It has generated  nan
In [29]: for param in agent.model1.parameters():
    print(param.data)
    
tensor([[    nan,     nan,     nan,     nan,     nan,     nan,     nan,     nan,
             nan],
        [    nan,     nan,     nan,     nan,     nan,     nan,     nan,     nan,
             nan],
        [    nan,     nan,     nan,     nan,     nan,     nan,     nan,     nan,
             nan],
        [    nan,     nan,     nan,     nan,     nan,     nan,     nan,     nan,
             nan],
        [-0.7366, -0.1832, -0.1841,  0.3141,  0.0334, -0.0575, -0.0015,  0.0069,
         -0.1040],
        [    nan,     nan,     nan,     nan,     nan,     nan,     nan,     nan,
             nan],
        [    nan,     nan,     nan,     nan,     nan,     nan,     nan,     nan,
             nan],
        [    nan,     nan,     nan,     nan,     nan,     nan,     nan,     nan,
             nan],
        [    nan,     nan,     nan,     nan,     nan,     nan,     nan,     nan,
             nan],
        [    nan,     nan,     nan,     nan,     nan,     nan,     nan,     nan,
             nan]])
tensor([   nan,    nan,    nan,    nan, 0.1520,    nan,    nan,    nan,    nan,
           nan])
tensor([[nan, nan, nan, nan, nan, nan, nan, nan, nan, nan],
        [nan, nan, nan, nan, nan, nan, nan, nan, nan, nan]])
tensor([nan, nan])
tensor([[ 4.5539e+00, -4.2161e+00,  2.1670e-01, -2.9069e+00, -4.2264e+00,
         -4.5930e+00, -3.9898e+00,  4.3225e+00,  4.2810e+00],
        [ 4.4970e+00, -4.4960e+00,  3.8166e+00, -2.9493e+00, -4.6124e+00,
         -4.6803e+00, -3.9606e+00,  4.0320e+00,  4.2299e+00],
        [-3.0815e-01, -8.5769e-02,  3.1897e-01,  3.0792e-01, -7.2797e-02,
          1.2266e-02,  2.3653e-01,  2.4632e-01,  3.8586e-02],
        [-8.0415e-02,  9.2836e-02,  3.7588e-01,  3.3804e-01,  1.5777e-02,
          7.4958e-02, -6.0354e-02,  8.1592e-02, -3.8448e-01],
        [-1.4028e-01,  1.0577e-01,  3.7370e-01,  2.5323e-01,  1.0640e-01,
         -1.5946e-01,  1.5165e-01, -1.5983e-04,  1.1991e-01],
        [ 3.5704e-02,  8.9192e-02, -4.2397e-02, -1.8162e-01,  2.7302e-02,
         -8.2681e-02,  2.4023e-01,  1.8748e-01, -3.6148e-01],
        [ 4.4627e+00, -4.1383e+00,  1.0344e+00, -2.9011e+00, -4.5565e+00,
         -4.4978e+00, -3.5755e+00,  4.2879e+00,  4.3042e+00],
        [ 4.0489e+00, -3.4591e+00,  1.2957e+00, -2.6597e+00, -3.5383e+00,
         -3.4929e+00, -3.4063e+00,  3.7202e+00,  4.0741e+00],
        [-5.6180e-02,  3.9072e-02, -5.6076e-02,  3.0225e-01, -9.5747e-02,
          1.5115e-01,  1.0766e-02,  2.7571e-01, -3.0291e-01],
        [ 4.2704e+00, -4.1625e+00,  9.1064e-01, -3.1105e+00, -4.2028e+00,
         -4.3451e+00, -3.8066e+00,  4.3735e+00,  4.4310e+00]])
tensor([ 4.3880,  3.9937, -0.0354, -0.0842, -0.1234, -0.2485,  4.0131,  4.0434,
        -0.3865,  4.1969])
tensor([[ 4.3842,  4.5874, -0.0854, -0.1604, -0.2546, -0.0998,  4.6371,  4.2951,
         -0.0997,  4.7081]])
tensor([3.3452])