Hey there,
I get the following runntime error:
Traceback (most recent call last):
  File "ppo_witches_multi2.py", line 420, in <module>
    learn_single(ppo1, update_timestep, eps_decay, env)
  File "ppo_witches_multi2.py", line 269, in learn_single
    ppo.my_update(memory)
  File "ppo_witches_multi2.py", line 174, in my_update
    loss.mean().backward()
  File "/home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/tensor.py", line 198, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
  File "/home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/autograd/__init__.py", line 100, in backward
    allow_unreachable=True)  # allow_unreachable flag
RuntimeError: expected dtype Double but got dtype Float (validate_dtype at /pytorch/aten/src/ATen/native/TensorIterator.cpp:143)
frame #0: c10::Error::Error(c10::SourceLocation, std::string const&) + 0x46 (0x7fa5570a6536 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libc10.so)
frame #1: at::TensorIterator::compute_types() + 0xce3 (0x7fa5947c6183 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #2: at::TensorIterator::build() + 0x44 (0x7fa5947c8b64 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #3: at::native::mse_loss_backward_out(at::Tensor&, at::Tensor const&, at::Tensor const&, at::Tensor const&, long) + 0x193 (0x7fa594616b93 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #4: <unknown function> + 0x10b7db7 (0x7fa594a42db7 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #5: at::native::mse_loss_backward(at::Tensor const&, at::Tensor const&, at::Tensor const&, long) + 0x172 (0x7fa59461f2d2 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #6: <unknown function> + 0x109e76f (0x7fa594a2976f in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #7: <unknown function> + 0x10c3c76 (0x7fa594a4ec76 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #8: <unknown function> + 0x2a9eceb (0x7fa596429ceb in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #9: <unknown function> + 0x10c3c76 (0x7fa594a4ec76 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #10: torch::autograd::generated::MseLossBackward::apply(std::vector<at::Tensor, std::allocator<at::Tensor> >&&) + 0x1f7 (0x7fa596231787 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #11: <unknown function> + 0x2d89c05 (0x7fa596714c05 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #12: torch::autograd::Engine::evaluate_function(std::shared_ptr<torch::autograd::GraphTask>&, torch::autograd::Node*, torch::autograd::InputBuffer&) + 0x16f3 (0x7fa596711f03 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #13: torch::autograd::Engine::thread_main(std::shared_ptr<torch::autograd::GraphTask> const&, bool) + 0x3d2 (0x7fa596712ce2 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #14: torch::autograd::Engine::thread_init(int) + 0x39 (0x7fa59670b359 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #15: torch::autograd::python::PythonEngine::thread_init(int) + 0x38 (0x7fa5a2e4a4d8 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_python.so)
frame #16: <unknown function> + 0xbd6df (0x7fa5a3f1b6df in /usr/lib/x86_64-linux-gnu/libstdc++.so.6)
frame #17: <unknown function> + 0x76db (0x7fa5a64b06db in /lib/x86_64-linux-gnu/libpthread.so.0)
frame #18: clone + 0x3f (0x7fa5a67e988f in /lib/x86_64-linux-gnu/libc.so.6)

After using .float() this error occurs:
Traceback (most recent call last):
  File "ppo_witches_multi2.py", line 424, in <module>
    learn_single(ppo1, update_timestep, eps_decay, env)
  File "ppo_witches_multi2.py", line 273, in learn_single
    ppo.my_update(memory)
  File "ppo_witches_multi2.py", line 178, in my_update
    loss.mean().backward()
  File "/home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/tensor.py", line 198, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
  File "/home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/autograd/__init__.py", line 100, in backward
    allow_unreachable=True)  # allow_unreachable flag
RuntimeError: [enforce fail at CPUAllocator.cpp:64] . DefaultCPUAllocator: can't allocate memory: you tried to allocate 517246976 bytes. Error code 12 (Cannot allocate memory)
frame #0: c10::ThrowEnforceNotMet(char const*, int, char const*, std::string const&, void const*) + 0x47 (0x7f4d434294f7 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libc10.so)
frame #1: c10::alloc_cpu(unsigned long) + 0x1f1 (0x7f4d43411201 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libc10.so)
frame #2: <unknown function> + 0x17f4a (0x7f4d43412f4a in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libc10.so)
frame #3: THStorage_resize + 0x3b (0x7f4d80ed867b in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #4: at::native::resize_(at::Tensor&, c10::ArrayRef<long>, c10::optional<c10::MemoryFormat>) + 0x4ab (0x7f4d80acad3b in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #5: <unknown function> + 0xdbe5d3 (0x7f4d80acb5d3 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #6: at::native::mm_cpu_out(at::Tensor&, at::Tensor const&, at::Tensor const&) + 0x27b (0x7f4d80887ceb in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #7: at::native::mm_cpu(at::Tensor const&, at::Tensor const&) + 0x177 (0x7f4d808892f7 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #8: <unknown function> + 0x107fdb8 (0x7f4d80d8cdb8 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #9: <unknown function> + 0x10c3ec0 (0x7f4d80dd0ec0 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #10: <unknown function> + 0x2c9b6fe (0x7f4d829a86fe in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #11: <unknown function> + 0x10c3ec0 (0x7f4d80dd0ec0 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #12: at::Tensor c10::Dispatcher::callUnboxed<at::Tensor, at::Tensor const&, at::Tensor const&>(c10::OperatorHandle const&, at::Tensor const&, at::Tensor const&) const + 0xb3 (0x7f4d8ef08ca3 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_python.so)
frame #13: <unknown function> + 0x28ac337 (0x7f4d825b9337 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #14: torch::autograd::generated::AddmmBackward::apply(std::vector<at::Tensor, std::allocator<at::Tensor> >&&) + 0x134 (0x7f4d825f4004 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #15: <unknown function> + 0x2d89c05 (0x7f4d82a96c05 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #16: torch::autograd::Engine::evaluate_function(std::shared_ptr<torch::autograd::GraphTask>&, torch::autograd::Node*, torch::autograd::InputBuffer&) + 0x16f3 (0x7f4d82a93f03 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #17: torch::autograd::Engine::thread_main(std::shared_ptr<torch::autograd::GraphTask> const&, bool) + 0x3d2 (0x7f4d82a94ce2 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #18: torch::autograd::Engine::thread_init(int) + 0x39 (0x7f4d82a8d359 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #19: torch::autograd::python::PythonEngine::thread_init(int) + 0x38 (0x7f4d8f1cc4d8 in /home/mlamprecht/Documents/witches_env/lib/python3.6/site-packages/torch/lib/libtorch_python.so)
frame #20: <unknown function> + 0xbd6df (0x7f4d9029d6df in /usr/lib/x86_64-linux-gnu/libstdc++.so.6)
frame #21: <unknown function> + 0x76db (0x7f4d928326db in /lib/x86_64-linux-gnu/libpthread.so.0)
frame #22: clone + 0x3f (0x7f4d92b6b88f in /lib/x86_64-linux-gnu/libc.so.6)


For me this error seems to be related with this one 11. I could solve it somehow using .float():
        # Optimize policy for K epochs:
        for _ in range(self.K_epochs):
            # Evaluating old actions and values :
            logprobs, state_values, dist_entropy = self.policy.evaluate(old_states, old_actions)
            advantages = rewards - state_values.detach()

            rewards    = rewards.float()
            advantages = advantages.float()
            loss       =  self.calculate_total_loss(state_values, logprobs, old_logprobs, advantages, rewards, dist_entropy)


            # take gradient step
            self.optimizer.zero_grad()
            loss.mean().backward()
            self.optimizer.step()

However using this method, the error just pops up one hour later (in training). I do not know why but I think this is because I use numpy rewards and somehow they are not float anymore…
Could it be that there is a rounding problem? / What else did I miss to .float ?
Another strange thing is that this problem did not occur for 4 weeks but now it is there.
On another pc I just got killed as output with no error message at all.
Any ideas on debugging would be great