I added PyTorch Lightning implementation based on this Colab Notebook 5.
I also tested by removing all PyTorch Lightning implementation and used only PyTorch -  memory leak was still present. Although on my dataset it managed to get through more than 85% of the first epoch.
Train Epoch: 1 [43200/50639 (85%)]	Loss: 2.600413
Traceback (most recent call last):
  File "entrypoint.py", line 169, in <module>
    experiment=experiment
  File "entrypoint.py", line 148, in main
    train(model, device, train_loader, criterion, optimizer, scheduler, epoch, iter_meter, experiment)
  File "entrypoint.py", line 46, in train
    loss.backward()
  File "/home/project/.venv/lib/python3.6/site-packages/comet_ml/monkey_patching.py", line 292, in wrapper
    return_value = original(*args, **kwargs)
  File "/home/project/.venv/lib/python3.6/site-packages/torch/tensor.py", line 198, in backward
    torch.autograd.backward(self, gradient, retain_graph, create_graph)
  File "/home/project/.venv/lib/python3.6/site-packages/torch/autograd/__init__.py", line 100, in backward
    allow_unreachable=True)  # allow_unreachable flag
RuntimeError: CUDA out of memory. Tried to allocate 1.03 GiB (GPU 0; 14.76 GiB total capacity; 11.60 GiB already allocated; 709.44 MiB free; 13.26 GiB reserved in total by PyTorch) (malloc at /pytorch/c10/cuda/CUDACachingAllocator.cpp:289)
frame #0: c10::Error::Error(c10::SourceLocation, std::string const&) + 0x46 (0x7f4139f78536 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libc10.so)
frame #1: <unknown function> + 0x1cf1e (0x7f413a1c1f1e in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libc10_cuda.so)
frame #2: <unknown function> + 0x1df9e (0x7f413a1c2f9e in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libc10_cuda.so)
frame #3: at::native::empty_cuda(c10::ArrayRef<long>, c10::TensorOptions const&, c10::optional<c10::MemoryFormat>) + 0x135 (0x7f413cd56535 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cuda.so)
frame #4: <unknown function> + 0xf7a66b (0x7f413b34e66b in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cuda.so)
frame #5: <unknown function> + 0xfc3f57 (0x7f413b397f57 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cuda.so)
frame #6: <unknown function> + 0x1075389 (0x7f41778d2389 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #7: <unknown function> + 0x10756c7 (0x7f41778d26c7 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #8: <unknown function> + 0xf30f5e (0x7f413b304f5e in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cuda.so)
frame #9: <unknown function> + 0xf3ccc7 (0x7f413b310cc7 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cuda.so)
frame #10: at::native::_cudnn_rnn_backward(at::Tensor const&, c10::ArrayRef<at::Tensor>, long, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, at::Tensor const&, long, long, long, bool, double, bool, bool, c10::ArrayRef<long>, at::Tensor const&, at::Tensor const&, std::array<bool, 4ul>) + 0x1a9 (0x7f413b313319 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cuda.so)
frame #11: <unknown function> + 0xfc20ad (0x7f413b3960ad in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cuda.so)
frame #12: <unknown function> + 0xfc3843 (0x7f413b397843 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cuda.so)
frame #13: <unknown function> + 0x2b08450 (0x7f4179365450 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #14: <unknown function> + 0x2b7b8a3 (0x7f41793d88a3 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #15: torch::autograd::generated::CudnnRnnBackward::apply(std::vector<at::Tensor, std::allocator<at::Tensor> >&&) + 0x708 (0x7f4179119d28 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #16: <unknown function> + 0x2d89c05 (0x7f41795e6c05 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #17: torch::autograd::Engine::evaluate_function(std::shared_ptr<torch::autograd::GraphTask>&, torch::autograd::Node*, torch::autograd::InputBuffer&) + 0x16f3 (0x7f41795e3f03 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #18: torch::autograd::Engine::thread_main(std::shared_ptr<torch::autograd::GraphTask> const&, bool) + 0x3d2 (0x7f41795e4ce2 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #19: torch::autograd::Engine::thread_init(int) + 0x39 (0x7f41795dd359 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_cpu.so)
frame #20: torch::autograd::python::PythonEngine::thread_init(int) + 0x38 (0x7f4185d1c4d8 in /home/project/.venv/lib/python3.6/site-packages/torch/lib/libtorch_python.so)
frame #21: <unknown function> + 0xbd6df (0x7f4199fd66df in /usr/lib/x86_64-linux-gnu/libstdc++.so.6)
frame #22: <unknown function> + 0x76db (0x7f41a08656db in /lib/x86_64-linux-gnu/libpthread.so.0)
frame #23: clone + 0x3f (0x7f41a0b9e88f in /lib/x86_64-linux-gnu/libc.so.6)