Very wired, the error in the toy program disappeared but now this happens if I run the main NN code
(with or without cudnn)
THCudaCheck FAIL file=/opt/conda/conda-bld/pytorch_1512387374934/work/torch/lib/THC/generic/THCTensorCopy.c line=20 error=6 : the launch timed out and was terminated
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
~/x/d/crnn.pytorch2/main.py in <module>()
    341 try:
    342     engine.train(opt.display_interval, opt.val_interval,
--> 343                  opt.save_interval, opt.epochs)
    344 except KeyboardInterrupt:
    345     engine.save()

~/x/d/crnn.pytorch2/engine.py in train(self, display_interval, val_interval, save_interval, epochs)
    123                 self.model.train()
    124 
--> 125                 cost, preds, skip = self.exec_batch(self.model, batch)
    126                 if skip: continue
    127 

~/x/d/crnn.pytorch2/main.py in exec_batch(model, batch)
    211 
    212 def exec_batch(model, batch):
--> 213     utils.loadData(image, batch.images)
    214     t, l = converter.encode(batch.labels_text)
    215     utils.loadData(text, t)

~/x/d/crnn.pytorch2/utils.py in loadData(v, data)
    148 
    149 def loadData(v, data):
--> 150     v.data.resize_(data.size()).copy_(data)
    151 
    152 

RuntimeError: cuda runtime error (6) : the launch timed out and was terminated at /opt/conda/conda-bld/pytorch_1512387374934/work/torch/lib/THC/generic/THCTensorCopy.c:20


Sometimes it looks like this
THCudaCheck FAIL file=/opt/conda/conda-bld/pytorch_1512387374934/work/torch/lib/THC/generic/THCStorage.cu line=58 error=6 : the launch timed out and was terminated
---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
~/x/d/crnn.pytorch2/main.py in <module>()
    116 
    117 #gpu_text = gpu_text.cuda()
--> 118 image = image.cuda()
    119 #pretrain = pretrain.cuda()
    120 ctc_criterion = ctc_criterion.cuda()

~/x/miniconda3/lib/python3.6/site-packages/torch/autograd/variable.py in cuda(self, device, async)
    296 
    297     def cuda(self, device=None, async=False):
--> 298         return CudaTransfer.apply(self, device, async)
    299 
    300     def cpu(self):

~/x/miniconda3/lib/python3.6/site-packages/torch/autograd/_functions/tensor.py in forward(ctx, i, device, async)
    199             return i.cuda(device, async=async)
    200         else:
--> 201             return i.cuda(async=async)
    202 
    203     @staticmethod

~/x/miniconda3/lib/python3.6/site-packages/torch/_utils.py in _cuda(self, device, async)
     67         else:
     68             new_type = getattr(torch.cuda, self.__class__.__name__)
---> 69             return new_type(self.size()).copy_(self, async)
     70 
     71 

~/x/miniconda3/lib/python3.6/site-packages/torch/cuda/__init__.py in _lazy_new(cls, *args, **kwargs)
    359     # We need this method only for lazy init, so we can remove it
    360     del _CudaBase.__new__
--> 361     return super(_CudaBase, cls).__new__(cls, *args, **kwargs)
    362 
    363 

RuntimeError: cuda runtime error (6) : the launch timed out and was terminated at /opt/conda/conda-bld/pytorch_1512387374934/work/torch/lib/THC/generic/THCStorage.cu:58


UPD:
Okay, now, with cudnn enabled, it sometimes runs one or two iterations and crashes with this error:
        [    0][  10/1502] | Loss: 621.920557 |                      : system of recording temperatures .
---------------------------------------------------------------------------
CuDNNError                                Traceback (most recent call last)
~/x/d/crnn.pytorch2/main.py in <module>()
    341 try:                                                                                                                                                                                              [100/696]
    342     engine.train(opt.display_interval, opt.val_interval,
--> 343                  opt.save_interval, opt.epochs)
    344 except KeyboardInterrupt:
    345     engine.save()

~/x/d/crnn.pytorch2/engine.py in train(self, display_interval, val_interval, save_interval, epochs)
    123                 self.model.train()
    124 
--> 125                 cost, preds, skip = self.exec_batch(self.model, batch)
    126                 if skip: continue
    127 

~/x/d/crnn.pytorch2/main.py in exec_batch(model, batch)
    215     utils.loadData(text, t)
    216     utils.loadData(length, l)
--> 217     preds, cost, skip = criterion(batch)
    218     return cost, preds, skip
    219 

~/x/d/crnn.pytorch2/main.py in plain_crit(batch)
    224 
    225 def plain_crit(batch):
--> 226     return _ctc_crit(model(image))
    227 
    228 def dpret_crit(batch):

~/x/miniconda3/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

~/x/d/crnn.pytorch2/models/gcrnn.py in forward(self, x)
    134         x = x.permute(2, 0, 1).contiguous()  # [w, b, c]
    135 
--> 136         output = self.decoder(x)
    137 
    138         return output

~/x/miniconda3/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

~/x/miniconda3/lib/python3.6/site-packages/torch/nn/modules/container.py in forward(self, input)
     65     def forward(self, input):
     66         for module in self._modules.values():
---> 67             input = module(input)
     68         return input
     69 

~/x/miniconda3/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

~/x/d/crnn.pytorch2/models/gcrnn.py in forward(self, input)
     68                                                                                                                                                                                                    [37/696]
     69     def forward(self, input):
---> 70         recurrent, _ = self.rnn(input)
     71         T, b, h = recurrent.size()
     72         t_rec = recurrent.view(T * b, h)

~/x/miniconda3/lib/python3.6/site-packages/torch/nn/modules/module.py in __call__(self, *input, **kwargs)
    323         for hook in self._forward_pre_hooks.values():
    324             hook(self, input)
--> 325         result = self.forward(*input, **kwargs)
    326         for hook in self._forward_hooks.values():
    327             hook_result = hook(self, input, result)

~/x/miniconda3/lib/python3.6/site-packages/torch/nn/modules/rnn.py in forward(self, input, hx)
    167             flat_weight=flat_weight
    168         )
--> 169         output, hidden = func(input, self.all_weights, hx)
    170         if is_packed:
    171             output = PackedSequence(output, batch_sizes)

~/x/miniconda3/lib/python3.6/site-packages/torch/nn/_functions/rnn.py in forward(input, *fargs, **fkwargs)
    383             return hack_onnx_rnn((input,) + fargs, output, args, kwargs)
    384         else:
--> 385             return func(input, *fargs, **fkwargs)
    386 
    387     return forward

~/x/miniconda3/lib/python3.6/site-packages/torch/autograd/function.py in _do_forward(self, *input)
    326         self._nested_input = input
    327         flat_input = tuple(_iter_variables(input))
--> 328         flat_output = super(NestedIOFunction, self)._do_forward(*flat_input)
    329         nested_output = self._nested_output
    330         nested_variables = _unflatten(flat_output, self._nested_output)

~/x/miniconda3/lib/python3.6/site-packages/torch/autograd/function.py in forward(self, *args)
    348     def forward(self, *args):
    349         nested_tensors = _map_variable_tensor(self._nested_input)
--> 350         result = self.forward_extended(*nested_tensors)
    351         del self._nested_input
    352         self._nested_output = result

~/x/miniconda3/lib/python3.6/site-packages/torch/nn/_functions/rnn.py in forward_extended(self, input, weight, hx)
    292             hy = tuple(h.new() for h in hx)
    293 
--> 294         cudnn.rnn.forward(self, input, hx, weight, output, hy)
    295 
    296         self.save_for_backward(input, hx, weight, output)

~/x/miniconda3/lib/python3.6/site-packages/torch/backends/cudnn/rnn.py in forward(fn, input, hx, weight, output, hy)
    303                 fn.cy_desc, ctypes.c_void_p(cy.data_ptr()) if cx is not None else None,
    304                 ctypes.c_void_p(workspace.data_ptr()), workspace.size(0),
--> 305                 ctypes.c_void_p(fn.reserve.data_ptr()), fn.reserve.size(0)
    306             ))
    307         else:  # inference

~/x/miniconda3/lib/python3.6/site-packages/torch/backends/cudnn/__init__.py in check_error(status)
    281 def check_error(status):
    282     if status is not 0:
--> 283         raise CuDNNError(status)
    284 
    285 

CuDNNError: 8: b'CUDNN_STATUS_EXECUTION_FAILED'

UPD:
The code that was crashing was the code responsible for pre training the network, if I use the pretrained weights all the upprementined happen, if I do not - then training proceeds, but it is quite slow, I also found out that computer beeps, not with the internal beeper but quite softly, while training the network, so most probably it is a hardware problem, what is the best way to diagnose it ? The beeping sound seem to arrive from the video card or the motherboard.