text: Title: DFL crashed with 3080
  Body: DFL crash and then I got his error. 
  =============== Model Summary ===============
  ==                                         ==
  ==            Model name: Matt2_SAEHD      ==
  ==                                         ==
  ==     Current iteration: 3594             ==
  ==                                         ==
  ==------------- Model Options -------------==
  ==                                         ==
  ==            resolution: 128              ==
  ==             face_type: wf               ==
  ==     models_opt_on_gpu: True             ==
  ==                 archi: df               ==
  ==               ae_dims: 128              ==
  ==                e_dims: 64               ==
  ==                d_dims: 64               ==
  ==           d_mask_dims: 22               ==
  ==       masked_training: True             ==
  ==       eyes_mouth_prio: True             ==
  ==           uniform_yaw: False            ==
  ==             adabelief: True             ==
  ==            lr_dropout: n                ==
  ==           random_warp: False            ==
  ==       true_face_power: 0.01             ==
  ==      face_style_power: 0.01             ==
  ==        bg_style_power: 0.0              ==
  ==               ct_mode: none             ==
  ==              clipgrad: False            ==
  ==              pretrain: False            ==
  ==       autobackup_hour: 0                ==
  == write_preview_history: False            ==
  ==           target_iter: 0                ==
  ==           random_flip: True             ==
  ==            batch_size: 8                ==
  ==             gan_power: 0.1              ==
  ==        gan_patch_size: 16               ==
  ==              gan_dims: 16               ==
  ==                                         ==
  ==-------------- Running On ---------------==
  ==                                         ==
  ==          Device index: 0                ==
  ==                  Name: GeForce RTX 3080 ==
  ==                  VRAM: 10.00GB          ==
  ==                                         ==
  =============================================
  
  Error: Input to reshape is a tensor with 8 values, but the requested shape has 334176
           [[node gradients/Mean_11_grad/Reshape (defined at D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\core\leras\ops\__init__.py:55) ]]
  
  Original stack trace for 'gradients/Mean_11_grad/Reshape':
    File "threading.py", line 884, in _bootstrap
    File "threading.py", line 916, in _bootstrap_inner
    File "threading.py", line 864, in run
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\mainscripts\Trainer.py", line 58, in trainerThread
      debug=debug,
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\models\ModelBase.py", line 189, in __init__
      self.on_initialize()
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\models\Model_SAEHD\Model.py", line 468, in on_initialize
      gpu_D_code_loss_gvs += [ nn.gradients (gpu_D_code_loss, self.code_discriminator.get_weights() ) ]
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\core\leras\ops\__init__.py", line 55, in tf_gradients
      grads = gradients.gradients(loss, vars, colocate_gradients_with_ops=True )
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 172, in gradients
      unconnected_gradients)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\gradients_util.py", line 684, in _GradientsHelper
      lambda: grad_fn(op, *out_grads))
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\gradients_util.py", line 340, in _MaybeCompile
      return grad_fn()  # Exit early
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\gradients_util.py", line 684, in <lambda>
      lambda: grad_fn(op, *out_grads))
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\math_grad.py", line 255, in _MeanGrad
      sum_grad = _SumGrad(op, grad)[0]
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\math_grad.py", line 216, in _SumGrad
      grad = array_ops.reshape(grad, output_shape_kept_dims)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\util\dispatch.py", line 201, in wrapper
      return target(*args, **kwargs)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\array_ops.py", line 195, in reshape
      result = gen_array_ops.reshape(tensor, shape, name)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\gen_array_ops.py", line 8377, in reshape
      "Reshape", tensor=tensor, shape=shape, name=name)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 750, in _apply_op_helper
      attrs=attr_protos, op_def=op_def)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\framework\ops.py", line 3536, in _create_op_internal
      op_def=op_def)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\framework\ops.py", line 1990, in __init__
      self._traceback = tf_stack.extract_stack()
  
  ...which was originally created as op 'Mean_11', defined at:
    File "threading.py", line 884, in _bootstrap
  [elided 3 identical lines from previous traceback]
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\models\ModelBase.py", line 189, in __init__
      self.on_initialize()
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\models\Model_SAEHD\Model.py", line 465, in on_initialize
      gpu_D_code_loss = (DLoss(gpu_src_code_d_ones , gpu_dst_code_d) + \
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\models\Model_SAEHD\Model.py", line 454, in DLoss
      return tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits), axis=[1,2,3])
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\util\dispatch.py", line 201, in wrapper
      return target(*args, **kwargs)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\math_ops.py", line 2312, in reduce_mean_v1
      return reduce_mean(input_tensor, axis, keepdims, name)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\util\dispatch.py", line 201, in wrapper
      return target(*args, **kwargs)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\math_ops.py", line 2372, in reduce_mean
      name=name))
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\gen_math_ops.py", line 5781, in mean
      name=name)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 750, in _apply_op_helper
      attrs=attr_protos, op_def=op_def)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\framework\ops.py", line 3536, in _create_op_internal
      op_def=op_def)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\framework\ops.py", line 1990, in __init__
      self._traceback = tf_stack.extract_stack()
  
  Traceback (most recent call last):
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\client\session.py", line 1375, in _do_call
      return fn(*args)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\client\session.py", line 1360, in _run_fn
      target_list, run_metadata)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\client\session.py", line 1453, in _call_tf_sessionrun
      run_metadata)
  tensorflow.python.framework.errors_impl.InvalidArgumentError: Input to reshape is a tensor with 8 values, but the requested shape has 334176
           [[{{node gradients/Mean_11_grad/Reshape}}]]
  
  During handling of the above exception, another exception occurred:
  
  Traceback (most recent call last):
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\mainscripts\Trainer.py", line 130, in trainerThread
      iter, iter_time = model.train_one_iter()
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\models\ModelBase.py", line 462, in train_one_iter
      losses = self.onTrainOneIter()
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\models\Model_SAEHD\Model.py", line 701, in onTrainOneIter
      self.D_train (warped_src, warped_dst)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\models\Model_SAEHD\Model.py", line 545, in D_train
      nn.tf_sess.run ([D_loss_gv_op], feed_dict={self.warped_src: warped_src, self.warped_dst: warped_dst})
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\client\session.py", line 968, in run
      run_metadata_ptr)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\client\session.py", line 1191, in _run
      feed_dict_tensor, options, run_metadata)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\client\session.py", line 1369, in _do_run
      run_metadata)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\client\session.py", line 1394, in _do_call
      raise type(e)(node_def, op, message)
  tensorflow.python.framework.errors_impl.InvalidArgumentError: Input to reshape is a tensor with 8 values, but the requested shape has 334176
           [[node gradients/Mean_11_grad/Reshape (defined at D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\core\leras\ops\__init__.py:55) ]]
  
  Original stack trace for 'gradients/Mean_11_grad/Reshape':
    File "threading.py", line 884, in _bootstrap
    File "threading.py", line 916, in _bootstrap_inner
    File "threading.py", line 864, in run
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\mainscripts\Trainer.py", line 58, in trainerThread
      debug=debug,
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\models\ModelBase.py", line 189, in __init__
      self.on_initialize()
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\models\Model_SAEHD\Model.py", line 468, in on_initialize
      gpu_D_code_loss_gvs += [ nn.gradients (gpu_D_code_loss, self.code_discriminator.get_weights() ) ]
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\core\leras\ops\__init__.py", line 55, in tf_gradients
      grads = gradients.gradients(loss, vars, colocate_gradients_with_ops=True )
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\gradients_impl.py", line 172, in gradients
      unconnected_gradients)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\gradients_util.py", line 684, in _GradientsHelper
      lambda: grad_fn(op, *out_grads))
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\gradients_util.py", line 340, in _MaybeCompile
      return grad_fn()  # Exit early
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\gradients_util.py", line 684, in <lambda>
      lambda: grad_fn(op, *out_grads))
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\math_grad.py", line 255, in _MeanGrad
      sum_grad = _SumGrad(op, grad)[0]
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\math_grad.py", line 216, in _SumGrad
      grad = array_ops.reshape(grad, output_shape_kept_dims)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\util\dispatch.py", line 201, in wrapper
      return target(*args, **kwargs)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\array_ops.py", line 195, in reshape
      result = gen_array_ops.reshape(tensor, shape, name)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\gen_array_ops.py", line 8377, in reshape
      "Reshape", tensor=tensor, shape=shape, name=name)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 750, in _apply_op_helper
      attrs=attr_protos, op_def=op_def)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\framework\ops.py", line 3536, in _create_op_internal
      op_def=op_def)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\framework\ops.py", line 1990, in __init__
      self._traceback = tf_stack.extract_stack()
  
  ...which was originally created as op 'Mean_11', defined at:
    File "threading.py", line 884, in _bootstrap
  [elided 3 identical lines from previous traceback]
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\models\ModelBase.py", line 189, in __init__
      self.on_initialize()
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\models\Model_SAEHD\Model.py", line 465, in on_initialize
      gpu_D_code_loss = (DLoss(gpu_src_code_d_ones , gpu_dst_code_d) + \
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\DeepFaceLab\models\Model_SAEHD\Model.py", line 454, in DLoss
      return tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits), axis=[1,2,3])
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\util\dispatch.py", line 201, in wrapper
      return target(*args, **kwargs)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\math_ops.py", line 2312, in reduce_mean_v1
      return reduce_mean(input_tensor, axis, keepdims, name)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\util\dispatch.py", line 201, in wrapper
      return target(*args, **kwargs)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\math_ops.py", line 2372, in reduce_mean
      name=name))
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\ops\gen_math_ops.py", line 5781, in mean
      name=name)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\framework\op_def_library.py", line 750, in _apply_op_helper
      attrs=attr_protos, op_def=op_def)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\framework\ops.py", line 3536, in _create_op_internal
      op_def=op_def)
    File "D:\Downloads\DeepFaceLab_NVIDIA\_internal\python-3.6.8\lib\site-packages\tensorflow\python\framework\ops.py", line 1990, in __init__
      self._traceback = tf_stack.extract_stack()
  
  Any Ideas
  
  