<p dir="auto">This error happens when training several models in a for loop:</p>
<p dir="auto"><code class="notranslate">W tensorflow/core/common_runtime/executor.cc:1076] 0x18ef02bb0 W tensorflow/core/common_runtime/executor.cc:1076] 0x18ef02bb0 Compute status: Resource exhausted: OOM when allocating tensor with shape [[Node: sub_2518 = Sub[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](sub_2518/x, Variable_1955/read)]] Traceback (most recent call last): File "/media/konet/01D15611945D72C0/Pycharm_ubuntu/deep_neural_network_binary/dnn_127_models_binary.py", line 398, in &lt;module&gt; fited = model.fit(Xtrain, ytrain, nb_epoch=n_epochs, batch_size=batch_size, show_accuracy=True, shuffle=True, validation_split=0.35) File "/usr/local/lib/python2.7/dist-packages/keras/models.py", line 581, in fit shuffle=shuffle, metrics=metrics) File "/usr/local/lib/python2.7/dist-packages/keras/models.py", line 239, in _fit outs = f(ins_batch) File "/usr/local/lib/python2.7/dist-packages/keras/backend/tensorflow_backend.py", line 336, in __call__ updated = session.run(self.outputs + self.updates, feed_dict=feed_dict) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 368, in run results = self._do_run(target_list, unique_fetch_targets, feed_dict_string) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 444, in _do_run e.code) tensorflow.python.framework.errors.ResourceExhaustedError: OOM when allocating tensor with shapedim { size: 3200 } dim { size: 2000 } [[Node: mul_4305 = Mul[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_1954/read, Variable_1956/read)]] [[Node: add_3103/_9711 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:0", send_device_incarnation=1, tensor_name="edge_423_add_3103", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]] Caused by op u'mul_4305', defined at: File "/media/konet/01D15611945D72C0/Pycharm_ubuntu/deep_neural_network_binary/dnn_127_models_binary.py", line 387, in &lt;module&gt; model.compile(loss='binary_crossentropy', optimizer='adam', class_mode='binary') File "/usr/local/lib/python2.7/dist-packages/keras/models.py", line 440, in compile train_loss) File "/usr/local/lib/python2.7/dist-packages/keras/optimizers.py", line 262, in get_updates m_t = (self.beta_1 * m) + (1 - self.beta_1) * g File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variables.py", line 452, in &lt;lambda&gt; setattr(Variable, operator, lambda a, b: Variable._RunOp(operator, a, b)) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variables.py", line 467, in _RunOp return getattr(ops.Tensor, operator)(a._AsTensor(), b) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/math_ops.py", line 426, in binary_op_wrapper return func(x, y, name=name) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_math_ops.py", line 728, in mul return _op_def_lib.apply_op("Mul", x=x, y=y, name=name) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/op_def_library.py", line 664, in apply_op op_def=op_def) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1834, in create_op original_op=self._default_original_op, op_def=op_def) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1043, in __init__ self._traceback = _extract_stack() : OOM when allocating tensor with shape [[Node: sub_2518 = Sub[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](sub_2518/x, Variable_1955/read)]] Traceback (most recent call last): File "/media/konet/01D15611945D72C0/Pycharm_ubuntu/deep_neural_network_binary/dnn_127_models_binary.py", line 398, in &lt;module&gt; fited = model.fit(Xtrain, ytrain, nb_epoch=n_epochs, batch_size=batch_size, show_accuracy=True, shuffle=True, validation_split=0.35) File "/usr/local/lib/python2.7/dist-packages/keras/models.py", line 581, in fit shuffle=shuffle, metrics=metrics) File "/usr/local/lib/python2.7/dist-packages/keras/models.py", line 239, in _fit outs = f(ins_batch) File "/usr/local/lib/python2.7/dist-packages/keras/backend/tensorflow_backend.py", line 336, in __call__ updated = session.run(self.outputs + self.updates, feed_dict=feed_dict) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 368, in run results = self._do_run(target_list, unique_fetch_targets, feed_dict_string) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/client/session.py", line 444, in _do_run e.code) tensorflow.python.framework.errors.ResourceExhaustedError: OOM when allocating tensor with shapedim { size: 3200 } dim { size: 2000 } [[Node: mul_4305 = Mul[T=DT_FLOAT, _device="/job:localhost/replica:0/task:0/gpu:0"](Variable_1954/read, Variable_1956/read)]] [[Node: add_3103/_9711 = _Recv[client_terminated=false, recv_device="/job:localhost/replica:0/task:0/cpu:0", send_device="/job:localhost/replica:0/task:0/gpu:0", send_device_incarnation=1, tensor_name="edge_423_add_3103", tensor_type=DT_FLOAT, _device="/job:localhost/replica:0/task:0/cpu:0"]()]] Caused by op u'mul_4305', defined at: File "/media/konet/01D15611945D72C0/Pycharm_ubuntu/deep_neural_network_binary/dnn_127_models_binary.py", line 387, in &lt;module&gt; model.compile(loss='binary_crossentropy', optimizer='adam', class_mode='binary') File "/usr/local/lib/python2.7/dist-packages/keras/models.py", line 440, in compile train_loss) File "/usr/local/lib/python2.7/dist-packages/keras/optimizers.py", line 262, in get_updates m_t = (self.beta_1 * m) + (1 - self.beta_1) * g File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variables.py", line 452, in &lt;lambda&gt; setattr(Variable, operator, lambda a, b: Variable._RunOp(operator, a, b)) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/variables.py", line 467, in _RunOp return getattr(ops.Tensor, operator)(a._AsTensor(), b) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/math_ops.py", line 426, in binary_op_wrapper return func(x, y, name=name) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/gen_math_ops.py", line 728, in mul return _op_def_lib.apply_op("Mul", x=x, y=y, name=name) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/ops/op_def_library.py", line 664, in apply_op op_def=op_def) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1834, in create_op original_op=self._default_original_op, op_def=op_def) File "/usr/local/lib/python2.7/dist-packages/tensorflow/python/framework/ops.py", line 1043, in __init__ self._traceback = _extract_stack() </code></p>
<p dir="auto">Any advice on how to solve this?</p>
<p dir="auto">Thank you</p>
<p dir="auto"><strong>System information</strong></p>
<ul dir="auto">
<li>Have I written custom code: Yes</li>
<li>OS Platform and Distribution: Manjaro testing, x86_64</li>
<li>TensorFlow installed from (source or binary): pypi binary</li>
<li>TensorFlow version (use command below): v2.0.0-rc2-26-g64c3d38 2.0.0</li>
<li>Python version: 3.7.4</li>
</ul>
<p dir="auto"><strong>Describe the current behavior</strong><br>
TensorFlow returns duplicated regularisation losses for layers which hold references to regularised variables built in other layers.</p>
<p dir="auto"><strong>Describe the expected behavior</strong><br>
Return a single regularisation loss per variable.</p>
<p dir="auto"><strong>Code to reproduce the issue</strong></p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="import tensorflow as tf


class A(tf.keras.layers.Layer):
    def __init__(self, layer):
        super(A, self).__init__()
        self.layer = layer

    def call(self, inputs):
        return self.layer(inputs)


class B(tf.keras.layers.Layer):
    def __init__(self):
        super(B, self).__init__()
        self.obj = tf.keras.layers.Dense(13, kernel_regularizer=tf.keras.regularizers.l1(5))
        self.layerB = A(self.obj)

    def call(self, inputs):
        return self.layerB(inputs)

model = B()

output = model(tf.ones([5, 10]))
print(len(model.losses))"><pre class="notranslate"><code class="notranslate">import tensorflow as tf


class A(tf.keras.layers.Layer):
    def __init__(self, layer):
        super(A, self).__init__()
        self.layer = layer

    def call(self, inputs):
        return self.layer(inputs)


class B(tf.keras.layers.Layer):
    def __init__(self):
        super(B, self).__init__()
        self.obj = tf.keras.layers.Dense(13, kernel_regularizer=tf.keras.regularizers.l1(5))
        self.layerB = A(self.obj)

    def call(self, inputs):
        return self.layerB(inputs)

model = B()

output = model(tf.ones([5, 10]))
print(len(model.losses))
</code></pre></div>
<p dir="auto">Once we rename self.obj to obj, i.e. not saving it as a class member, we end up with a single regularisation loss.</p>
<p dir="auto"><strong>Other info / logs</strong><br>
More info and logs in <a class="issue-link js-issue-link" data-error-text="Failed to load title" data-id="503574506" data-permission-text="Title is private" data-url="https://github.com/tensorflow/addons/issues/577" data-hovercard-type="issue" data-hovercard-url="/tensorflow/addons/issues/577/hovercard" href="https://github.com/tensorflow/addons/issues/577">tensorflow/addons#577</a><br>
Plus <a class="user-mention notranslate" data-hovercard-type="user" data-hovercard-url="/users/guillaumekln/hovercard" data-octo-click="hovercard-link-click" data-octo-dimensions="link_type:self" href="https://github.com/guillaumekln">@guillaumekln</a> 's response <a class="issue-link js-issue-link" data-error-text="Failed to load title" data-id="503574506" data-permission-text="Title is private" data-url="https://github.com/tensorflow/addons/issues/577" data-hovercard-type="issue" data-hovercard-url="/tensorflow/addons/issues/577/hovercard?comment_id=539530375&amp;comment_type=issue_comment" href="https://github.com/tensorflow/addons/issues/577#issuecomment-539530375">tensorflow/addons#577 (comment)</a></p>