<p dir="auto">for multi-outputs graph models, do we already support per-output sample and class weights?</p>
<p dir="auto">Update:<br>
I just saw class weights docstring says it supports a dict. But sample weights doesn't say that</p>
<p dir="auto">Hi,<br>
There seems to be a bug with Batch Normalization layer when using it for shared layers. I traced the problem to the running mean growing uncontrollably and then going to nan. It surfaced in my kind of a triplet loss model. See code attached to reproduce. Several other issues have this problem:<br>
<a class="issue-link js-issue-link" data-error-text="Failed to load title" data-id="392119044" data-permission-text="Title is private" data-url="https://github.com/keras-team/keras/issues/11897" data-hovercard-type="issue" data-hovercard-url="/keras-team/keras/issues/11897/hovercard" href="https://github.com/keras-team/keras/issues/11897">#11897</a><br>
<a class="issue-link js-issue-link" data-error-text="Failed to load title" data-id="304930902" data-permission-text="Title is private" data-url="https://github.com/keras-team/keras/issues/9646" data-hovercard-type="issue" data-hovercard-url="/keras-team/keras/issues/9646/hovercard" href="https://github.com/keras-team/keras/issues/9646">#9646</a></p>
<div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="import numpy as np

import keras

import keras.backend as K

from keras.optimizers import Adam
from keras.applications import ResNet50
from keras.applications.resnet50 import preprocess_input as preprocess_resnet50
from keras.models import Model, Input

from keras.layers import Input,Lambda,subtract,GlobalMaxPooling2D,Dense,GlobalAveragePooling2D,concatenate,Activation
from keras.layers import concatenate, Lambda, Input, Dense, Dropout, Flatten, Conv2D, MaxPooling2D, \
        BatchNormalization, Activation, GlobalAveragePooling2D,GlobalMaxPooling2D, SeparableConv2D,DepthwiseConv2D

def get_keras_model(model, input_shape):   
    
    i = Input(shape=input_shape)
    x=model.output
    x = GlobalMaxPooling2D()(x)
    x=BatchNormalization()(x)
    x=Dense(512,kernel_initializer='he_normal')(x)
    x=Activation('relu')(x)
    x=BatchNormalization()(x)
    o=Dense(5005,kernel_initializer='he_normal', activation='softmax')(x)
    return Model(inputs=model.input, outputs=o)

def get_resnet50(input_shape=(192,256,3)):
    K.clear_session()
    resnet=ResNet50(include_top=False, weights='imagenet', input_tensor=None, input_shape=input_shape, pooling=None)

    for l in range(0,len(resnet.layers)):
        if type(resnet.layers[l])==keras.layers.normalization.BatchNormalization:
            resnet.layers[l].trainable=True
        else:
            resnet.layers[l].trainable=False
    return resnet


resnet=ResNet50(include_top=False, weights='imagenet', input_tensor=None, input_shape=(192,256,3), pooling=None)
model=get_keras_model(get_resnet50(input_shape=(192,256,3)),input_shape=(192,256,3))
#model.load_weights(path+'models/Resnet50/loss_{:.4f}_{:.4f}_{:02d}.hdf5'.format(4.9773,0.6173,1))
trimmed_resnet50=Model(inputs=model.input,outputs=model.layers[173].output)
x=trimmed_resnet50.output
x = GlobalAveragePooling2D()(x)
x=BatchNormalization()(x)
x=Dense(1024,kernel_initializer='he_normal')(x)
x=Activation('relu')(x)
x=BatchNormalization()(x)
o=Dense(128,kernel_initializer='he_normal')(x)
base_net=Model(inputs=model.input,outputs=o)
#base_net.summary()


def create_model(base_model, input_shape=(192,256,3)):
    input_tensor1 = Input(shape=input_shape)
    input_tensor2 = Input(shape=input_shape)
    input_tensor3 = Input(shape=input_shape)

    reduce_sum = Dense(1,activation='linear',kernel_initializer='ones',bias_initializer='zeros',name='reduce_sum')

    x1 = base_model(input_tensor1)
    x2 = base_model(input_tensor2)
    x3 = base_model(input_tensor3)

    d12 = subtract([x1,x2])
    d13 = subtract([x1,x3])

    d12 = Lambda(lambda val: (val)**2)(d12)
    d13 = Lambda(lambda val: (val)**2)(d13)

    d12 = reduce_sum(d12)
    d13 = reduce_sum(d13)
    
    d12 = Lambda(lambda val: keras.backend.sqrt(val+K.epsilon()))(d12)
    d13 = Lambda(lambda val: keras.backend.sqrt(val+K.epsilon()))(d13)
    
    d = concatenate([d12,d13])
    d = Activation('softmax')(d)

    model = Model(inputs=[input_tensor1,input_tensor2,input_tensor3], outputs=d)
    metric = Model(inputs=input_tensor1, outputs=x1)


    for l in model.layers:
        if l.name == 'reduce_sum':
            print('reduce sum')
            l.trainable=False
    return model,metric
    
triplet_model,metric = create_model(base_net)

train_1=np.random.uniform(-128,128,size=(1000,192,256,3))
train_2=np.random.uniform(-128,128,size=(1000,192,256,3))
train_3=np.random.uniform(-128,128,size=(1000,192,256,3))
train_y=np.array([1,0]*1000).reshape(1000,2)

triplet_model.compile(loss='categorical_crossentropy',optimizer=Adam(1e-5, clipnorm=1.0),metrics=['accuracy'])
triplet_model.fit([train_1,train_2,train_3], train_y,batch_size=16,epochs=5)

intermediate=Model(inputs=metric.layers[1].get_input_at(0), outputs=metric.layers[1].layers[3].output)
output=intermediate.predict(train_1[0:1])
print(metric.layers[1].layers[3].name)
print(output.shape, output)

print(intermediate.layers[3].weights[2])
print(K.eval(intermediate.layers[3].weights[2]))
"><pre class="notranslate"><code class="notranslate">import numpy as np

import keras

import keras.backend as K

from keras.optimizers import Adam
from keras.applications import ResNet50
from keras.applications.resnet50 import preprocess_input as preprocess_resnet50
from keras.models import Model, Input

from keras.layers import Input,Lambda,subtract,GlobalMaxPooling2D,Dense,GlobalAveragePooling2D,concatenate,Activation
from keras.layers import concatenate, Lambda, Input, Dense, Dropout, Flatten, Conv2D, MaxPooling2D, \
        BatchNormalization, Activation, GlobalAveragePooling2D,GlobalMaxPooling2D, SeparableConv2D,DepthwiseConv2D

def get_keras_model(model, input_shape):   
    
    i = Input(shape=input_shape)
    x=model.output
    x = GlobalMaxPooling2D()(x)
    x=BatchNormalization()(x)
    x=Dense(512,kernel_initializer='he_normal')(x)
    x=Activation('relu')(x)
    x=BatchNormalization()(x)
    o=Dense(5005,kernel_initializer='he_normal', activation='softmax')(x)
    return Model(inputs=model.input, outputs=o)

def get_resnet50(input_shape=(192,256,3)):
    K.clear_session()
    resnet=ResNet50(include_top=False, weights='imagenet', input_tensor=None, input_shape=input_shape, pooling=None)

    for l in range(0,len(resnet.layers)):
        if type(resnet.layers[l])==keras.layers.normalization.BatchNormalization:
            resnet.layers[l].trainable=True
        else:
            resnet.layers[l].trainable=False
    return resnet


resnet=ResNet50(include_top=False, weights='imagenet', input_tensor=None, input_shape=(192,256,3), pooling=None)
model=get_keras_model(get_resnet50(input_shape=(192,256,3)),input_shape=(192,256,3))
#model.load_weights(path+'models/Resnet50/loss_{:.4f}_{:.4f}_{:02d}.hdf5'.format(4.9773,0.6173,1))
trimmed_resnet50=Model(inputs=model.input,outputs=model.layers[173].output)
x=trimmed_resnet50.output
x = GlobalAveragePooling2D()(x)
x=BatchNormalization()(x)
x=Dense(1024,kernel_initializer='he_normal')(x)
x=Activation('relu')(x)
x=BatchNormalization()(x)
o=Dense(128,kernel_initializer='he_normal')(x)
base_net=Model(inputs=model.input,outputs=o)
#base_net.summary()


def create_model(base_model, input_shape=(192,256,3)):
    input_tensor1 = Input(shape=input_shape)
    input_tensor2 = Input(shape=input_shape)
    input_tensor3 = Input(shape=input_shape)

    reduce_sum = Dense(1,activation='linear',kernel_initializer='ones',bias_initializer='zeros',name='reduce_sum')

    x1 = base_model(input_tensor1)
    x2 = base_model(input_tensor2)
    x3 = base_model(input_tensor3)

    d12 = subtract([x1,x2])
    d13 = subtract([x1,x3])

    d12 = Lambda(lambda val: (val)**2)(d12)
    d13 = Lambda(lambda val: (val)**2)(d13)

    d12 = reduce_sum(d12)
    d13 = reduce_sum(d13)
    
    d12 = Lambda(lambda val: keras.backend.sqrt(val+K.epsilon()))(d12)
    d13 = Lambda(lambda val: keras.backend.sqrt(val+K.epsilon()))(d13)
    
    d = concatenate([d12,d13])
    d = Activation('softmax')(d)

    model = Model(inputs=[input_tensor1,input_tensor2,input_tensor3], outputs=d)
    metric = Model(inputs=input_tensor1, outputs=x1)


    for l in model.layers:
        if l.name == 'reduce_sum':
            print('reduce sum')
            l.trainable=False
    return model,metric
    
triplet_model,metric = create_model(base_net)

train_1=np.random.uniform(-128,128,size=(1000,192,256,3))
train_2=np.random.uniform(-128,128,size=(1000,192,256,3))
train_3=np.random.uniform(-128,128,size=(1000,192,256,3))
train_y=np.array([1,0]*1000).reshape(1000,2)

triplet_model.compile(loss='categorical_crossentropy',optimizer=Adam(1e-5, clipnorm=1.0),metrics=['accuracy'])
triplet_model.fit([train_1,train_2,train_3], train_y,batch_size=16,epochs=5)

intermediate=Model(inputs=metric.layers[1].get_input_at(0), outputs=metric.layers[1].layers[3].output)
output=intermediate.predict(train_1[0:1])
print(metric.layers[1].layers[3].name)
print(output.shape, output)

print(intermediate.layers[3].weights[2])
print(K.eval(intermediate.layers[3].weights[2]))

</code></pre></div>
<p dir="auto">I'm trying to train a kind-of triplet loss CNN which has three shared Resnet50 models.<br>
Please make sure that the boxes below are checked before you submit your issue.<br>
If your issue is an <strong>implementation question</strong>, please ask your question on <a href="http://stackoverflow.com/questions/tagged/keras" rel="nofollow">StackOverflow</a> or <a href="https://keras-slack-autojoin.herokuapp.com/" rel="nofollow">on the Keras Slack channel</a> instead of opening a GitHub issue.</p>
<p dir="auto">Thank you!</p>
<ul dir="auto">
<li>
<p dir="auto">[ X] Check that you are up-to-date with the master branch of Keras. You can update with:<br>
<code class="notranslate">pip install git+git://github.com/keras-team/keras.git --upgrade --no-deps</code></p>
</li>
<li>
<p dir="auto">[ X] Check that your version of TensorFlow is up-to-date. The installation instructions can be found <a href="https://www.tensorflow.org/get_started/os_setup" rel="nofollow">here</a>.</p>
</li>
<li>
<p dir="auto">[ X] Provide a link to a GitHub Gist of a Python script that can reproduce your issue (or just copy the script here if it is short).</p>
</li>
</ul>