<p>There is a technique called <a href="https://www.coursera.org/learn/deep-neural-network/lecture/htA0l/gradient-checking" rel="nofollow noreferrer">Gradient checking</a>. </p>

<p><a href="https://i.sstatic.net/TlW2i.png" rel="nofollow noreferrer"><img src="https://i.sstatic.net/TlW2i.png" alt="enter image description here"></a></p>

<p>With it, you can assert if you are calculating the correct gradient in the components of your ANN. The code implementation is:</p>

<pre><code>def gradient_check_n(parameters, gradients, X, Y, epsilon = 1e-7):

parameters_values, _ = dictionary_to_vector(parameters)
grad = gradients_to_vector(gradients)
num_parameters = parameters_values.shape[0]
J_plus = np.zeros((num_parameters, 1))
J_minus = np.zeros((num_parameters, 1))
gradapprox = np.zeros((num_parameters, 1))

# Compute gradapprox
for i in range(num_parameters):


    thetaplus = np.copy(parameters_values)                                      
    thetaplus[i][0] = thetaplus[i][0]+  epsilon                              
    J_plus[i], _ = forward_propagation_n(X, Y, vector_to_dictionary( thetaplus  ))                                  

    thetaminus =  np.copy(parameters_values)                                      
    thetaminus[i][0] = thetaplus[i][0]-  epsilon                                  
    J_minus[i], _ = forward_propagation_n(X, Y, vector_to_dictionary( thetaminus  ))                            

    gradapprox[i] = (J_plus[i]-J_minus[i])/(2*epsilon)



numerator = np.linalg.norm(grad-gradapprox)                              
denominator = np.linalg.norm(grad) +  np.linalg.norm(gradapprox)                            
difference = numerator/denominator                             

if difference &gt; 2e-7:
    print ("There is a mistake in the backward propagation. Difference = " + str(difference))
else:
    print ("Backward propagation is Okay. Difference = " + str(difference))

return difference
</code></pre>

<p>Where parameters is a dictionary with the parameters "W1","b1"...."Wl","bl" 
and grad is the output of the L_model_backward which contains gradients of the cost with respect to the parameters. Also, if you could share the x_train,y_train so we can debug it, it would be great, good luck.</p>

CC BY-SA 3.0
2018-04-25T04:23:48.983Z
2018-04-25T04:23:48.983Z
Bitzel
<p>I tried to build a neural network from scratch to build a cat or dog binary classifier using a sigmoid output unit. I seem to get the output value around 0.5(+/- 0.002) for every input. This seems really weird to me. Here's my code, Please let me know if there is a mistake in the implementation.</p>

<pre><code>def initialize_parameters_deep(layer_dims):
    l=len(layer_dims)
    parameters={}
    for l in range(1,len(layer_dims)):
        parameters['W'+str(l)]=np.random.randn(layer_dims[l],layer_dims[l-1])*0.01
        parameters['b'+str(l)]=np.zeros((layer_dims[l],1))
    return parameters

def linear_forward(A,W,b):
    Z=np.dot(W,A)+b
    cache=(A,W,b)
    return Z,cache


def sigmoid(Z):
    A = 1/(1+np.exp(-Z))
    cache=Z
    return A, cache


def relu(Z):
    A = np.maximum(0,Z)

    assert(A.shape == Z.shape)

    cache = Z 
    return A, cache

def relu_backward(dA, cache):
    Z = cache
    dZ = np.array(dA, copy=True) # just converting dz to a correct object.

    # When z &lt;= 0, you should set dz to 0 as well. 
    dZ[Z &lt;= 0] = 0

    assert (dZ.shape == Z.shape)

    return dZ

def sigmoid_backward(dA, cache):
    Z = cache

    s = 1/(1+np.exp(-Z))
    dZ = dA * s * (1-s)

    assert (dZ.shape == Z.shape)

    return dZ


def linear_activation_forward(A_prev,W,b,activation):
    if(activation=='sigmoid'):
        Z,linear_cache=linear_forward(A_prev,W,b)
        A,activation_cache=sigmoid(Z)
    elif activation=='relu':
        Z,linear_cache=linear_forward(A_prev,W,b)
        A,activation_cache=relu(Z)
    cache=(linear_cache,activation_cache)
    return A,cache

def L_model_forward(X,parameters):
    A=X
    L=len(parameters)//2
    caches=[]
    for l in range(1,L):
        A,cache=linear_activation_forward(A,parameters['W'+str(l)],parameters['b'+str(l)],'relu')
        caches.append(cache)
    AL,cache=linear_activation_forward(A,parameters['W'+str(L)],parameters['b'+str(L)],'sigmoid')
    caches.append(cache)
    return AL,caches

def compute_cost(AL,Y):
    m=Y.shape[1]
    cost=-1/m*np.sum(np.multiply(np.log(AL),Y)+np.multiply(np.log(1-AL),1-Y))
    return cost

def linear_backward(dZ,cache):
    A_prev,W,b=cache
    m=A_prev.shape[1]
    dW = np.dot(dZ,A_prev.T)/m
    db = np.sum(dZ,axis=1,keepdims=True)/m
    dA_prev = np.dot(W.T,dZ)
    return dA_prev,dW,db

def linear_activation_backward(activation,dA_prev,cache):
    linear_cache,activation_cache=cache
    if activation=='sigmoid':

        dZ=sigmoid_backward(dA_prev,activation_cache)
        dA_prev,dW,db=linear_backward(dZ,linear_cache)
    if activation=='relu':
        dZ=relu_backward(dA_prev,activation_cache)
        dA_prev,dW,db=linear_backward(dZ,linear_cache)
    return dA_prev,dW,db

def L_model_backward(AL,Y,caches):
    L=len(caches)
    m = AL.shape[1]
    Y = Y.reshape(AL.shape)
    dAL = - (np.divide(Y, AL) - np.divide(1 - Y, 1 - AL))

    grads={}
    current_cache=caches[-1]
    grads['dA'+str(L-1)],grads['dW'+str(L)],grads['db'+str(L)]=linear_activation_backward('sigmoid',dAL,current_cache)

    for l in reversed(range(L-1)):
        current_cache=caches[l]
        dA_prev_temp, dW_temp, db_temp = linear_activation_backward('relu',grads['dA'+str(l+1)],current_cache)
        grads["dA" + str(l)] = dA_prev_temp
        grads["dW" + str(l + 1)] = dW_temp
        grads["db" + str(l + 1)] = db_temp
    return grads
def Grad_Desc(parameters,grads,learning_rate):
    L=len(parameters)//2
    for l in range(L):
        parameters['W'+str(l+1)]=parameters['W'+str(l+1)]-learning_rate*grads['dW'+str(l+1)]
        parameters['b'+str(l+1)]=parameters['b'+str(l+1)]-learning_rate*grads['db'+str(l+1)] 
    return parameters

def L_layer_model(X,Y,learning_rate,num_iter,layer_dims):
    parameters=initialize_parameters_deep(layer_dims)
    costs=[]
    for i in range(num_iter):
        AL,caches=L_model_forward(X,parameters)
        cost=compute_cost(AL,Y)
        grads=L_model_backward(AL,Y,caches)
        parameters=Grad_Desc(parameters,grads,learning_rate)
        if i%100==0:
            print(cost)
            costs.append(cost)
    plt.plot(np.squeeze(costs))
def predict(X,parameters):
    AL,caches=L_model_forward(X,parameters)
    prediction=(AL&gt;0.5)
    return AL,prediction

L_layer_model(x_train,y_train,0.0075,12000,[12288,20,7,5,1])
prediction=predict(x_train,initialize_parameters_deep([12288,20,7,5,1])) 
</code></pre>

CC BY-SA 3.0
2018-04-19T12:13:32.463Z
Typical. Fell for the same thing when I made my first one.
FreezePhoenix
CC BY-SA 3.0
2018-04-19T12:17:05.713Z
Uh... Which language are you using? Reminds me, we need to make tags for those.
FreezePhoenix
CC BY-SA 3.0
2018-04-19T12:31:39.427Z
Python...Can you make suggestions please?Is there anything I need to change?
user165213
CC BY-SA 3.0
2018-04-19T12:33:03.003Z
Yes. **Why are you starting from scratch?** You should rethink about making your own AI library... Very hard.
FreezePhoenix
CC BY-SA 3.0
2018-04-19T12:34:16.547Z
Just want to understand how exactly it works..
user165213
CC BY-SA 3.0
2018-04-19T12:35:13.613Z
How familiar are you with JS?
FreezePhoenix
CC BY-SA 3.0
2018-04-19T12:36:18.260Z
I am not very confident about JS. Python would be better or if you could just tell me if there's anything wrong in the logic of my code, that would be great.
user165213
CC BY-SA 3.0
2018-04-19T12:37:16.847Z
I tried implementing a shallow network with two layers and it worked well. However, when I try generalizing for deep networks, I face this problem.
user165213
CC BY-SA 3.0
2018-04-20T12:00:26.967Z
@user165213 I think what you need to do is split the image up into pieces... Not feed the network a whole.
FreezePhoenix
<neural-networks><deep-learning><image-recognition>
Neural network returns about the same output(mean) for every input
user165213
# Neural network returns about the same output(mean) for every input

(Asked by: username_0 on 2018-04-19)

I tried to build a neural network from scratch to build a cat or dog binary classifier using a sigmoid output unit. I seem to get the output value around 0.5(+/- 0.002) for every input. This seems really weird to me. Here's my code, Please let me know if there is a mistake in the implementation.
def initialize_parameters_deep(layer_dims):
    l=len(layer_dims)
    parameters={}
    for l in range(1,len(layer_dims)):
        parameters['W'+str(l)]=np.random.randn(layer_dims[l],layer_dims[l-1])*0.01
        parameters['b'+str(l)]=np.zeros((layer_dims[l],1))
    return parameters

def linear_forward(A,W,b):
    Z=np.dot(W,A)+b
    cache=(A,W,b)
    return Z,cache


def sigmoid(Z):
    A = 1/(1+np.exp(-Z))
    cache=Z
    return A, cache


def relu(Z):
    A = np.maximum(0,Z)

    assert(A.shape == Z.shape)

    cache = Z 
    return A, cache

def relu_backward(dA, cache):
    Z = cache
    dZ = np.array(dA, copy=True) # just converting dz to a correct object.

    # When z <= 0, you should set dz to 0 as well. 
    dZ[Z <= 0] = 0

    assert (dZ.shape == Z.shape)

    return dZ

def sigmoid_backward(dA, cache):
    Z = cache

    s = 1/(1+np.exp(-Z))
    dZ = dA * s * (1-s)

    assert (dZ.shape == Z.shape)

    return dZ


def linear_activation_forward(A_prev,W,b,activation):
    if(activation=='sigmoid'):
        Z,linear_cache=linear_forward(A_prev,W,b)
        A,activation_cache=sigmoid(Z)
    elif activation=='relu':
        Z,linear_cache=linear_forward(A_prev,W,b)
        A,activation_cache=relu(Z)
    cache=(linear_cache,activation_cache)
    return A,cache

def L_model_forward(X,parameters):
    A=X
    L=len(parameters)//2
    caches=[]
    for l in range(1,L):
        A,cache=linear_activation_forward(A,parameters['W'+str(l)],parameters['b'+str(l)],'relu')
        caches.append(cache)
    AL,cache=linear_activation_forward(A,parameters['W'+str(L)],parameters['b'+str(L)],'sigmoid')
    caches.append(cache)
    return AL,caches

def compute_cost(AL,Y):
    m=Y.shape[1]
    cost=-1/m*np.sum(np.multiply(np.log(AL),Y)+np.multiply(np.log(1-AL),1-Y))
    return cost

def linear_backward(dZ,cache):
    A_prev,W,b=cache
    m=A_prev.shape[1]
    dW = np.dot(dZ,A_prev.T)/m
    db = np.sum(dZ,axis=1,keepdims=True)/m
    dA_prev = np.dot(W.T,dZ)
    return dA_prev,dW,db

def linear_activation_backward(activation,dA_prev,cache):
    linear_cache,activation_cache=cache
    if activation=='sigmoid':

        dZ=sigmoid_backward(dA_prev,activation_cache)
        dA_prev,dW,db=linear_backward(dZ,linear_cache)
    if activation=='relu':
        dZ=relu_backward(dA_prev,activation_cache)
        dA_prev,dW,db=linear_backward(dZ,linear_cache)
    return dA_prev,dW,db

def L_model_backward(AL,Y,caches):
    L=len(caches)
    m = AL.shape[1]
    Y = Y.reshape(AL.shape)
    dAL = - (np.divide(Y, AL) - np.divide(1 - Y, 1 - AL))

    grads={}
    current_cache=caches[-1]
    grads['dA'+str(L-1)],grads['dW'+str(L)],grads['db'+str(L)]=linear_activation_backward('sigmoid',dAL,current_cache)

    for l in reversed(range(L-1)):
        current_cache=caches[l]
        dA_prev_temp, dW_temp, db_temp = linear_activation_backward('relu',grads['dA'+str(l+1)],current_cache)
        grads["dA" + str(l)] = dA_prev_temp
        grads["dW" + str(l + 1)] = dW_temp
        grads["db" + str(l + 1)] = db_temp
    return grads
def Grad_Desc(parameters,grads,learning_rate):
    L=len(parameters)//2
    for l in range(L):
        parameters['W'+str(l+1)]=parameters['W'+str(l+1)]-learning_rate*grads['dW'+str(l+1)]
        parameters['b'+str(l+1)]=parameters['b'+str(l+1)]-learning_rate*grads['db'+str(l+1)] 
    return parameters

def L_layer_model(X,Y,learning_rate,num_iter,layer_dims):
    parameters=initialize_parameters_deep(layer_dims)
    costs=[]
    for i in range(num_iter):
        AL,caches=L_model_forward(X,parameters)
        cost=compute_cost(AL,Y)
        grads=L_model_backward(AL,Y,caches)
        parameters=Grad_Desc(parameters,grads,learning_rate)
        if i%100==0:
            print(cost)
            costs.append(cost)
    plt.plot(np.squeeze(costs))
def predict(X,parameters):
    AL,caches=L_model_forward(X,parameters)
    prediction=(AL>0.5)
    return AL,prediction

L_layer_model(x_train,y_train,0.0075,12000,[12288,20,7,5,1])
prediction=predict(x_train,initialize_parameters_deep([12288,20,7,5,1]))

**Tags:** neural-networks, deep-learning, image-recognition

> **Comments:**
> **username_1**: Typical. Fell for the same thing when I made my first one.  
> **username_1**: Uh... Which language are you using? Reminds me, we need to make tags for those.  
> **username_0**: Python...Can you make suggestions please?Is there anything I need to change?  
> **username_1**: Yes. **Why are you starting from scratch?** You should rethink about making your own AI library... Very hard.  
> **username_0**: Just want to understand how exactly it works..  
> **username_1**: How familiar are you with JS?  
> **username_0**: I am not very confident about JS. Python would be better or if you could just tell me if there's anything wrong in the logic of my code, that would be great.  
> **username_0**: I tried implementing a shallow network with two layers and it worked well. However, when I try generalizing for deep networks, I face this problem.  
> **username_1**: @username_0 I think what you need to do is split the image up into pieces... Not feed the network a whole.  

---

## 1 answers

### Answer by username_2 on 2018-04-25. Score: 1

There is a technique called Gradient checking. 

With it, you can assert if you are calculating the correct gradient in the components of your ANN. The code implementation is:
def gradient_check_n(parameters, gradients, X, Y, epsilon = 1e-7):

parameters_values, _ = dictionary_to_vector(parameters)
grad = gradients_to_vector(gradients)
num_parameters = parameters_values.shape[0]
J_plus = np.zeros((num_parameters, 1))
J_minus = np.zeros((num_parameters, 1))
gradapprox = np.zeros((num_parameters, 1))

# Compute gradapprox
for i in range(num_parameters):


    thetaplus = np.copy(parameters_values)                                      
    thetaplus[i][0] = thetaplus[i][0]+  epsilon                              
    J_plus[i], _ = forward_propagation_n(X, Y, vector_to_dictionary( thetaplus  ))                                  

    thetaminus =  np.copy(parameters_values)                                      
    thetaminus[i][0] = thetaplus[i][0]-  epsilon                                  
    J_minus[i], _ = forward_propagation_n(X, Y, vector_to_dictionary( thetaminus  ))                            

    gradapprox[i] = (J_plus[i]-J_minus[i])/(2*epsilon)



numerator = np.linalg.norm(grad-gradapprox)                              
denominator = np.linalg.norm(grad) +  np.linalg.norm(gradapprox)                            
difference = numerator/denominator                             

if difference > 2e-7:
    print ("There is a mistake in the backward propagation. Difference = " + str(difference))
else:
    print ("Backward propagation is Okay. Difference = " + str(difference))

return difference

Where parameters is a dictionary with the parameters "W1","b1"...."Wl","bl" 
and grad is the output of the L_model_backward which contains gradients of the cost with respect to the parameters. Also, if you could share the x_train,y_train so we can debug it, it would be great, good luck.


---

