Answers.Body: <p>It seems I've solved the issue. There was several mistakes:<br>
  1. I've generated random weights from 0 to 1. As a result, too big numbers passed through softmax function (>10000), and the function wasn't calculated correctly. I divided each initial weight on the number of neurons in previous layer and solved the issue.<br>
  2. I've calculated separate delta for biases while delta must be the same for main weights and biases.<br>
  If anyone is interested, here is the correct code (83% and 89% precision after first and second launch):</p>
  
  <pre><code>import numpy as np
  
  def ReLU(x):
      return np.maximum(0, x)
  
  def ReLU_derivative(x):
      return np.greater(x, 0).astype(int)
  
  def softmax(x):
      shift = x - np.max(x)
      return np.exp(shift) / np.sum(np.exp(shift))
  
  def softmax_derivative(x):
      sm_array = softmax(x)
      J = np.zeros((x.size, x.size))
      for i in range(x.size):
          for j in range(x.size):
              delta = np.equal(i, j).astype(int)
              J[j, i] = sm_array[0][i] * (delta - sm_array[0][j])
      return J
  
  class Layer:
      def __init__(self, size, prev_layer=None):
          self.size = size
          self.prevLayer = prev_layer
          self.nextLayer = None
          self.data = None
          self.data_in = None
          if prev_layer is not None:
              self.prevLayer.nextLayer = self
              self.W = np.random.random((self.prevLayer.size, size)) / (self.prevLayer.size + 1)
              self.W_bias = np.random.random((1, size)) / (self.prevLayer.size + 1)
          else:
              self.W = None
              self.W_bias = None
  
      def forward(self):
          if self.prevLayer is not None:
              self.data_in = np.dot(self.prevLayer.data, self.W)
              self.data_in += np.dot([[1]], self.W_bias)
              if self.nextLayer is not None:
                  self.data = ReLU(self.data_in)
                  self.nextLayer.forward()
              else:
                  self.data = softmax(self.data_in)
          else:
              self.nextLayer.forward()
  
      def backprop(self, expected_output=None, prev_delta=None):
          if prev_delta is None:
              delta = np.dot(-(expected_output - self.data), softmax_derivative(self.data_in))
          else:
              delta = np.dot(prev_delta, self.nextLayer.W.T) * ReLU_derivative(self.data_in)
          training_velocity = 0.1
          W_dif = np.dot(self.prevLayer.data.T, delta) * training_velocity
          W_bias_dif = np.dot([[1]], delta) * training_velocity
          if self.prevLayer.prevLayer is not None:
              self.prevLayer.backprop(prev_delta=delta)
          self.W -= W_dif
          self.W_bias -= W_bias_dif
  
  f_images = open("train-images.idx3-ubyte", "br")
  f_images.seek(4)
  f_labels = open("train-labels.idx1-ubyte", "br")
  f_labels.seek(8)
  images_number = int.from_bytes(f_images.read(4), byteorder='big')
  rows_number = int.from_bytes(f_images.read(4), byteorder='big')
  cols_number = int.from_bytes(f_images.read(4), byteorder='big')
  
  input_layer = Layer(rows_number*cols_number)
  hidden_layer1 = Layer(rows_number*cols_number*7//10, input_layer)
  hidden_layer2 = Layer(rows_number*cols_number*7//10, hidden_layer1)
  output_layer = Layer(10, hidden_layer2)
  digits = np.array([np.zeros(10)])
  
  print("Training:")
  input_image = np.array([np.zeros(rows_number * cols_number)])
  for k in range(images_number):
      for i in range(rows_number):
          for j in range(cols_number):
              input_image[0][i*cols_number+j] = int.from_bytes(f_images.read(1), byteorder='big') / 255.0
      input_layer.data = input_image
      input_layer.forward()
      current_digit = int.from_bytes(f_labels.read(1), byteorder='big')
      digits[0][current_digit] = 1
      output_layer.backprop(expected_output=digits)
      digits[0][current_digit] = 0
      if((k+1) % 1000 == 0):
          print(str(k+1) + " / " + str(images_number))
  f_images.close()
  f_labels.close()
  
  f_images = open("t10k-images.idx3-ubyte", "br")
  f_images.seek(4)
  f_labels = open("t10k-labels.idx1-ubyte", "br")
  f_labels.seek(8)
  images_number = int.from_bytes(f_images.read(4), byteorder='big')
  rows_number = int.from_bytes(f_images.read(4), byteorder='big')
  cols_number = int.from_bytes(f_images.read(4), byteorder='big')
  
  print("\r\nTesting:")
  correct = 0
  for k in range(images_number):
      for i in range(rows_number):
          for j in range(cols_number):
              input_image[0][i*cols_number+j] = int.from_bytes(f_images.read(1), byteorder='big')
      input_layer.data = input_image
      input_layer.forward()
      current_digit = int.from_bytes(f_labels.read(1), byteorder='big')
      if np.argmax(output_layer.data[0]) == current_digit:
          correct += 1
      if((k+1) % 1000 == 0):
          print(str(k+1) + " / " + str(images_number))
  
  print("\r\nCorrect: " + str(correct) + " / " + str(images_number))
  f_images.close()
  f_labels.close()
  </code></pre>
  
Answers.ContentLicense: CC BY-SA 4.0
Answers.CreationDate: 2019-02-10T22:31:52.487Z
Answers.LastActivityDate: 2019-02-10T22:31:52.487Z
Answers.UserDisplayName: Андрій Немченко
Body: <p>I'm a beginner in machine learning and I was trying to make a test neural network for digits recognition from scratch using Numpy. I used MNIST dataset for training and testing. Input layer have 28*28 neurons which correspond to each pixel of image that must be recognized. Output layer have 10 neurons which correspond to each digit (0-9), and return values from 0 to 1 which mean chance that the corresponding digit is displayed on the image. Class Layer represents a separate layer and contains links to previous and next layer (if prevLayer is None, the current layer is input layer; if nextLayer is None, the current layer is output). The forward() method is responsible for passing data through neural network. The backprop() method is responsible for training of neural network via Backpropagation algorithm. A Layer object contains weights (W) between previous and current layer (input layer object doesn't contain weights). 'data_in' property contains a vector of calculated values before passing them into activation function. Property 'data' contains the values after activation function. But, unfortunately, it doesn't work: returned value of loss function doesn't decrease during training and neural network returns the same result during testing. I assume that bugs might be associated with backprop() and softmax_derivatime() methods. I tried in vain to find all bugs. Here's my code:</p>
  
  <pre><code>import numpy as np
  
  def ReLU(x):
      return np.maximum(0, x)
  
  def ReLU_derivative(x):
      return np.greater(x, 0).astype(int)
  
  def softmax(x):
      shift = x - np.max(x)
      return np.exp(shift) / np.sum(np.exp(shift))
  
  def softmax_derivative(x):
      sm_array = softmax(x)
      J = np.zeros((x.size, x.size))
      for i in range(x.size):
          for j in range(x.size):
              delta = np.equal(i, j).astype(int)
              J[j, i] = sm_array[0][i] * (delta - sm_array[0][j])
      return J
  
  class Layer:
      def __init__(self, size, prev_layer=None):
          self.size = size
          self.prevLayer = prev_layer
          self.nextLayer = None
          self.data = None
          self.data_in = None
          if prev_layer is not None:
              self.prevLayer.nextLayer = self
              self.W = np.random.random((self.prevLayer.size, size))
              self.W_bias = np.array([np.random.random(size)])
          else:
              self.W = None
              self.W_bias = None
  
      def forward(self):
          if self.prevLayer is not None:
              self.data_in = np.dot(self.prevLayer.data, self.W)
              self.data_in += np.dot([[1]], self.W_bias)
              if self.nextLayer is not None:
                  self.data = ReLU(self.data_in)
                  self.nextLayer.forward()
              else:
                  self.data = softmax(self.data_in)
          else:
              self.nextLayer.forward()
  
      def backprop(self, expected_output=None, prev_delta=None):
          if prev_delta is None:
              #print(self.data_in)
              delta = np.dot(-(expected_output - self.data), softmax_derivative(self.data_in))
              delta_bias = delta
          else:
              delta = np.dot(prev_delta, self.nextLayer.W.T) * ReLU_derivative(self.data_in)
              delta_bias = np.dot(prev_delta, self.nextLayer.W_bias.T) * ReLU_derivative(self.data_in)
          training_velocity = 0.1
          W_dif = np.dot(self.prevLayer.data.T, delta) * training_velocity
          W_bias_dif = np.dot([[1]], delta_bias) * training_velocity
          if self.prevLayer.prevLayer is not None:
              self.prevLayer.backprop(prev_delta=delta)
          self.W -= W_dif
          self.W_bias -= W_bias_dif
  
  f_images = open("train-images.idx3-ubyte", "br")
  f_images.seek(4)
  f_labels = open("train-labels.idx1-ubyte", "br")
  f_labels.seek(8)
  images_number = int.from_bytes(f_images.read(4), byteorder='big')
  rows_number = int.from_bytes(f_images.read(4), byteorder='big')
  cols_number = int.from_bytes(f_images.read(4), byteorder='big')
  
  input_layer = Layer(rows_number*cols_number)
  hidden_layer1 = Layer(rows_number*cols_number*7//10, input_layer)
  hidden_layer2 = Layer(rows_number*cols_number*7//10, hidden_layer1)
  output_layer = Layer(10, hidden_layer2)
  digits = np.array([np.zeros(10)])
  
  input_image = np.array([np.zeros(rows_number * cols_number)])
  for k in range(images_number):
      for i in range(rows_number):
          for j in range(cols_number):
              input_image[0][i*cols_number+j] = int.from_bytes(f_images.read(1), byteorder='big') / 255.0 * 2 - 1
      input_layer.data = input_image
      input_layer.forward()
      current_digit = int.from_bytes(f_labels.read(1), byteorder='big')
      digits[0][current_digit] = 1
      output_layer.backprop(expected_output=digits)
      print(np.sum((digits - output_layer.data)**2)/2)
      digits[0][current_digit] = 0
      if((k+1) % 1000 == 0):
          print(str(k+1) + " / " + str(images_number))
  f_images.close()
  f_labels.close()
  
  f_images = open("t10k-images.idx3-ubyte", "br")
  f_images.seek(4)
  f_labels = open("t10k-labels.idx1-ubyte", "br")
  f_labels.seek(8)
  images_number = int.from_bytes(f_images.read(4), byteorder='big')
  rows_number = int.from_bytes(f_images.read(4), byteorder='big')
  cols_number = int.from_bytes(f_images.read(4), byteorder='big')
  
  for k in range(images_number):
      for i in range(rows_number):
          for j in range(cols_number):
              input_image[0][i*cols_number+j] = int.from_bytes(f_images.read(1), byteorder='big')
      input_layer.data = input_image
      input_layer.forward()
      current_digit = int.from_bytes(f_labels.read(1), byteorder='big')
      print(output_layer.data)
  
  f_images.close()
  f_labels.close()
  </code></pre>
  
  <p>I would appreciate for any help. Thanks in advance!</p>
  
Tags: <neural-networks><machine-learning><backpropagation><object-recognition><perceptron>
Title: A neural network for digits recognition doesn't work (MNIST, Numpy)
UserDisplayName: Андрій Немченко
ThreadText: # A neural network for digits recognition doesn't work (MNIST, Numpy)
  
  (Asked by: username_0 on 2019-02-08)
  
  I'm a beginner in machine learning and I was trying to make a test neural network for digits recognition from scratch using Numpy. I used MNIST dataset for training and testing. Input layer have 28*28 neurons which correspond to each pixel of image that must be recognized. Output layer have 10 neurons which correspond to each digit (0-9), and return values from 0 to 1 which mean chance that the corresponding digit is displayed on the image. Class Layer represents a separate layer and contains links to previous and next layer (if prevLayer is None, the current layer is input layer; if nextLayer is None, the current layer is output). The forward() method is responsible for passing data through neural network. The backprop() method is responsible for training of neural network via Backpropagation algorithm. A Layer object contains weights (W) between previous and current layer (input layer object doesn't contain weights). 'data_in' property contains a vector of calculated values before passing them into activation function. Property 'data' contains the values after activation function. But, unfortunately, it doesn't work: returned value of loss function doesn't decrease during training and neural network returns the same result during testing. I assume that bugs might be associated with backprop() and softmax_derivatime() methods. I tried in vain to find all bugs. Here's my code:
  import numpy as np
  
  def ReLU(x):
      return np.maximum(0, x)
  
  def ReLU_derivative(x):
      return np.greater(x, 0).astype(int)
  
  def softmax(x):
      shift = x - np.max(x)
      return np.exp(shift) / np.sum(np.exp(shift))
  
  def softmax_derivative(x):
      sm_array = softmax(x)
      J = np.zeros((x.size, x.size))
      for i in range(x.size):
          for j in range(x.size):
              delta = np.equal(i, j).astype(int)
              J[j, i] = sm_array[0][i] * (delta - sm_array[0][j])
      return J
  
  class Layer:
      def __init__(self, size, prev_layer=None):
          self.size = size
          self.prevLayer = prev_layer
          self.nextLayer = None
          self.data = None
          self.data_in = None
          if prev_layer is not None:
              self.prevLayer.nextLayer = self
              self.W = np.random.random((self.prevLayer.size, size))
              self.W_bias = np.array([np.random.random(size)])
          else:
              self.W = None
              self.W_bias = None
  
      def forward(self):
          if self.prevLayer is not None:
              self.data_in = np.dot(self.prevLayer.data, self.W)
              self.data_in += np.dot([[1]], self.W_bias)
              if self.nextLayer is not None:
                  self.data = ReLU(self.data_in)
                  self.nextLayer.forward()
              else:
                  self.data = softmax(self.data_in)
          else:
              self.nextLayer.forward()
  
      def backprop(self, expected_output=None, prev_delta=None):
          if prev_delta is None:
              #print(self.data_in)
              delta = np.dot(-(expected_output - self.data), softmax_derivative(self.data_in))
              delta_bias = delta
          else:
              delta = np.dot(prev_delta, self.nextLayer.W.T) * ReLU_derivative(self.data_in)
              delta_bias = np.dot(prev_delta, self.nextLayer.W_bias.T) * ReLU_derivative(self.data_in)
          training_velocity = 0.1
          W_dif = np.dot(self.prevLayer.data.T, delta) * training_velocity
          W_bias_dif = np.dot([[1]], delta_bias) * training_velocity
          if self.prevLayer.prevLayer is not None:
              self.prevLayer.backprop(prev_delta=delta)
          self.W -= W_dif
          self.W_bias -= W_bias_dif
  
  f_images = open("train-images.idx3-ubyte", "br")
  f_images.seek(4)
  f_labels = open("train-labels.idx1-ubyte", "br")
  f_labels.seek(8)
  images_number = int.from_bytes(f_images.read(4), byteorder='big')
  rows_number = int.from_bytes(f_images.read(4), byteorder='big')
  cols_number = int.from_bytes(f_images.read(4), byteorder='big')
  
  input_layer = Layer(rows_number*cols_number)
  hidden_layer1 = Layer(rows_number*cols_number*7//10, input_layer)
  hidden_layer2 = Layer(rows_number*cols_number*7//10, hidden_layer1)
  output_layer = Layer(10, hidden_layer2)
  digits = np.array([np.zeros(10)])
  
  input_image = np.array([np.zeros(rows_number * cols_number)])
  for k in range(images_number):
      for i in range(rows_number):
          for j in range(cols_number):
              input_image[0][i*cols_number+j] = int.from_bytes(f_images.read(1), byteorder='big') / 255.0 * 2 - 1
      input_layer.data = input_image
      input_layer.forward()
      current_digit = int.from_bytes(f_labels.read(1), byteorder='big')
      digits[0][current_digit] = 1
      output_layer.backprop(expected_output=digits)
      print(np.sum((digits - output_layer.data)**2)/2)
      digits[0][current_digit] = 0
      if((k+1) % 1000 == 0):
          print(str(k+1) + " / " + str(images_number))
  f_images.close()
  f_labels.close()
  
  f_images = open("t10k-images.idx3-ubyte", "br")
  f_images.seek(4)
  f_labels = open("t10k-labels.idx1-ubyte", "br")
  f_labels.seek(8)
  images_number = int.from_bytes(f_images.read(4), byteorder='big')
  rows_number = int.from_bytes(f_images.read(4), byteorder='big')
  cols_number = int.from_bytes(f_images.read(4), byteorder='big')
  
  for k in range(images_number):
      for i in range(rows_number):
          for j in range(cols_number):
              input_image[0][i*cols_number+j] = int.from_bytes(f_images.read(1), byteorder='big')
      input_layer.data = input_image
      input_layer.forward()
      current_digit = int.from_bytes(f_labels.read(1), byteorder='big')
      print(output_layer.data)
  
  f_images.close()
  f_labels.close()
  
  I would appreciate for any help. Thanks in advance!
  
  **Tags:** neural-networks, machine-learning, backpropagation, object-recognition, perceptron
  
  ---
  
  ## 1 answers
  
  ### Answer by username_0 on 2019-02-10. Score: 1 | Accepted answer
  
  It seems I've solved the issue. There was several mistakes:
  1. I've generated random weights from 0 to 1. As a result, too big numbers passed through softmax function (>10000), and the function wasn't calculated correctly. I divided each initial weight on the number of neurons in previous layer and solved the issue.
  2. I've calculated separate delta for biases while delta must be the same for main weights and biases.
  If anyone is interested, here is the correct code (83% and 89% precision after first and second launch):
  import numpy as np
  
  def ReLU(x):
      return np.maximum(0, x)
  
  def ReLU_derivative(x):
      return np.greater(x, 0).astype(int)
  
  def softmax(x):
      shift = x - np.max(x)
      return np.exp(shift) / np.sum(np.exp(shift))
  
  def softmax_derivative(x):
      sm_array = softmax(x)
      J = np.zeros((x.size, x.size))
      for i in range(x.size):
          for j in range(x.size):
              delta = np.equal(i, j).astype(int)
              J[j, i] = sm_array[0][i] * (delta - sm_array[0][j])
      return J
  
  class Layer:
      def __init__(self, size, prev_layer=None):
          self.size = size
          self.prevLayer = prev_layer
          self.nextLayer = None
          self.data = None
          self.data_in = None
          if prev_layer is not None:
              self.prevLayer.nextLayer = self
              self.W = np.random.random((self.prevLayer.size, size)) / (self.prevLayer.size + 1)
              self.W_bias = np.random.random((1, size)) / (self.prevLayer.size + 1)
          else:
              self.W = None
              self.W_bias = None
  
      def forward(self):
          if self.prevLayer is not None:
              self.data_in = np.dot(self.prevLayer.data, self.W)
              self.data_in += np.dot([[1]], self.W_bias)
              if self.nextLayer is not None:
                  self.data = ReLU(self.data_in)
                  self.nextLayer.forward()
              else:
                  self.data = softmax(self.data_in)
          else:
              self.nextLayer.forward()
  
      def backprop(self, expected_output=None, prev_delta=None):
          if prev_delta is None:
              delta = np.dot(-(expected_output - self.data), softmax_derivative(self.data_in))
          else:
              delta = np.dot(prev_delta, self.nextLayer.W.T) * ReLU_derivative(self.data_in)
          training_velocity = 0.1
          W_dif = np.dot(self.prevLayer.data.T, delta) * training_velocity
          W_bias_dif = np.dot([[1]], delta) * training_velocity
          if self.prevLayer.prevLayer is not None:
              self.prevLayer.backprop(prev_delta=delta)
          self.W -= W_dif
          self.W_bias -= W_bias_dif
  
  f_images = open("train-images.idx3-ubyte", "br")
  f_images.seek(4)
  f_labels = open("train-labels.idx1-ubyte", "br")
  f_labels.seek(8)
  images_number = int.from_bytes(f_images.read(4), byteorder='big')
  rows_number = int.from_bytes(f_images.read(4), byteorder='big')
  cols_number = int.from_bytes(f_images.read(4), byteorder='big')
  
  input_layer = Layer(rows_number*cols_number)
  hidden_layer1 = Layer(rows_number*cols_number*7//10, input_layer)
  hidden_layer2 = Layer(rows_number*cols_number*7//10, hidden_layer1)
  output_layer = Layer(10, hidden_layer2)
  digits = np.array([np.zeros(10)])
  
  print("Training:")
  input_image = np.array([np.zeros(rows_number * cols_number)])
  for k in range(images_number):
      for i in range(rows_number):
          for j in range(cols_number):
              input_image[0][i*cols_number+j] = int.from_bytes(f_images.read(1), byteorder='big') / 255.0
      input_layer.data = input_image
      input_layer.forward()
      current_digit = int.from_bytes(f_labels.read(1), byteorder='big')
      digits[0][current_digit] = 1
      output_layer.backprop(expected_output=digits)
      digits[0][current_digit] = 0
      if((k+1) % 1000 == 0):
          print(str(k+1) + " / " + str(images_number))
  f_images.close()
  f_labels.close()
  
  f_images = open("t10k-images.idx3-ubyte", "br")
  f_images.seek(4)
  f_labels = open("t10k-labels.idx1-ubyte", "br")
  f_labels.seek(8)
  images_number = int.from_bytes(f_images.read(4), byteorder='big')
  rows_number = int.from_bytes(f_images.read(4), byteorder='big')
  cols_number = int.from_bytes(f_images.read(4), byteorder='big')
  
  print("\r\nTesting:")
  correct = 0
  for k in range(images_number):
      for i in range(rows_number):
          for j in range(cols_number):
              input_image[0][i*cols_number+j] = int.from_bytes(f_images.read(1), byteorder='big')
      input_layer.data = input_image
      input_layer.forward()
      current_digit = int.from_bytes(f_labels.read(1), byteorder='big')
      if np.argmax(output_layer.data[0]) == current_digit:
          correct += 1
      if((k+1) % 1000 == 0):
          print(str(k+1) + " / " + str(images_number))
  
  print("\r\nCorrect: " + str(correct) + " / " + str(images_number))
  f_images.close()
  f_labels.close()
  
  
  ---
  
  