       activation)))
 
 ;; Gradient descent
-
+;; 1) forward propagate
+;;  * sum((weights * inputs) + biases) for each neuron, layer by layer
+;; 2) back propagate deltas
+;;  * After propagating forward, we need to figure out the error (often called the
+;;    delta) for each neuron.  For the outputs this is just the
+;;    (output - expected_output) * activation-fn-prime, but for the hidden units the output error has to
+;;    be propagated back across the weights to distributed the error according to
+;;    which hidden units were responsible for it.
+;; 3) compute gradients
+;;  * Then for each weight you multiply its output delta by its input activation to get
+;;    its gradient.  This will correspond to the direction and magnitude of the
+;;    error for that weight, so any update should happen in the opposite direction.
+;; 4) update weights
+;;  * multiply gradient by the learning-rate to smooth out jitter in updates.
 (defn backprop
   [{:keys [biases weights layer-sizes] :as net} input expected-output]
-  (println "backprop")
-  (println "input: " input " => " expected-output)
-  (println "\nforward -----------------------------")
+  ;(println "\n--------------- backprop ---------------------")
   (let [bias-gradients (map #(mat/zero-array (mat/shape %)) biases)
         weight-gradients (map #(mat/zero-array (mat/shape %)) weights)
         [activations zs] (reduce

          (fn [[bias-gradients weight-gradients] sample-index]
            (let [input (mat/get-row data sample-index)
                  label (mat/get-row labels sample-index)
                  [bias-deltas weight-deltas] (backprop net input label)
                  bias-gradients (map mat/add! bias-gradients bias-deltas)
                  weight-gradients (map mat/add! weight-gradients weight-deltas)]
              [bias-gradients weight-gradients]))
          [bias-gradients weight-gradients]
          batch)
        biases (map (fn [b nb] (mat/sub! b (mat/mul batch-rate nb))) biases bias-gradients)
        weights (map (fn [w nw] (mat/sub! w (mat/mul batch-rate nw))) weights weight-gradients)]
    (println "biases:\n" biases)
    (println "weights:\n" weights)
    (assoc net :biases biases :weights weights)))

(defn sgd

  [net test-data test-labels]
  (let [results (map (fn [data label]
                       (let [res (feed-forward net data)]
                         ;(argmax res)
                         [(Math/round (first res))]
                         ))
                     (mat/rows test-data) (mat/rows test-labels))
        _ (println "RESULTS: " results)
        _ (println "LABELS: " test-labels)
        score (count (filter #(= (first %) (second %)) (map vector results test-labels)))]
    score))

