 (defn train-networks
   []
   (let [opt-fns {:adam opt/adam}
-        network-fns {:linear create-linear
-                     :logistic create-logistic
-                     :linear-dropout-full create-dropout-full
-                     :logistic-dropout-full create-dropout-logistic-full
+        network-fns {
+                     ;; :logistic create-logistic
+                     ;; :logistic-dropout-full create-dropout-logistic-full
+                     :multiple-target-dropout create-multi-target-dropout
                      }
         network-opts (for [opt-fn opt-fns network network-fns]
                        [network opt-fn])]
-    (resource/with-resource-context
-      (try
-       (with-bindings {#'cudnn/*cudnn-datatype* (float 0.0)}
-         (let [train-config (gpu-train/upload-datasets
-                             {:dataset [@training-data @training-data]
-                              :cv-dataset [@test-data @test-data]})]
-           ;;Until we start to make sense of cuda streams, there is no benefit
-           ;;of a pmap here and I was getting crashes.
-           (doall (map #(train-network % train-config) network-opts))))
-       (catch Exception e (do (println "caught error: ") (println  e) (throw e)))))
-    (map (fn [[key val]]
-           [key (:score val)]) @networks)))
+    (try
+      (with-bindings {#'cudnn/*cudnn-datatype* (float 0.0)}
+        ;;Until we start to make sense of cuda streams, there is no benefit
+        ;;of a pmap here and I was getting crashes.
+        (doall (map #(train-network %) network-opts)))
+      (catch Exception e (do (println "caught error: ") (println  e) (throw e))))
+    (keys @networks)))
 
 
 (defn check-l2-constraint

  (let [network-desc [(desc/input 28 28 1)
                      (desc/linear->logistic 144)
                      (desc/linear 784)]]
    (build-and-create-network network-desc)))




                      (desc/linear->logistic 144 :l2-max-constraint 2.0)
                      (desc/dropout 0.5)
                      (desc/linear 784)]]
    (single-target-network network-desc)))

(defn create-multi-target-dropout
  []
  (let [network-desc [(desc/input 28 28 1)
                      (desc/dropout 0.8)
                      (desc/linear->logistic 144 :l2-max-constraint 2.0)
                      (desc/dropout 0.5)
                      ;;Note carefully the order of the leaves of the network.  There
                      ;;is currently an implicit dependency here on that order, the order
                      ;;of the loss functions and the order of the training and test
                      ;;labels which is probably an argument to specify all of that
                      ;;in the network description
                      (desc/split [[(desc/linear 784)] [(desc/linear->softmax 10)]])]
        labels {:training [@training-data @training-labels]
                :test [@test-data @test-labels]}
        loss-fn [(opt/mse-loss) (opt/softmax-loss)]]
    {:network-desc network-desc
     :labels labels
     :loss-fn loss-fn}))


(defn train-and-evaluate
  [network-params optimiser]
  (resource/with-resource-context
   (let [n-epochs 20
         batch-size 10
         {:keys [network-desc loss-fn labels]} network-params
         network (build-and-create-network network-desc)
         training-labels (get labels :training)
         network (gpu-train/train network optimiser loss-fn @training-data training-labels
                                  batch-size n-epochs)]
     {:network (vec (flatten (desc/network->description network)))})))


(defonce networks (atom nil))


(defn train-network
  [[network-entry opt-entry]]
  (let [[net-name network-fn] network-entry
        [opt-name opt-fn] opt-entry
        _ (println "training" net-name)
        network-and-score (train-and-evaluate (network-fn) (opt-fn))]
    (swap! networks assoc [net-name opt-name]
           network-and-score)))

