     (for [fitness-case (get @penn-fitness-cases fitness-set)]
       (let [inputs (:inputs fitness-case)
             target (:target fitness-case)
-            state (run-push program
-                            #_(loop [to-push (reverse inputs)
-                                   with-inputs (make-push-state)]
-                              (if (empty? to-push)
-                                with-inputs
-                                (recur (rest to-push)
-                                       (push-item (first to-push)
-                                                  :input
-                                                  with-inputs))))
-                            (assoc (make-push-state) :input inputs))
-        ;    top-string (top-item :string state)]
-        ;(if (string? top-string)
-        ;  (if (= top-string target) 0 1)
-        ;  10000)))))
-            [most second-most] (take 2 (reverse (sort-by val (frequencies (:string state)))))
+            push-state (run-push program (assoc (make-push-state) :input inputs))
+            [most second-most] (take 2 (reverse (sort-by val (frequencies (:string push-state)))))
             answer (if (or (not most)
                            (= (second most) (second second-most)))
                      nil
                      (first most))]
         (if (= answer target) 0 1)))))
             
+(defn rmse
+  "Returns the root of the mean square error for use in error reporting."
+  [errors]
+  (sqrt (/ (apply + (map #(* % %) errors))
+           (count errors))))            
 
 (defn penn-report
-  "Customized generational report."
+  "Customized generational report for the penn problem, which also resamples (and prints) 
+  the training cases if appropriate."
   [best population generation error-function report-simplifications]
   (let [best-program (not-lazy (:program best))
         best-test-errors (penn-error-function :test best-program)]

  (:require [clojure.string :as string]
            [clojure-csv.core :as csv]))

(defn rmse
  "Returns the root of the mean square error for use in error reporting."
  [errors]
  (sqrt (/ (apply + (map #(* % %)
                         errors))
           (count errors))))

(defn read-data []
  "Reads data into a sequence of sequences."
  (let [f (slurp* (str "src/clojush/problems/classification/data/"  ;; todo?: allow command-line arg
                       ;"GAMETES_Epistasis_2-Way_20atts_0.4H_EDM-1_1.txt"
                       ;"a_5000s_2000her_0.1__maf_0.2_EDM-1_01.txt"
                       "xor_2_a_20s_1600_EDM-1_01.txt"
                       ;"xor_3_a_20s_1600_EDM-1_01.txt"
                       ;"xor_4_a_20s_1600_EDM-1_01.txt"
                       ))
        lines (csv/parse-csv f :delimiter \tab)]
    (println "Total number of data lines:" (count lines))
    (mapv #(mapv read-string %) lines)))

(def training-proportion 0.1) ;; proportion of training cases to use each generation

(def resampling true)

(defn define-fitness-cases
  "Returns a map with two keys: train and test. Train maps to a

   :evalpush-limit 1600
   :population-size 1000
   :max-generations 300
   :parent-selection :leaky-lexicase ;:lexicase
   :lexicase-leakage 0.5
   :genetic-operator-probabilities {:uniform-addition-and-deletion 0.5
                                    :alternation 0.5}
   :uniform-addition-and-deletion-rate [0.001 0.01 0.1]
   :alternation-rate [0.001 0.01 0.1]
   :alignment-deviation [0 1 10 100]
   :problem-specific-report penn-report
   :problem-specific-initial-report penn-initial-report
   ;:print-behavioral-diversity true ;; requires maintaining @population-behaviors 
   :report-simplifications 0
   :final-report-simplifications 1000
   ;; === parameters for experimenting with age-mediated parent selection (AMPS)
   ;:age-mediated-parent-selection [0.05 0.5]
   ;:age-combining-function :max
   ;:age-combining-function :proportionate 
   ;:age-combining-function :first-reuse 
   ;; === parameters for experimenting with autoconstruction
   ;:autoconstructive true
   ;:autoconstructive-genome-instructions :uniform
   ;:autoconstructive-diversification-test :three-gens-size-and-instruction
