      :train (vec (take (int (* training-proportion train-num)) (lshuffle all-training-cases)))
      :test all-testing-cases}))
 
-;; Define the fitness cases once per run, so that train and test
-;; subsets stay the same throughout a run.
 (def penn-fitness-cases (atom (define-fitness-cases)))
+
+;; The error function runs the program on the specified subset of the data, returning 
+;; errors of 0 whenever the resulting string stack has the correct answer as its most 
+;; frequent value, and 1 otherwise.
 
 (defn penn-error-function
   "Error function for the penn problem."

    (for [fitness-case (get @penn-fitness-cases fitness-set)]
      (let [inputs (:inputs fitness-case)
            target (:target fitness-case)
            state (run-push program
                            #_(loop [to-push (reverse inputs)
                                   with-inputs (make-push-state)]
                              (if (empty? to-push)
                                with-inputs
                                (recur (rest to-push)
                                       (push-item (first to-push)
                                                  :input
                                                  with-inputs))))
                            (assoc (make-push-state) :input inputs))
        ;    top-string (top-item :string state)]
        ;(if (string? top-string)
        ;  (if (= top-string target) 0 1)
        ;  10000)))))
            [most second-most] (take 2 (reverse (sort-by val (frequencies (:string state)))))
            answer (if (or (not most)
                           (= (second most) (second second-most)))
                     nil
                     (first most))]
        (if (= answer target) 0 1)))))
            

(defn penn-report
  "Customized generational report."
  [best population generation error-function report-simplifications]
  (let [best-program (not-lazy (:program best))
        best-test-errors (penn-error-function :test best-program)]

  (:require [clojure.string :as string]
            [clojure-csv.core :as csv]))

;; This problem file allows for experimentation with Penn Machine Learning Benchmarks problems,
;; or other other problems with data in the Penn ML format.
;;
;; A random 50/50 split of the data will be created each time this is run, with one half of the
;; split used for training and the other half reserved for testing, which is done only for to
;; report on the best (lowest total training error) individual at the end of each generation. 
;;
;; Although 50% of the data in the file is *available* for use in training, this implementation
;; allows one to specify that only some proportion of that data is actually used. In addition,
;; it allows for the specific subset of training that is used to be resampled each generation.
;;
;; Unlike most Clojush problems, this one contains parameters that cannot be set from the command
;; line. This is because they are required for reading in the data, which currently happens when 
;; this file is loaded, before the command-line parameters are interpreted. This means that to 
;; change these aspects of the problem you must run it from the source code (not just including 
;; Clojush as a dependency in another project) and edit/save this file prior to running the 
;; system. Other parameters can still be set from the command line, overriding the parameters
;; specified in the argmap definition below.
;;
;; Access to the input variables could, in principle, be handled in various ways. What is done
;; here is to provide a boolean input for whether each input variable is each of the values
;; that appear in the data. For example, for some of the target data files the variables in
;; the data are all 0, 1, or 2. In this case, evolving programs will have access to three
;; boolean inputs for each input variable. If a particular variable is 0 then these inputs
;; will be true, false, false, while if it is 1 they will be false, true, false, and if it
;; is 2 they will be false, false, true.
;;
;; Similarly, specification of the program output could, in principle, be handled in various
;; ways. What is done here is to provide string literals for each of the values in the "CLASS"
;; (output) column in the data, and to use the :string stack only for the purpose of accumulating
;; instances of these literals, which act as votes. At the end of program execution, whichever
;; literal occurs most frequently on the :string stack is taken to be the output of the program.
;;
;; Some of the values for standard pushgp parameters specified here, in the argmap definition
;; below, are intended to help lineages weather the disruptions caused by generational resampling
;; of the training data. Specifically, we specify the use of leaky lexicase selection with
;; a high leakage value, and the production of some offspring with very low variation rates.

;; Here are the three data-related parameters that can only be set here, in the source code:

;; The data file, which must be in src/clojush/problems/classification/data/:
(def data-file 
  ;"GAMETES_Epistasis_2-Way_20atts_0.4H_EDM-1_1.txt"
  ;"a_5000s_2000her_0.1__maf_0.2_EDM-1_01.txt"
  ;"xor_2_a_20s_1600_EDM-1_01.txt"
  "xor_3_a_20s_1600_EDM-1_01.txt"
  ;"xor_4_a_20s_1600_EDM-1_01.txt"
  )

;; The proportion of the training data that will be used to evaluate individuals each generation:
(def training-proportion 0.2)

;; A flag indicating whether the subset of the training data used for evaluation should be 
;; resampled each generation:
(def resampling true)

;; Here we define functions to read the data and to split it into training and testing sets.
;; We also extract a subset of the full training set to use for evaluation if training-proportion
;; is less than 1, and we store this in an atom so that we can update the training data
;; if resampling is true.

(defn read-data []
  "Reads data into a sequence of sequences."
  (let [f (slurp* (str "src/clojush/problems/classification/data/" data-file))
        lines (csv/parse-csv f :delimiter \tab)]
    (println "Total number of data lines:" (count lines))
    (mapv #(mapv read-string %) lines)))

(defn define-fitness-cases
  "Returns a map with two keys: train and test. Train maps to a
