 
 (defn roundtrip-edn [proc edn encoding]
   (try
-    (let [transit-out (write-transit edn encoding)
-          result (roundtrip-transit proc transit-out encoding)]
-      (if (not (or (= (:status result) :error)
-                   (= edn (:data-actual result) (:data-expected result))))
-        (assoc result :status :warning :edn edn)
-        result))
+    (let [transit-out (write-transit edn encoding)]
+      (roundtrip-transit proc transit-out encoding))
     (catch Throwable e
       (if (= (-> e ex-data :status) :timeout)
         (throw (ex-info "Response timeout" (assoc (ex-data e) :edn edn)))
         (throw e)))))
 
+(defn test-transit [transits proc encoding]
+  (mapv #(roundtrip-transit proc % encoding) transits))
+
+(defn test-edn [forms proc encoding]
+  (mapv #(roundtrip-edn proc % encoding) forms))
+
+(defn test-timing [transits proc encoding]
+  (println "collecting timing information...")
+  (dotimes [x 10000]
+    (mapv #(roundtrip-transit proc % encoding) transits))
+  (let [start (System/currentTimeMillis)]
+    (mapv #(roundtrip-transit proc % encoding) transits)
+    (- (System/currentTimeMillis) start)))
+
 (def extension {:json ".json"
                 :msgpack ".mp"})
 
-(defn test-exemplar-files [proc encoding]
-  (let [files (filter #(and (.isFile %) (.endsWith (.getName %) (extension encoding)))
-                      (file-seq (io/file "../transit/simple-examples")))]
-    (mapv #(roundtrip-transit proc (read-bytes %) encoding)
-          files)))
-
-(defn test-corners-of-edn [proc encoding]
-  (mapv #(roundtrip-edn proc % encoding) cc/forms))
-
-(defn test-corners-of-transit-json [proc encoding]
-  (mapv #(roundtrip-transit proc (.getBytes %) encoding) cc/transit-json))
-
-(defn test-generated-edn [forms proc encoding]
-  (mapv #(roundtrip-edn proc % encoding) forms))
+(defn exemplar-transit [encoding]
+  (map #(read-bytes %)
+       (filter #(and (.isFile %) (.endsWith (.getName %) (extension encoding)))
+               (file-seq (io/file "../transit/simple-examples")))))
+
+(defn filter-tests [proc encoding opts]
+  (let [transit-exemplars (exemplar-transit encoding)]
+    (filter #((:pred %) proc encoding opts)
+            [{:pred (constantly true)
+              :path [:tests :exemplar-file]
+              :test #(test-transit transit-exemplars proc encoding)}
+             {:pred (constantly true)
+              :path [:tests :corner-case-edn]
+              :test #(test-edn cc/forms proc encoding)}
+             {:pred (fn [_ e _] (= e :json))
+              :path [:tests :corner-case-transit-json]
+              :test #(test-transit (map (fn [s] (.getBytes s)) cc/transit-json) proc encoding)}
+             {:pred (fn [_ _ o] (:gen o))
+              :path [:tests :generated-edn]
+              :test #(test-edn (:generated-forms opts) proc encoding)}
+             {:pred (fn [_ _ o] (:time o))
+              :path [:time]
+              :test #(let [ms (test-timing transit-exemplars proc encoding)]
+                       {:ms ms
+                        :count (count transit-exemplars)
+                        :encoding encoding})}])))
 
 (defn verify-impl-encoding [command encoding opts]
   (assert (contains? extension encoding)

              "-impls" (assoc a :impls (set (mapv #(str "transit-" %) v)))
              "-enc" (assoc a :enc (keyword (first v)))
              "-gen" (assoc a :gen (Integer/valueOf (first v)))
              a))
          {}
          (partition 2 (partition-by #(.startsWith % "-") args))))

(defn -main [& args]
  (binding [*color* true]
    (verify-all (read-options args))
    (shutdown-agents)))

        results {:command command
                 :encoding encoding}]
    (try
      (let [tests (filter-tests proc encoding opts)
            results (reduce (fn [r {:keys [path test]}]
                              (assoc-in r path (test)))
                            results
                            tests)]
        (stop-process proc)
        results)
      (catch Throwable e
        (stop-process proc)
        (merge results {:exception e})))))

(declare report)

(defn- run-test [project encoding opts]
  (println "testing" project "...")
  (let [command (str "../" project "/bin/roundtrip")]
    (if (= encoding :msgpack)
      (println "msgpack tests are disabled until we have a working implementation.")
