 
 (defn roundtrip-edn [proc edn encoding]
   (try
-    (let [transit-out (write-transit edn encoding)
-          result (roundtrip-transit proc transit-out encoding)]
-      (if (not (or (= (:status result) :error)
-                   (= edn (:data-actual result) (:data-expected result))))
-        (assoc result :status :warning :edn edn)
-        result))
+    (let [transit-out (write-transit edn encoding)]
+      (roundtrip-transit proc transit-out encoding))
     (catch Throwable e
       (if (= (-> e ex-data :status) :timeout)
         (throw (ex-info "Response timeout" (assoc (ex-data e) :edn edn)))
         (throw e)))))
 
+(defn test-transit [transits proc encoding]
+  (mapv #(roundtrip-transit proc % encoding) transits))
+
+(defn test-edn [forms proc encoding]
+  (mapv #(roundtrip-edn proc % encoding) forms))
+
+(defn test-timing [transits proc encoding]
+  (println "collecting timing information...")
+  (dotimes [x 10000]
+    (mapv #(roundtrip-transit proc % encoding) transits))
+  (let [start (System/currentTimeMillis)]
+    (mapv #(roundtrip-transit proc % encoding) transits)
+    (- (System/currentTimeMillis) start)))
+
 (def extension {:json ".json"
                 :msgpack ".mp"})
 
-(defn test-exemplar-files [proc encoding]
-  (let [files (filter #(and (.isFile %) (.endsWith (.getName %) (extension encoding)))
-                      (file-seq (io/file "../transit/simple-examples")))]
-    (mapv #(roundtrip-transit proc (read-bytes %) encoding)
-          files)))
-
-(defn test-corners-of-edn [proc encoding]
-  (mapv #(roundtrip-edn proc % encoding) cc/forms))
-
-(defn test-corners-of-transit-json [proc encoding]
-  (mapv #(roundtrip-transit proc (.getBytes %) encoding) cc/transit-json))
-
-(defn test-generated-edn [forms proc encoding]
-  (mapv #(roundtrip-edn proc % encoding) forms))
+(defn exemplar-transit [encoding]
+  (map #(read-bytes %)
+       (filter #(and (.isFile %) (.endsWith (.getName %) (extension encoding)))
+               (file-seq (io/file "../transit/simple-examples")))))
+
+(defn filter-tests [proc encoding opts]
+  (let [transit-exemplars (exemplar-transit encoding)]
+    (filter #((:pred %) proc encoding opts)
+            [{:pred (constantly true)
+              :path [:tests :exemplar-file]
+              :test #(test-transit transit-exemplars proc encoding)}
+             {:pred (constantly true)
+              :path [:tests :corner-case-edn]
+              :test #(test-edn cc/forms proc encoding)}
+             {:pred (fn [_ e _] (= e :json))
+              :path [:tests :corner-case-transit-json]
+              :test #(test-transit (map (fn [s] (.getBytes s)) cc/transit-json) proc encoding)}
+             {:pred (fn [_ _ o] (:gen o))
+              :path [:tests :generated-edn]
+              :test #(test-edn (:generated-forms opts) proc encoding)}
+             {:pred (fn [_ _ o] (:time o))
+              :path [:time]
+              :test #(let [ms (test-timing transit-exemplars proc encoding)]
+                       {:ms ms
+                        :count (count transit-exemplars)
+                        :encoding encoding})}])))
 
 (defn verify-impl-encoding [command encoding opts]
   (assert (contains? extension encoding)

      ::read-error)))

(defn start-process [command encoding]
  ;; TODO: what if there is execption here? need to report that we
  ;; couldn't start the process?
  ;; TODO: how do we communicate that the encoding is not supported?
  (let [p (.start (ProcessBuilder. [command (name encoding)]))
        out (BufferedOutputStream. (.getOutputStream p))
        in (BufferedInputStream. (.getInputStream p))]
    {:out out :in in :p p}))

(defn stop-process [proc]
  (try
    (.write (:out proc) 3) ;; send Ctrl+C
    (.flush (:out proc))
    (.destroy (:p proc))
    (catch Throwable _
      ;; TODO: report this?
      )))

(defn write-to-stream [proc transit-data]
  #_(println "Send bytes:")
  #_(println (Hex/encodeHexString transit-data))
  (.write (:out proc) transit-data 0 (count transit-data))
  (.flush (:out proc)))

(defn read-from-stream [proc]
  (let [bytes (ByteArrayOutputStream.)]
    (loop [state `(:wait)
           counter 0
           prev nil]
      (let [b (.read (:in proc))]
        (if (== b -1)
          (.toByteArray bytes)
          (let [c (char b)]
            (case (first state)
              :wait (cond (= c \[) (do (.write bytes b)
                                       (recur '(:array) (inc counter) c))
                          (= c \{) (do (.write bytes b)
                                       (recur '(:object) (inc counter) c))
                          :else (recur state counter c))
              :string (do (.write bytes b)
                          (cond (and (= prev \\) (= c \\))
                                ;; set prev to non-escape char
                                (recur state counter \space)
                                (and (not= prev \\) (= c \"))
                                (recur (rest state) counter c)
                                :else (recur state counter c)))
              :array (do (.write bytes b)
                         (cond (and (not= prev \\) (= c \"))
                               (recur (conj state :string) counter c)
                               (and (= c \[))
                               (recur state (inc counter) c)
                               (and (= c \]) (pos? (dec counter)))
                               (recur state (dec counter) c)
                               (= c \])
                               (.toByteArray bytes)
                               :else (recur state counter c)))
              :object (do (.write bytes b)
                          (cond (and (not= prev \\) (= c \"))
                                (recur (conj state :string) counter c)
                                (and (= c \{))
                                (recur state (inc counter) c)
                                (and (= c \}) (pos? (dec counter)))
                                (recur state (dec counter) c)
                                (= c \})
                                (.toByteArray bytes)
                                :else (recur state counter c))))))))))

(defn read-response [proc transit-out timeout-ms]
  (let [f (future (read-from-stream proc))]
    (let [response (deref f timeout-ms ::timeout)]
      (if (= response ::timeout)
        (throw (ex-info "Response timeout" {:status :timeout :p proc :transit transit-out}))

;; Copyright (c) Cognitect, Inc.
;; All rights reserved.

(ns transit.verify
  "Provides tools for testing all transit implementations at
  once. Tests each implementaion with several sources of data:
  exemplar files, problem edn data, problem transit data and generated
  data. In addition, it can capture comparative timing results.

  From the REPL, the main entry point is `verify-all` which takes an
  options map as an argument. With an empty map it will test each
  encoding for all implementations located in sibling project
  directories which have a `bin/roundtrip` script. Options can be used
  to control which project is tested, which encoding to test, turn on
  generative testing and collect timing information."
  (:require [transit.read :as r]
            [transit.write :as w]
            [clojure.java.io :as io]
            [transit.generators :as gen]
            [transit.corner-cases :as cc])
  (:import [java.io PrintStream ByteArrayOutputStream ByteArrayInputStream
            BufferedInputStream BufferedOutputStream FileInputStream]))

(def TIMEOUT
  "Timeout for roundtrip requests to native implementation"
  10000)

(defn read-bytes
  "Read the contents of the passed file into a byte array and return
  the byte array."
  [file]
  (assert (.exists file) "file must exist")
  (assert (.isFile file) "file must actually be a file")
  (let [size (.length file)
        bytes (make-array Byte/TYPE size)
        in (BufferedInputStream. (FileInputStream. file))]
    (loop [n (.read in bytes 0 size)
           total 0]
      (if (or (= n -1) (= (+ n total) size))
        bytes
        (let [offset (+ n total)
              size (- size offset)]
          (recur (.read in bytes offset size) offset))))))

(defn write-transit
  "Given an object and an encoding, return a byte array containing the
  encoded value of the object."
  [o encoding]
  (let [out (ByteArrayOutputStream.)
        w (w/writer out encoding)]
    (w/write w o)
    (.toByteArray out)))

(defn read-transit
  "Given a byte array containing an encoded value and the encoding used,
  return the decoded object."
  [bytes encoding]
  (try
    (let [in (ByteArrayInputStream. bytes)
          r (r/reader in encoding)]
