 
        {:model model
         :model-runner model-runner
-        :modeled-files (-> model-runner (.model model test-set) pairlist->map)}
+        ;:modeled-files (-> model-runner (.model model test-set) pairlist->map)
+        }
        ))))
 
 ;(def dir-path (expand-home "~/atom-finder/src/java"))

(def dir-path (expand-home "~/opt/src/atom-finder"))
(def dir-file (->> dir-path java.io.File.))

(def ngram-res (time-mins (ngram-java dir-path)))

(def statistics (-> ngram-res :model-runner (.getStats (-> ngram-res :modeled-files))))

(printf "Modeled %d tokens, average entropy:\t%.4f\n" (.getCount statistics) (.getAverage statistics))

(defn random-split [amount lst]
  (let [shuffled-lst (shuffle lst)]

        model-runner (init-modeler lexer :self-test true)]
    (->> dir-file
         (.model model-runner model)
         pairlist->seq
         (mapcat (fn [[file problist]]
                   (map-indexed (fn [idx probs]
                                  {:file (-<>> file .getPath (str/replace <> dir-path ""))
                                   :line (inc idx)
                                   :probability probs})
                                problist)))
         )))

(defmacro ignore-output
  [& block]
  `(with-open [w# (clojure.java.io/writer "/dev/null")]
    (binding [*out* w#, *err* w#]
      ~@block)))

(->> ngram-res :model
     model-all-lines
     (maps-to-csv "atom-finder-token-probabilities.csv")
     ignore-output)

     '((map (fn [[file line]]
            (print-line-context 0 (str dir-path "/" file) line :top-border false)
            (println "\n")))
     time-mins
