     (.perLine false)
     (.selfTesting self-test)))
 
-(defn ngram-java
-  ([train-path] (ngram-java train-path train-path))
+(defn ngram-dir
+  ([train-path] (ngram-dir train-path train-path))
   ([train-path test-path]
    (let [train-set (java.io.File. (expand-home train-path))
          test-set (java.io.File. (expand-home test-path))

        model-runner (init-modeler lexer :self-test true)]
    (->> dir-file
         (.model model-runner model)
         pairlist->map
         (take 10)
         (mapcat (fn [[file problist]]
                   (map-indexed (fn [idx probs]
                                  {:file (-<>> file .getPath (str/replace <> dir-path ""))
                                   :line (inc idx)
                                   :probability probs})
                                problist)))
         (maps-to-csv "atom-finder-token-probabilities.csv"))))

(->> ngram-res :model
     model-all-lines
     (map (fn [[file line]]
            (print-line-context 0 (str dir-path "/" file) line :top-border false)
            (println "\n")))
     time-mins

            ))

(defn pair->vec [p] [(.left p) (.right p)])
(def pairlist->seq (%->> .iterator iterator-seq (map pair->vec)))
(def pairlist->map (%->> pairlist->seq (into {})))

(defn init-java-lexer []
  (doto (LexerRunner. (JavaLexer.))
