             [me.raynes.fs :as fs]
             [schema.core :as s]
             [schema.macros :as sm]
+            [corpus-utils.text :as text]
             [corpus-utils.document :refer [DocumentSchema]]))
 
 ;; :sentence -> :s

       (into [])
       #_((fn [a] (doto a println)))))

(sm/defn parse :- [DocumentSchema]
  "Each XML file from The Sun corpus contains several articles, so we return a vector of documents."
  [filename :- s/Str]
  (let [root-loc (-> filename
                     io/input-stream
                     xml/parse

        [corpus year number] ((juxt :雑誌名 :年 :号) (-> root-loc z/node :attrs ))]
    (->> root-loc
         z/down ;; Descend to the :記事 level.
         (iterate z/right) ;; FIXME broken. Should be a lazyseq of documents
         ;; Documents can be empty...
         (remove nil?)
         (take 5)
         (map (partial parse-document corpus year number filename)))))

(sm/defn document-seq
  [data-dir :- s/Str]
  (->> (fs/glob (str data-dir "/*.xml"))
       (mapcat parse-document-seq)))

;;(parse "/data/taiyo-corpus/XML/t189506.xml")
;;(sm/with-fn-validation (parse "/data/taiyo-corpus/XML/t189506.xml"))
