 ;;      {:tags #{ ... },
 ;;       :sentences [ ... ]}]
 
-(s/defn backtrack-with-distance :- {:loc ZipperLocation :depth s/Num}
+(defn backtrack-with-distance
   "Modified from `fast-zip.core/next` source. Like zip/next, but also keeps track of how far up the tree it goes."
-  [loc :- ZipperLocation]
+  [loc]
   (loop [p loc
          depth 0]
     (if (and (not (identical? :end (.path p))) (fz/up p))
       (or (if-let [r (fz/right (fz/up p))] {:loc r :depth (inc depth)}) (recur (fz/up p) (inc depth)))
-      {:loc (ZipperLocation. (.ops loc) (.node loc) :end)
+      {:loc   (ZipperLocation. (.ops loc) (.node loc) :end)
        :depth depth})))
 
+(s/def ::loc #(instance? ZipperLocation %))
+(s/def ::depth int?)
+(s/fdef backtrack-with-distance
+  :args (s/cat :loc ::loc)
+  :ret (s/keys :req-un [::loc ::depth]))
+
 ;; FIXME break into emitter and consume-sequence-and-build-sentences functions; naming: next-direction conflates direction and depth
 
-(s/defn walk-and-emit
+(defn walk-and-emit
   "Traverses xml-data (parsed with clojure.xml/parse or clojure.data.xml/parse) using a zipper and incrementally builds up and returns the document as a vector of maps (representing paragraphs), each element of which contains tags and a vector of sentences."
-  [xml-data]
+  [paragraph-level-tags xml-data]
   (loop [xml-loc (fz/xml-zip xml-data)
          tag-stack []
          par-loc (fz/down (fz/vector-zip [{:tags [] :sentences []}]))]

                      :_ #_"小説の主人公あるいは語り手の人称(T)"
                      :_ #_"小説の主人公あるいは語り手の人称(N)"

                      :protagonist-personal-pronoun #_"小説の主人公あるいは語り手の人称"
                      :other-addressing #_"小説以外の文章の内容"]
              #_(->> bccwj-meta-annotation-data
                     first
                     rest)]
          (r/fold
           (r/monoid
            (fn [a [basename & record]]
              (assoc a basename
                     (for-map [[k v] (zipmap header record)
                               :when (not= k :_)]
                         k (cond (empty? v) nil

                                 ;; Coerce into Likert-scale Longs.
                                 (and (#{:target-audience :hard-soft :informality :addressing :other-addressing} k)
                                      (re-seq #"^\d+" v))
                                 (Long/parseLong v)

                                 ;; Coerce into binary values.
                                 (#{:forward-or-afterward :dialog
                                     :quotation-type :visual
                                     :db-or-list :archaic
                                     :foreign-language :math-or-code
                                     :legalese :questionable-content
                                     :low-content} k)
                                 (case (subs v 0 1) "1" true "0" false true)

                                 :else v))))
            (fn [] {}))
           (next bccwj-meta-annotation-data)))]

    (->> (merge-with merge bccwj-metadata bccwj-annotations)
         (map-vals (fn [vs] (into {} (r/remove #(nil? (val %)) vs))))
         vals)))

(comment
  (pprint (take 10 (map (comp :category :metadata) (filter #(= ((comp :subcorpus :metadata) %) "PM") (document-seq "/data/BCCWJ-2012-dvd1/DOC/" "/data/BCCWJ-2012-dvd1/C-XML/VARIABLE/")))))
  (def cats (into #{} (map (comp :category :metadata) (document-seq "/data/BCCWJ-2012-dvd1/DOC/" "/data/BCCWJ-2012-dvd1/C-XML/VARIABLE/"))))
  (def c-topics (into #{} (map :topic (mapcat (comp #(remove empty? %) #(filter map? %)) cats))))
  (def c-audiences (into #{} (map :audience (mapcat (comp #(remove empty? %) #(filter map? %)) cats))))
  (def c-medias (into #{} (map :media (mapcat (comp #(remove empty? %) #(filter map? %)) cats))))
  (def ndc-topics (into #{} (mapcat (comp #(remove empty? %) #(remove map? %)) cats)))
  (clojure.set/intersection c-topics ndc-topics)
  ;; How to merge these intersecting labels that are recorded at different levels?
  #{"情報科学" "宗教" "商業" "旅行" "物理学" "総記" "仏教" "キリスト教" "化学" "機械" "哲学" "ドイツ語" "家事" "法律" "数学" "日本語" "教育" "伝記" "社会" "生物学" "水産業"}
)

(s/defn parse-document :- SentencesSchema
  [filename :- s/Str]
  (->> filename
       io/input-stream
       xml/parse
       walk-and-emit
       (map #(hash-map ; Remove nils/empty strings from :tags and :sentences.
              :tags (into #{} (r/filter identity (:tags %)))
              :sentences (into [] (r/remove empty? (:sentences %)))))
       (remove #(empty? (:sentences %))) ; Remove paragraphs with no sentences.
       vec))

(s/defn document-seq :- [DocumentSchema] ;; TODO validate lazy-seq?
  [options :- {:corpus-dir s/Str
               :metadata-dir s/Str
               s/Keyword s/Any}]
  (lazy-seq
   (->> (parse-metadata (:metadata-dir options))
        (map (fn [metadata]
               (let [filename (str (:corpus-dir options) (:subcorpus metadata) "/" (:basename metadata) ".xml")]
                 {:metadata metadata
                  :paragraphs (parse-document filename)}))))))

(comment
  (s/with-fn-validation
   (take 10 (map (comp :category :metadata) (document-seq "/data/BCCWJ-2012-dvd1/DOC/" "/data/BCCWJ-2012-dvd1/C-XML/VARIABLE/")))))

        (println "Failed, exception is " error)
        (println "Async HTTP POST: " status)))))

(defn parse-metadata
  [metadata-dir]
  (let [metadata (utils/read-tsv (str metadata-dir "Joined_info.txt") true)
        fixed-ndc (utils/read-tsv (str metadata-dir "BCCWJ-NDC.txt") true)
        copyright (into {} (utils/read-tsv (str metadata-dir "CopyRight_Annotation.txt") true))
        ndc-map (into {} (utils/read-tsv-URL (io/resource "ndc-3digits.tsv") false))
        c-map (for-map [[k1 v1] (c-code "1")
                        [k2 v2] (c-code "2")
                        [k3 v3] (c-code "34")]
                (str k1 k2 k3) #:metadata{:audience v1 :media v2 :topic v3})
        name-map {"LB" "書籍"
                  "PB" "書籍"
                  "OB" "書籍"
                  "OL" "法律"
                  "OW" "白書"
                  "OM" "国会会議録"
                  "OP" "広報紙"
                  "PN" "新聞"
                  "PM" "雑誌"
                  "OV" "韻文"
                  "OT" "検定教科書"
                  "OY" "Yahoo!ブログ"
                  "OC" "Yahoo!知恵袋"}

        ndc-metadata
        (r/reduce
          (r/monoid
            (fn [m [ndc basename _ title subtitle number-volume _ publisher year isbn _
                    genre-1 genre-2 genre-3 genre-4 _ _
                    author author-year author-gender corpus-name]]
              ;; We replace the second genre field with the more informative ndc field. genre-1 is the top NDC category.
              ;; genre-3 is the c-code, and genre-4 is unused for the books corpus.
              (assoc m basename [genre-1 ndc genre-3 genre-4]))
            (fn [] {}))
          fixed-ndc)

        bccwj-metadata
        (r/reduce
          (r/monoid
            (fn [m
                 [basename _ title subtitle _ _ publisher year isbn _
                  genre-1 genre-2 genre-3 genre-4 _ _
                  author author-year author-gender corpus-name]]
              (let [metadata-patch? (if-let [v (get ndc-metadata basename)] v false)
                    [genre-1 genre-2 genre-3 genre-4] (if metadata-patch? metadata-patch? [genre-1 genre-2 genre-3 genre-4])]
                (assoc m basename
                         #:metadata{:title       (str title (if-not (= "" (string/trim subtitle)) ": " subtitle))
                                    :author      author
                                    :gender      (let [authors (string/split author-gender #"/")]
                                                   (if (every? #(= (first authors) %) authors)
                                                     (case (first authors)
                                                       "男" :male
                                                       "女" :female
                                                       "" nil
                                                       :mixed)))
                                    :author-year (try (Integer/parseInt author-year) (catch Exception e))
                                    :publisher   (if (empty? publisher) "BCCWJ" publisher)
                                    :copyright   (copyright basename)
                                    :audience    (if (= (name-map corpus-name) "書籍") (:metadata/audience (c-map genre-3)))
                                    :media       (if (= (name-map corpus-name) "書籍") (:metadata/media (c-map genre-3)))
                                    :topic       (if (= (name-map corpus-name) "書籍") (:metadata/topic (c-map genre-3)))
                                    :year        (Integer/parseInt year)
                                    :basename    basename
                                    :corpus      "BCCWJ"
                                    :subcorpus   (name-map corpus-name)
                                    :category    (->> (condp re-seq corpus-name
                                                        #"(L|O|P)B" (->> [genre-1 genre-2]
                                                                         (r/remove #(or (= "なし" %)
                                                                                        (= "分類なし" %)))
                                                                         (r/mapcat #(ndc/ndc-map % [%]))
                                                                         (r/map #(string/replace % #"^\d\s" ""))
                                                                         (r/mapcat #(condp re-seq %

                                                                                      #"^(\d+)\.(\d+)$"
                                                                                      (let [[[_ top-level extra] & _] (re-seq #"^(\d+)\.(\d+)$" %)]
                                                                                        ;; We fallback on the top-level NDC concept if no direct match is found.
                                                                                        (get ndc/ndc-map top-level))

                                                                                      #"^[\d\s]+$"
                                                                                      (do (println "Removing unknown category: " basename genre-1 genre-2)
                                                                                          [nil])

                                                                                      [%]))
                                                                         (r/remove nil?))
                                                        #"OL" (r/map #(string/replace % #"\d\d\s" "")
                                                                     [genre-1 genre-2 genre-3 genre-4])
                                                        #"OT" [genre-1 (str "G" (case genre-2
                                                                                  "小" (Integer/parseInt genre-3)
                                                                                  "中" (+ 6 (Integer/parseInt genre-3))
                                                                                  "高" 10))]
                                                        [genre-1 genre-2 genre-3 genre-4])
                                                      (r/remove empty?)
                                                      (r/reduce (fn ;; Remove repeated categories.
                                                                  ([] [])
                                                                  ([a b] (if (= (peek a) b) a (conj a b)))))
                                                      (concat [(name-map corpus-name)])
                                                      (into []))})))
            (fn [] {}))
          metadata)

        bccwj-meta-annotation-data (utils/read-tsv-xz (io/resource "annotations-2013.tsv.xz") false)
        bccwj-annotations
        (let [header [:metadata/forward-or-afterward #_"前書きや後書きか"
                      :metadata/dialog #_"対話系か"
                      :metadata/quotation-type #_"引用系か"
                      :metadata/visual #_"視覚表現多用系（図表・イラスト・写真の多用）か"
                      :metadata/db-or-list #_"データベースやリスト系か"
                      :metadata/archaic #_"明治時代より以前の古い言葉，古い言い回しが多いか"
                      :metadata/foreign-language #_"外国語が多いか"
                      :metadata/math-or-code #_"数式やプログラミング言語などが多いか"
                      :metadata/legalese #_"法律文が多いか"
                      :metadata/questionable-content #_"教育上のぞましくなさそうか"
                      :metadata/low-content #_"そのほか，一定量の「本文」が認めがたいか"

                      :metadata/target-audience #_"対象とする読者"
                      :metadata/hard-soft #_"文章の硬軟"
                      :metadata/informality #_"くだけているか"
                      :metadata/addressing #_"文体(O)"

                      :_ #_"小説の主人公あるいは語り手の人称(H)"
                      :_ #_"小説の主人公あるいは語り手の人称(M)"
