 (ns corpus-utils.bccwj
-  (:require ;;[clojure.zip :as z]
-            [fast-zip.core :as fz]
-            [clojure.xml :as xml]
-            [clojure.java.io :as io]
-            [clojure.string :as string]
-            [clojure.core.reducers :as r]
-            ;;[org.httpkit.client :as http]
-            [plumbing.core :refer [for-map map-vals]]
-            [schema.core :as s]
-            [corpus-utils.c-code :refer [c-code]]
-            [corpus-utils.utils :as utils]
-            [corpus-utils.document :refer [MetadataSchema DocumentSchema SentencesSchema]])
+  (:require
+    [fast-zip.core :as fz]
+    [clojure.xml :as xml]
+    [clojure.java.io :as io]
+    [clojure.string :as string]
+    [clojure.core.reducers :as r]
+    ;;[org.httpkit.client :as http]
+    [plumbing.core :refer [for-map map-vals]]
+    [clojure.spec.alpha :as s]
+    [corpus-utils.ndc :as ndc]
+    [corpus-utils.c-code :refer [c-code]]
+    [corpus-utils.utils :as utils]
+    [corpus-utils.document]
+    [me.raynes.fs :as fs])
   (:import [fast_zip.core ZipperLocation]
-           [java.util.zip ZipFile ZipEntry]))
+           [java.io File InputStream]
+           [org.apache.commons.compress.archivers.zip ZipFile ZipArchiveEntry]))
 
 ;; # Importer for BCCWJ-Formatted C-XML Data
 ;;
 ;; Imports C-XML-type data.
 ;; M-XML is also supported, but is not recommended as we do our own parsing.
 
-
 ;; ## Utility functions
-;; TODO integration
-(s/defn walk-zip-files
-  [zip-name :- java.io.File]
-  (let [z (ZipFile. zip-name)]
-    (for [e (enumeration-seq (.entries z))
-          :when (not (.isDirectory ^ZipEntry e))]
-      (.getInputStream z e))))
+(defn walk-zip-file
+  [apply-fn path]
+  (with-open [z (ZipFile. (fs/file path))]
+    (let [files (enumeration-seq (.getEntries z))]
+      (doall (for [file files
+                   :let [filename (.getName ^ZipArchiveEntry file)]
+                   :when (and (not (.isDirectory file)) (= ".xml" (string/lower-case (fs/extension filename))))]
+               (apply-fn (fs/base-name filename true) (.getInputStream z file)))))))
+
+(s/fdef walk-zip-file
+  :args (s/cat :zip-file #(instance? File %))
+  :ret (s/coll-of #(instance? InputStream %)))
 
 ;; ## XML Tags
 ;;

                            :same (if (= :br tag) tag-stack (conj (pop tag-stack) coded-tag)) ; :br is an exception as a paragraph break, as it does not increase XML tree depth
                            ((apply comp (repeat next-direction pop)) tag-stack))] ; Discard equal to up depth.
        (recur
         (or xml-loc-down xml-loc-right (:loc xml-loc-up)) ; Same as (fz/next xml-loc).
         new-tag-stack
         (cond (paragraph-level-tags tag) ; Insert new paragraph, inserting the new tag stack.
               (-> par-loc (fz/insert-right {:tags new-tag-stack :sentences [""]}) fz/right)

               (= :sentence tag) ; Insert new sentence.
               (let [tag-stack (-> par-loc fz/node :tag-stack)]
                 (fz/edit par-loc update-in [:sentences] conj ""))

               (string? xml-node) ; Update last-inserted sentence's text.
               (fz/edit par-loc update-in [:sentences (-> par-loc fz/node :sentences count dec)] #(str % xml-node))

               :else par-loc)))))) ; Do nothing.

;; ## Metadata
;;

;;      {:tags #{ ... },
;;       :sentences [ ... ]}]

(defn backtrack-with-distance
  "Modified from `fast-zip.core/next` source. Like zip/next, but also keeps track of how far up the tree it goes."
  [loc]
  (loop [p loc
         depth 0]
    (if (and (not (identical? :end (.path p))) (fz/up p))
      (or (if-let [r (fz/right (fz/up p))] {:loc r :depth (inc depth)}) (recur (fz/up p) (inc depth)))
      {:loc   (ZipperLocation. (.ops loc) (.node loc) :end)
       :depth depth})))

(s/def ::loc #(instance? ZipperLocation %))
(s/def ::depth int?)
(s/fdef backtrack-with-distance
  :args (s/cat :loc ::loc)
  :ret (s/keys :req-un [::loc ::depth]))

;; FIXME break into emitter and consume-sequence-and-build-sentences functions; naming: next-direction conflates direction and depth

(defn walk-and-emit
  "Traverses xml-data (parsed with clojure.xml/parse or clojure.data.xml/parse) using a zipper and incrementally builds up and returns the document as a vector of maps (representing paragraphs), each element of which contains tags and a vector of sentences."
  [paragraph-level-tags xml-data]
  (loop [xml-loc (fz/xml-zip xml-data)
         tag-stack []
         par-loc (fz/down (fz/vector-zip [{:tags [] :sentences []}]))]
