     [clojure.core.reducers :as r]
     ;;[org.httpkit.client :as http]
     [net.cgrand.xforms :as x]
+    [parallel.xf :as xf]
     [clojure.spec.alpha :as s]
     [corpus-utils.ndc :as ndc]
     [corpus-utils.c-code :refer [c-code]]

                 [basename _ title subtitle _ _ publisher year isbn _
                  genre-1 genre-2 genre-3 genre-4 _ _
                  author author-year author-gender corpus-name]]
              (let [metadata-patch? (if-let [v (get ndc-metadata basename)] v false)
                    [genre-1 genre-2 genre-3 genre-4] (if metadata-patch? metadata-patch? [genre-1 genre-2 genre-3 genre-4])
                    category (->> (condp re-seq corpus-name
                                    #"(L|O|P)B" (->> [genre-1 genre-2]

                      xml/parse
                      (walk-and-emit p-tags))]
    (into []
          (comp (map (fn [doc]                              ; Remove nils/empty strings from :tags and :sentences.
                       {:paragraph/tags      (into #{} (filter identity) (:paragraph/tags doc))
                        :paragraph/sentences (into [] (remove empty?) (:paragraph/sentences doc))}))
                (remove #(empty? (:paragraph/sentences %)))) ; Remove paragraphs with no sentences.
          document)))

(s/fdef parse-document
  :args (s/cat :stream any? #_#(instance? InputStream %) :corpus string?)
  :ret :document/paragraphs)

;; ## Metadata
