 
 
 
+;;; REDUCER IMPLEMENTATION
+
+(defn merge-docs [docid documents]
+  (let [doc (merge/merge-fields documents)]
+    (if (and (:html doc) (:text doc))
+      [[docid doc]]
+      (do (h/counter "No html/text")
+          (log/warn "No html/text; skipping " docid)))))
+
+(defn assign-docids [requests]
+  (doseq [r requests]
+    (ids/handle-docid-request r))
+  (ids/save-docids "altcrawl")
+  (ids/save-next-docid)
+  nil)
+
+(defn my-reduce [key documents]
+  (cond
+    (= key :docid-request) (assign-docids documents)
+    (= key :exception) nil
+    (integer? key) (merge-docs key documents)))
+
+
+
 ;;; MAPPER METHODS
 
 (defn mapper-configure [this job]

;;  |                   |                 |
;; handle-html         handle-pdf        warn
;;  |       |           |      |              
;;  |      warn         |     request Docid   
;;  |                   |                     
;; scraper-dispatch    convert-pdf            
;;  |

            [org.altlaw.util.jruby :as ruby]
            [org.altlaw.util.context :as context]
            [org.altlaw.util.log :as log]
            [org.altlaw.util.merge-fields :as merge]
            [org.altlaw.db.docids :as ids]
            [org.altlaw.extract.pdf :as pdf]
            [clojure.contrib.walk :as walk]
            [clojure.contrib.singleton :as sing]
            [clojure.set :as set])
  (:use clojure.contrib.json.read
        org.altlaw.util.log)
  (:import (org.altlaw.extract PROHTMLToText)
           (org.apache.commons.codec.binary Base64)
           (javax.script ScriptContext ScriptEngine ScriptEngineManager)
           (java.io StringReader)))

