             [org.altlaw.util.jruby :as ruby]
             [org.altlaw.util.context :as context]
             [org.altlaw.util.log :as log]
+            [org.altlaw.util.merge-fields :as merge]
             [org.altlaw.db.docids :as ids]
+            [org.altlaw.extract.pdf :as pdf]
             [clojure.contrib.walk :as walk]
-            [clojure.contrib.singleton :as sing])
+            [clojure.contrib.singleton :as sing]
+            [clojure.set :as set])
   (:use clojure.contrib.json.read
         org.altlaw.util.log)
-  (:import (org.apache.commons.codec.binary Base64)
+  (:import (org.altlaw.extract PROHTMLToText)
+           (org.apache.commons.codec.binary Base64)
            (javax.script ScriptContext ScriptEngine ScriptEngineManager)
            (java.io StringReader)))
 

;;  |                   |                 |
;; handle-html         handle-pdf        warn
;;  |       |           |      |              
;;  |      warn         |     request Docid   
;;  |                   |                     
;; scraper-dispatch    convert-pdf            
;;  |




;;; REDUCER IMPLEMENTATION

(defn merge-docs [docid documents]
  (let [doc (merge/merge-fields documents)]
    (if (and (:html doc) (:text doc))
      [[docid doc]]
      (do (h/counter "No html/text")
          (log/warn "No html/text; skipping " docid)))))

(defn assign-docids [requests]
  (doseq [r requests]
    (ids/handle-docid-request r))
  (ids/save-docids "altcrawl")
  (ids/save-next-docid)
  nil)

(defn my-reduce [key documents]
  (cond
    (= key :docid-request) (assign-docids documents)
    (= key :exception) nil
    (integer? key) (merge-docs key documents)))



;;; MAPPER METHODS

(defn mapper-configure [this job]
