 (ns org.altlaw.jobs.pre.altcrawl
-  (:require [org.altlaw.util.hadoop :as hadoop]
+  (:require [org.altlaw.util.hadoop :as h]
             [org.altlaw.util.files :as files]
             [org.altlaw.util.tsv :as tsv]
-            [clojure.contrib.walk :as walk])
+            [org.altlaw.util.jruby :as ruby]
+            [org.altlaw.util.context :as context]
+            [org.altlaw.util.log :as log]
+            [org.altlaw.db.docids :as ids]
+            [clojure.contrib.walk :as walk]
+            [clojure.contrib.singleton :as sing])
   (:use clojure.contrib.json.read
         org.altlaw.util.log)
   (:import (org.apache.commons.codec.binary Base64)
            (javax.script ScriptContext ScriptEngine ScriptEngineManager)
            (java.io StringReader)))
 
-(hadoop/setup-mapreduce)
+(h/setup-mapreduce)
 
-(declare *reporter* *raw-input*)
-
-(defn counter [group name]
-  (.incrCounter *reporter* group name 1))
-
-(def *ruby* nil)
-
-(def *ruby-context* nil)
-
-(def *docid-map* nil)
-
-(defn get-docid [uri]
-  (get *docid-map* uri))
+(def scraper-handler
+     (sing/per-thread-singleton
+      (fn []
+        (ruby/eval-jruby "require 'org/altlaw/extract/scrape/scraper_handler'")
+        (ruby/eval-jruby "ScraperHandler.new"))))
 
 (defn- decode-response-body [download]
   (assoc download :response_body_bytes

    (files/guess-mime-type-by-content (:response_body_bytes download))))

(defmethod mime-type-dispatch "application/pdf" [download]
  (counter "MIME types" "PDF")
  (debug "mime-type-dispatch got PDF from " (:request_uri download))
  (handle-pdf download))

(defmethod mime-type-dispatch "text/html" [download]
  (counter "MIME types" "HTML")
  (debug "mime-type-dispatch got HTML from " (:request_uri download))
  (run-scrapers download))

(defmethod mime-type-dispatch :default [download]
  (counter "MIME types" "unknown")
  (warn "mime-type-dispatch found no handler for " (:request_uri download)))


;;; Step 1: dispatch on Status code of the download

(defn status-code-dispatch [download]
  (counter "Status codes" (str (:response_status_code download)))
  (if (= 200 (:response_status_code download))
    (mime-type-dispatch download)
    (warn "status-code-dispatch saw "

          " from " (:request_uri download))))


(defn my-map [key download]
  (status-code-dispatch (decode-response-body download)))



;;; MAPPER METHODS

(defn mapper-configure [this job]
  (context/use-hadoop-jobconf job))

(def mapper-map (partial h/json-map my-map))



