 (ns org.altlaw.util.crawler
   (:require [org.altlaw.db.download-log :as dl]
+            [org.altlaw.util.log :as log]
             [clojure.contrib.singleton :as sing]
             [clojure.contrib.java-utils :as j])
   (:import (org.restlet Client Context)

  (merge (basic-response-map response)
         (extended-response-map response)))

(defn- execute-request
  "Calls the client to handle the request, logs the URL in the
  download log, and returns the download record as a keyword=>value

  [request]
  (let [response (.handle (crawler-http-client) request)
        result (merge (request-map request)
                      (response-map response))
        code (:response_status_code result)]
    (dl/log-download (:request_uri result))
    (if (#{301 302 303 307} code)
      (handle-redirect result)
      result)))

(defn crawl
  "Given a URL, downloads it and returns a download record as a
