 (ns attercop.spider
-  (:require [org.httpkit.client :as http]
+  (:require [clojure.string :as s]
+            [org.httpkit.client :as http]
             [clojure.core.async
              :refer [chan >! >!! <!! go thread close! timeout alts!]]
             [clojurewerkz.urly.core :as urly]

(defn normalize-href
  [referrer href]
  (urly/absolutize href referrer))


(defn allowed-domain?

               (let [html-nodes (enlive-utils/html->nodes html)
                     resp (assoc resp :html-nodes html-nodes)
                     links (when (follow? url)
                             (->> (extract-hrefs html-nodes)
                                  (map (partial normalize-href url))
                                  (remove skip?)))]
                 (doseq [link links]
