                        (catch NullPointerException e nil))]
                  (do
                    (recur (concat (rest url-queue) new-bodies)
-                          (clojure.set/union visited (set [url]))
+                          (clojure.set/union visited
+                                             (set [url])
+                                             (set new-visited))
                           lookahead
                           leaf?
                           extract

       (merge-with concat acc {:bodies  (utils/random-take
                                         10
                                         stuff)
                               :visited (utils/random-take
                                         10
                                         hrefs-and-texts)})))
   {}
   xpaths-and-urls))

                 
                  :corpus corpus
                 
                  :prefix (uri/host (uri/uri url))}
                 
                 ;; this is an initial test.
                 ;; I personally prefer performing the
                 ;; leaf test right at the beginning
                 :else)
               (do (utils/sayln :continuing-crawl)
                   (let [state-action (extract body
                                               (first url-queue)
