        (swap!
         *url-documents* utils/atom-merge-with set/union {a-link (set [body])})
        body)))
-
-(defn update-tf
-  [xpaths-tfs]
-  (doseq [[xpath tf] xpaths-tfs]
-    (swap! *xpath-tf* utils/atom-merge-with concat {xpath [tf]})))
-
-(defn record-explore-potential
-  "At first, we just use # of distinct links"
-  [a-record]
-  (-> a-record
-      (records/record-anchors)))
 
 (defn sample
   ([urls host n]

(def *sample-fraction* (/ 1 4))   ; fraction of links to look at
                                  ; before sampling

(def *xpath-hrefs*   (atom {}))   ; hold the unique hrefs for each xpath
(def *xpath-df*      (atom {}))   ; hold the df score for each xpath
(def *xpath-tf*      (atom {}))   ; per-page tf scores
(def *visited*       (atom
                      (set [])))  ; set of visited documents
(def *url-documents* (atom {}))   ; url -> body map

           (recur urls host (dec n) (conj sampled-list sampled))
           (recur urls host 0 sampled-list))))))

(defn novelty
  [a-diff]
  (reduce
