                                 :similarity))
                           explorations)]
     (when (and (seq explorations) (< 0 (count explorations)))
-      (> (/ (apply + similarities) (count explorations)) *page-sim-thresh*))))
+      (try (> (/ (apply + similarities) (count explorations)) *page-sim-thresh*)
+           (catch Exception e nil)))))
 
 (defn enum-candidate-info
   "Uses the information sent and the snapshot info"

           (recur urls host (dec n) (conj sampled-list sampled))
           (recur urls host 0 sampled-list))))))

(defn update-df
  "Each XPath's df score is incremented"
  [xpaths-hrefs]
  (let [xpaths     (map first xpaths-hrefs)
        xpaths-cnt (map (fn [x] {x 1}) xpaths)]
    (doseq [xpath-cnt xpaths-cnt]
      (swap! *xpath-df* utils/atom-merge-with +' xpath-cnt))))

(defn update-hrefs
  "Union of explored hrefs and observed hrefs"
  [xpaths-hrefs]
  (swap!
   *xpath-hrefs*
   utils/atom-merge-with
   set/union
   (into {} xpaths-hrefs)))

(defn novelty
  [a-diff]
  (reduce

(def *sample-fraction* (/ 1 4))   ; fraction of links to look at
                                  ; before sampling

(def *visited*       (atom
                      (set [])))  ; set of visited documents
(def *url-documents* (atom {}))   ; url -> body map
