               mined      (when-not leaf?
                            (rich-char-extractor/filter-content
                             content))
-              mined-links (reverse
-                           (filter
-                            #(and (-> % :url)
-                                  (not (some #{(-> % :url)} (clojure.set/union
-                                                             (set visited)
-                                                             (set [url])
-                                                             (set (map (fn [x] (:url x)) queue))))))
-                            (reduce
-                             concat
-                             (map
-                              (fn [{xpath :xpath score :score hrefs :hrefs text :texts}]
-                                (map
-                                 (fn [h]
-                                   {:url h
-                                    :src-xpath (cons xpath src-xp)
-                                    :src-url url
-                                    :src-nav-num (cons (if body score 0) src-nav-num)})
-                                 hrefs))
-                              mined))))]
+              mined-links (try
+                           (reverse
+                            (filter
+                             #(and (-> % :url)
+                                   (not (some #{(-> % :url)} (clojure.set/union
+                                                              (set visited)
+                                                              (set [url])
+                                                              (set (map (fn [x] (:url x)) queue))))))
+                             (reduce
+                              concat
+                              (map
+                               (fn [{xpath :xpath score :score hrefs :hrefs text :texts}]
+                                 ;; take only a 25 % sample from here.
+                                 (take
+                                  (Math/ceil (/ (count hrefs) 4))
+                                  (map
+                                   (fn [h]
+                                     {:url h
+                                      :src-xpath (cons xpath src-xp)
+                                      :src-url url
+                                      :src-nav-num (cons (if body score 0) src-nav-num)})
+                                   hrefs)))
+                               mined))))
+                           (catch Exception e []))]
           (if (and (not leaf?) (seq mined-links))
             (recur (concat (rest queue)
                            mined-links)

                                                      (set [url])
                                                      (set (map :url queue))))
                              (catch Exception e nil))
              
              score   (cur-nav-fraction body content)
              
              leaf?      (or (not body)
                             (rich-char-extractor/leaf?


  ([queue visited to-eliminate leaf-paths]
     (if (seq queue)
       (do
        (Thread/sleep 2000)
        (println :url (-> queue first :url))
        (println :src-url (-> queue first :src-url))
        (println :src-xpath (-> queue first :src-xpath))
        (println :score-seq (-> queue first :src-nav-num))
        (let [url     (-> queue first :url)
              src-xp  (-> queue first :src-xpath)
              src-nav-num (-> queue first :src-nav-num)
              body    (utils/download-with-cookie url)

              
              ;; ask the rich extractor to sample and extract on this page.
              content    (try (rich-char-extractor/state-action
