 (ns crawler.corpus
   "Tools for processing a downloaded corpus"
   (:require [cheshire.core :as json]
-            [clojure.java.io :as io])
-  (:use [clojure.pprint :only [pprint]]
+            [clojure.java.io :as io]
+            [crawler.dom :as dom])
+  (:use [clj-xpath.core :only [$x:node+]]
+        [clojure.pprint :only [pprint]]
         [structural-similarity.xpath-text :only [similar?]])
   (:import [java.io PushbackReader]))
 

          :candidate? (pagination-candidate? src-body tgt-body)}))
     corpus))))

(defn pagination-in-corpus
  [corpus]
  (let [candidates (-> corpus pagination-candidates frequencies)

                             (:src-xpath c))))
                  sorted-candidates)))
         sorted-candidates)]
    (scored-pagination spurious-candidates-removed corpus)))

(defn pagination-in-corpus-file
  [a-corpus-file]
