                              (:src-xpath c))))
                   sorted-candidates)))
          sorted-candidates)]
-    (sort-by
-     pagination-candidate-score)))
+    (scored-pagination spurious-candidates-removed corpus)))
 
 (defn pagination-in-corpus-file
   [a-corpus-file]

          :candidate? (pagination-candidate? src-body tgt-body)}))
     corpus))))

(defn pagination-in-corpus
  [corpus]
  (let [candidates (-> corpus pagination-candidates frequencies)

(ns crawler.corpus
  "Tools for processing a downloaded corpus"
  (:require [cheshire.core :as json]
            [clojure.java.io :as io]
            [crawler.dom :as dom])
  (:use [clj-xpath.core :only [$x:node+]]
        [clojure.pprint :only [pprint]]
        [structural-similarity.xpath-text :only [similar?]])
  (:import [java.io PushbackReader]))

