             [crawler.rank :as rank]
             [crawler.records :as records]
             [crawler.utils :as utils]
-            [itsy.core :as itsy]
             [org.bovinegenius [exploding-fish :as uri]])
   (:use [clojure.tools.logging :only (info error)]
         [clj-logging-config.log4j])

    updates              :updates
    in-host-xpaths-hrefs :in-host-xpaths-hrefs}]
  
  (let [explorations-ds (exploration->exploration-ds explorations)

        cluster-member? (fn [a-cluster x]
                          (some
                           (fn [a-member]
                             (>= (page/signature-edit-distance-similarity
                                  (-> a-member :r1)
                                  (-> x :r1)
                                  0.25
                                  0.75)
                                 *page-sim-thresh*))
                           a-cluster))
        
        clusters        (reduce
                         (fn [clusters x]
                           (cluster/assign
                            (into [] clusters) x cluster-member?))
                         []
                         explorations-ds)

        self-cluster    (cluster/assign-where?
                         clusters
                         {:r1 signature
                          :incoming-xpath nil
                          :url url}
                         cluster-member?)

        content-candids (map
                         #(nth clusters %)

        xpath-expls)))
    explorations)))

(defn cluster-member?
  [a-cluster x]
  (some
   (fn [a-member]
     (>= (page/signature-edit-distance-similarity
          (-> a-member :r1)
          (-> x :r1)
          0.25
          0.75)
         *page-sim-thresh*))
   a-cluster))

(defn cluster-explorations
  "Explorations are converted to an exploration-ds and
   these are clustered together. We don't assign the
   source URL into the clusters (is this a good idea?)"
  [{url                  :src-link
    explorations         :explorations
    xpaths-hrefs         :xpaths-hrefs
    signature            :signature
    dfs                  :dfs
    hrefs                :hrefs
    novelties            :novelties
    updates              :updates
    in-host-xpaths-hrefs :in-host-xpaths-hrefs}]

  (let [explorations-ds (exploration->exploration-ds explorations)]
    
    (reduce
     (fn [clusters x]
       (cluster/assign
        (into [] clusters) x cluster-member?))
     []
     explorations-ds)))

(defn process-page-cluster
  "Page clusters are explored"
  [explorations-ds-clustered]
  
  '*)

(defn cluster-explorations-titanic
  "Explorations are grouped into
clusters based on their similarities.

titanic because this guy is going down"
  [{url                  :src-link
    explorations         :explorations
    xpaths-hrefs         :xpaths-hrefs
