 (def retries 3)
 
 (def empty-state {
-    :table {}
-    :starter []
+    :forwardtable {}
+    :backwardtable {}
+    :n-grams []
 })
 
-(defn split-sentence-to-words [sentence] (map #(.intern %) (re-seq word-pattern sentence)))
+
+(defn make-canonicalizer [] {
+    :whm (new java.util.WeakHashMap)
+    :lock (new java.lang.Object)})
+
+(defn canonical-get [c v]
+    {
+        :pre [v]
+        :post [% (= v %)]
+    }
+
+    (locking (:lock c)
+        (if-let [result
+            (if-let [wr (.get (:whm c) v)]
+                (if-let [ret (.get wr)]
+                    ret))]
+            result
+            (do
+                (.put (:whm c) v (new java.lang.ref.WeakReference v))
+                (canonical-get c v) ))))
+
+(def canon (let [c (make-canonicalizer)]
+    (fn [v] (canonical-get c v))))
+
+(defn split-sentence-to-words [sentence] (map canon (re-seq word-pattern sentence)))
 
 (defn select [c i]
     (let [[word weight] (first c)]

        (send-message irc channel
            (join " " (generate-sentence
                @state-ref
                (let [keywords (set (filter #(not (.contains % (:name @irc))) (split-sentence-to-words message)))]
                    (if (<= (count keywords) 3) keywords #{}))))))
    (if (.startsWith message "!burrito")
        (send-message irc channel

        (let [[prefix remainder] (split-at prefix-length words)]
            (recur
                (update-transition
                    (canon (vec prefix))
                    (first remainder)
                    table)
                (rest words)))
        table))
(defn update-ngrams [ngrams words]
    (if (>= (count words) prefix-length)
        (let [[prefix remainder] (split-at prefix-length words)]
            (recur
                (conj ngrams (canon (vec prefix)))
                (rest words)))
        ngrams))

(defn update-state [message state]
    (let [words (concat [:begin] (split-sentence-to-words message) [:end])]
        (assoc state
            :forwardtable
                (update-table (:forwardtable state) words)
            :backwardtable
                (update-table (:backwardtable state) (reverse words))
            :n-grams
                (update-ngrams (:n-grams state) words))))
(defn log-sentence! [message]
    (with-open [out (writer corpus-file :append true)]
        (.write out (str message "\n"))))
