 (def retries 3)
 
 (def empty-state {
-    :table {}
-    :starter []
+    :forwardtable {}
+    :backwardtable {}
+    :n-grams []
 })
 
-(defn split-sentence-to-words [sentence] (map #(.intern %) (re-seq word-pattern sentence)))
+
+(defn make-canonicalizer [] {
+    :whm (new java.util.WeakHashMap)
+    :lock (new java.lang.Object)})
+
+(defn canonical-get [c v]
+    {
+        :pre [v]
+        :post [% (= v %)]
+    }
+
+    (locking (:lock c)
+        (if-let [result
+            (if-let [wr (.get (:whm c) v)]
+                (if-let [ret (.get wr)]
+                    ret))]
+            result
+            (do
+                (.put (:whm c) v (new java.lang.ref.WeakReference v))
+                (canonical-get c v) ))))
+
+(def canon (let [c (make-canonicalizer)]
+    (fn [v] (canonical-get c v))))
+
+(defn split-sentence-to-words [sentence] (map canon (re-seq word-pattern sentence)))
 
 (defn select [c i]
     (let [[word weight] (first c)]

            (cons [word had-choice] (further-stream-of-shite (concat (rest prefix) [word]) t)))))
(defn stream-of-shite [prefix t]
    (concat (map vector prefix (repeat false)) (further-stream-of-shite prefix t)))
(defn generate-sentence [state keywords]
    (let [{:keys [table starters]} state]
        (if (empty? keywords)
            (drop-last (map first
                (apply max-key #(count (filter true? (map second %)))
                    (take retries (repeatedly #(take 1000 (stream-of-shite
                        (nth starters (rand-int (count starters)))
                        table)))))))
            (let [filtered-starters
                    (filter
                        (fn [s] (not (empty? (filter
                            (fn [k] (not (empty? (filter #(.contains % k) s))))
                            keywords))))
                        starters)]
                (if (empty? filtered-starters)
                    (generate-sentence state [])
                    (generate-sentence
                        (assoc state :starters filtered-starters)
                        []))))))

(defn value-or-default [value default]
    (if (nil? value) default value))

        (send-message irc channel
            (join " " (generate-sentence
                @state-ref
                (let [keywords (set (remove #(.contains % (:name @irc)) (split-sentence-to-words message)))]
                    (if (<= (count keywords) 3) keywords #{}))))))
    (if (.startsWith message "!burrito")
        (send-message irc channel
