 
 (defn insert-exercise-to-index [assoc-fn index ex]
   (let [terms (get-ex-terms ex)
-        ex-id (str (:_id ex))] 
+        ex-id (str (:_id ex))]
+    (swap! timestamps assoc ex-id (cljc/to-long (:lastModifiedDate ex)))
     (reduce (partial insert-term assoc-fn ex-id) index terms)))
 
 (defn insert-exercise-inmem-index [ex]
-  (dosync (alter inverted-index (partial insert-exercise-to-index assoc) ex)))
-  
+  (count (dosync (alter inverted-index (partial insert-exercise-to-index assoc) ex))))
+
+(defn count-freqs [] 
+  (let [freqs (reduce (fn [res entry] (assoc res (first entry) (count (second entry)))) {} @inverted-index)]
+    (count (reset! term-freq freqs))))
+
 (defn rebuild-index []
   (let [new-index (transient {})]
-    (count 
      (dosync 
        (ref-set inverted-index 
                (persistent! 
-                (reduce (partial insert-exercise-to-index assoc!) new-index (mc/find-maps EXERCISE {}))))))))
+                (reduce (partial insert-exercise-to-index assoc!) new-index (mc/find-maps EXERCISE {}))))))
+  (count-freqs))
 
-(defn search-ids [& terms]
-  (if (> (count terms) 0)
-    (take search-per-page (apply clojure.set/intersection (map #(@inverted-index %) terms)))
-    '()))
+(defn to-fast-order [terms]
+  (let [freqs @term-freq
+        count-term #(if (freqs %) (freqs %) 0)]
+    (sort-by count-term terms)))
 
-(defn search [& terms]
-  (let [ids  (apply search-ids terms)]
+(defn intersect [terms]
+  (let [result (apply clojure.set/intersection (map #(@inverted-index %) (to-fast-order terms)))
+        ts @timestamps
+        sortfn #(if (ts %) (ts %) 0)
+        sorted-result (sort-by sortfn > result)] ;; todo use lazy sort
+    sorted-result))
+
+(defn search-ids [page terms]
+  (let [result (intersect terms)]
+    (take-last search-per-page (take (* page search-per-page) result))))
+
+(defn get-one [id]
+  (try 
+    (mc/find-one-as-map EXERCISE {:_id (ObjectId. id)})
+    (catch Exception exception
+      (.error logger (str "Unable to get ex " id " " exception)))))
+
+(defn search [page terms]
+  (let [ids  (search-ids page terms)]
     (.debug logger (str "Terms " terms " search result count: " (count ids)))
-    (as-ex-result-list (sort-by :creationDate time/after? (filter (partial not= nil) (map #(mc/find-one-as-map EXERCISE {:_id (ObjectId. %)}) ids))))))
+    (as-ex-result-list (filter identity (map get-one ids)))))
 
 (defn search-wrapper [query]
   (let [query-string (:q query)
-        terms (filter not-too-short? (re-seq re-term (.toLowerCase query-string)))]
+        terms (filter valid-term? (re-seq re-term (.toLowerCase query-string)))]
     (if (> (count terms) 0)
-      []
-      (apply search terms))))
+      (search 1 terms)
+      [])))

          
(def inverted-index (ref {}))

(defn insert-term [assoc-fn ex-id index term]
  (if-let [postings (index term)]
    (let [new-val (conj postings ex-id)]


(defn to-term-seq [^String words]
  (if words
    (filter valid-term? (re-seq re-term (.toLowerCase (strip-html words))))
    []))

(defn to-term-list [exercise] 
