       (assoc row (keyword col-name) token)
        )))
 
-(def aa (read-text "D:/training/pubmed-oa-subset.csv" "col3"))
-
-(frequencies (apply concat (map :col3 aa)))
-
-(sort-map-by)
+(defn sort-map-value
+  [map-obj]
+  (into (sorted-map-by (fn [key1 key2]
+                         (compare [(get map-obj key2) key2]
+                                  [(get map-obj key1) key1])))
+        map-obj))
 
 
-(defn document-count-filter
-  )
+(defn heuristic-filter
+  [csv-file col-name]
+  (let [dataset (read-text csv-file col-name)]
+    (->> dataset
+         (map (keyword col-name))
+         (apply concat)
+         frequencies
+         sort-map-value
+         (drop 40)
+         (into {})
+         (filter #(> (val %) 5))
+         (into {})
+         keys
+         set
+         )))
+
+(heuristic-filter "E:/training/pubmed-oa-subset.csv" "col3")
+(#{1 2 3} 4)
+
+
+
+(def aa (read-text "E:/training/pubmed-oa-subset.csv" "col3"))
 
 
 
 
-(defn xxx
-  [x]
-  (if (nil? x)
-    3
-    4))
 
 
-(tokenizer sstt)
-(tokenizer bb)
+(rest {:a 2 :b 3 :c 4})
 
-(string/lower-case "fafa#")
+
+
+(filter #(<= (val %) 5) {:a 2 :b 3 :d 10})
+(filter [])
 
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;tips;;;;;;;;;;;;;;;;;;;;

(string/upper-case "thk")
(string/lower-case "THK")

(#(string/split % punc-regex) sstt)

(remove even? [2 3 4])

hash-map
(apply assoc {} [3 5 7 9 8 10])

(apply assoc {} 3 4 5 60)

(assoc {:3 4} :3 5)






(def bb
  (->>
 (io/read-dataset "E:/training/pubmed-oa-subset.csv")
 :rows
 first
 :col3))

(def punc-regex  #"[,.\s;:-?!\/\\\"\'_\(\)\{\}\[\]@—\$&\^\*\+\t\r\n\a\e\f\v]")

(incanter/view (io/read-dataset "E:/training/pubmed-oa-subset.csv"))


(seq "hahahah")
