   [tokens]
   (map s/lower-case tokens))
 
+(defn sanitize-tokens
+  [tokens]
+  (-> tokens
+      (clojure.string/replace #"[^a-zA-Z\s]" "")
+      (clojure.string/split #"\s+")
+      normalize-tokens))
+
 (defn get-tokens-from-doc
   "Takes reddit doc from json data"
   [doc]

        url (doc "url")
        comments (doc "comments")
        comment-string (concat (map #(get % "text") comments))]
    (->
     (apply str body " " comment-string)
     (clojure.string/replace #"[^a-zA-Z\s]" "")
     (clojure.string/split #"\s+")
     normalize-tokens)))

(defn token-seq-from-file
  "Returns [docid token] lazy seq from file"
  [json-datafile]
  (apply
   concat
   (map
    (fn [line id]
      (map #(vector id %) (get-tokens-from-doc (json/read-str line))))
    (line-seq (io/reader json-datafile))
    (iterate inc 0))))

(defn sort-id-token-pairs
  "Sorts id token pairs

  "Main function for creating indicies. 
   Reads in datafile, tokenizes, sorts, and writes to files"
  [jsonfile]
  (create-indicies-helper jsonfile))

(defn read-in-dictionary
  []
