     :ts ts}))
 
 
-(defn store-article [{:keys [url content-type ts] :as expanded-url}]
+(defn store-article
+  "Fetch html document, extract title and store them"
+  [{:keys [url content-type ts] :as expanded-url}]
   (let [raw-html (slurp url)
         html-title (-> (java.io.StringReader. raw-html) enlive/html-resource (enlive/select [:head :title]) first :content first)]
     (mc/insert-and-return



(defn store
  "Stores the given tweet in mongodb"
  [tweet]
  (let [oid (ObjectId.)
        doc (update-in tweet [:created_at] (fn [x] (f/parse custom-formatter x)))

(def news-accounts #{"FAZ_NET" "dpa" "tagesschau" "SPIEGELONLINE" "SZ" "BILD" "DerWesten" "ntvde" "tazgezwitscher" "welt" "ZDFheute" "N24_de" "sternde" "focusonline"} )


(defn init-mongo
  "Define mongodb indices on first start"
  []
  (do
    (mc/ensure-index db "articles" (array-map :ts 1))
    (mc/ensure-index db "origins" (array-map :ts 1))
