     hid
     (:_id (store-hashtag text))))
 
+
+(defn store-raw-html
+  "Fetch html document and store raw binary in database"
+  [{:keys [url content-type ts] :as expanded-url} url-id]
+  (let [raw-html (if (= url :not-available)
+                   nil
+                   (slurp url))
+        html-title (if raw-html
+                     (-> (java.io.StringReader. raw-html)
+                         enlive/html-resource
+                         (enlive/select [:head :title])
+                         first
+                         :content
+                         first)
+                     :not-available)]
+    (mc/insert-and-return
+     @db
+     "htmls"
+     {:raw raw-html
+      :url url-id})))
+
+
 (defn get-url-id
   "Get url id if exists otherwise store url"
-  [url uid tid ts]
+  [url uid tid ts source?]
   (if-let [url-id (:_id (mc/find-one-as-map @db "urls" {:url url}))]
     url-id
-    (if-let [expanded-url (:url (expand-url url))]
-      (if-let [x-url-id (:_id (mc/find-one-as-map @db "urls" {:url expanded-url}))]
+    (if-let [expanded-url (expand-url url)]
+      (if-let [x-url-id (:_id (mc/find-one-as-map @db "urls" {:url (:url expanded-url)}))]
         x-url-id
-        (:_id (store-url expanded-url uid tid ts)))
+        (if source?
+          (let [new-url-id (:_id (store-url (:url expanded-url) uid tid ts))]
+            (do
+              (store-raw-html expanded-url new-url-id)
+              new-url-id))
+          nil))
       nil)))
 
 
-(defn get-type [{:keys [in_reply_to_status_id retweeted_status entities]}]
+(defn get-type
+  "Dispatches tweet type"
+  [{:keys [in_reply_to_status_id retweeted_status entities]}]
   (if in_reply_to_status_id
     :reply
     (if retweeted_status



(defn store-mention
  "Yeah"
  [uid pid]
  (mc/insert
   @db

(defn store-article
  "Fetch html document, extract title and store them"
  [{:keys [url content-type ts] :as expanded-url}]
  (let [raw-html (if (= url "Not available") nil (slurp url))
        html-title (if raw-html
                     (-> (java.io.StringReader. raw-html)
                         enlive/html-resource
                         (enlive/select [:head :title])
                         first
                         :content
                         first)
                     nil)
        #_(fetch-url-title url)]
    (mc/insert-and-return
     @db
     "articles"
