   "Expands shortened url strings, thanks to http://www.philippeadjiman.com/blog/2009/09/07/the-trick-to-write-a-fast-universal-java-url-expander/"
   [url-str]
   (let [url (java.net.URL. url-str)
-        conn (.openConnection url)]
-    (do (.setInstanceFollowRedirects conn false)
-        (.connect conn)
-        (let [expanded-url (.getHeaderField conn "Location")
-              content-type (.getContentType conn)]
+        conn (try
+               (.openConnection url)
+               (catch Exception e (do (error (str e))
+                                      false)))]
+    (if conn
+      (do (.setInstanceFollowRedirects conn false)
           (try
-            (do (.close (.getInputStream conn))
-                {:url expanded-url
-                 :content-type content-type})
+            (do
+              (.connect conn)
+              (let [expanded-url (.getHeaderField conn "Location")
+                    content-type (.getContentType conn)]
+                (try
+                  (do (.close (.getInputStream conn))
+                      {:url expanded-url
+                       :content-type content-type})
+                  (catch Exception e (do (error (str e))
+                                         {:url "Not available"
+                                          :content-type content-type})))))
             (catch Exception e (do (error (str e))
-                                   {:url "Not available"
-                                    :content-type content-type})))))))
+                                   nil))))
+      nil)))
 
 
 (defn store-origin




  ;; export urls
  (time
   (let [origins (mc/find-maps @db "origins" {:source {$ne nil}})]
     (doall
      (map
       (fn [{:keys [article tweet ts source]}]
         (let [uid (:_id (mc/find-one-as-map @db "user" {:screen_name source}))
               url (if article
                     (:url (mc/find-map-by-id @db "articles" article))
                     nil)]
           (if url
             (store-url url uid ts)
             :unknown)))
       origins))))


  ;; export un-parsed urls
  (letfn [(expand-urls [{{:keys [urls]} :entities}]
            (pmap #(let [expanded-url (expand-url (:expanded_url %))]
                    (if expanded-url
                      (:url expanded-url)
                      %)) urls))]
    (let [tweets (mc/find-maps @db "tweets" {:created_at {$gt (t/date-time 2014 7 1)
                                                          $lt (t/date-time 2014 7 29)}
                                             :entities.urls {$ne []}

    :date date}))


(defn store-published [uid tid url-id type hids ts]
  (mc/insert
   @db
   "published"
