     :date date}))
 
 
-(defn store-published [uid tid url-id type hids ts]
+(defn store-publications [uid tid url-id type hids ts]
   (mc/insert-and-return
    @db
-   "published"
+   "publications"
    {:user uid
     :tweet tid
     :url url-id

                (if-not (empty? (:urls entities))
                  :source
                  :unrelated))))
          (transact-published [{:keys [user _id entities created_at] :as tweet}]
            (let [uid (:_id (mc/find-one-as-map @db "users" {:id (:id user)}))
                  url-id (if-not (empty? (:urls entities))
                           (:_id (mc/find-one-as-map @db "urls" (:tweet _id)))
                           nil)
                  hids (if-not (empty? (:hashtags entities))
                         (->> (:hashtags entities)
                              (pmap #(:_id (mc/find-one-as-map @db "hashtags" (:text (:text %)))))
                              (into #{})
                              vec)
                         nil)
                  t-type (dispatch-type tweet)]
              (store-published uid _id url-id t-type hids created_at)))]
    (let [tweets (mc/find-maps @db "tweets"
                               {:created_at {$gt (t/date-time 2014 8 1)
                                             $lt (t/date-time 2014 9 1)}
                                :user.screen_name {$in news-accounts}})]
      (time
       (doall
        (pmap transact-published tweets)))))

  ;; export retweets
  (letfn [(transact-published [{:keys [user _id created_at entities id_str retweeted_status] :as tweet}]
            (let [uid (:_id (mc/find-one-as-map @db "users" {:id (:id user)}))
                  hids (if-not (empty? (:hashtags entities))
                         (->> (:hashtags entities)
                              (pmap #(:_id (mc/find-one-as-map @db "hashtags" (:text (:text %)))))
                              (into #{})
                              vec)
                         nil)
                  user-mentions (if-not (empty (:user_mentions entities))
                                  (->> (:user_mentions entities)
                                       (pmap #(:_id (mc/find-one-as-map @db "users" {:id (:id %)}))))
                                  #{})
                  pub-id (store-published uid _id nil :retweet hids created_at)]
              (doall (pmap #(when-not (nil? %) (store-mention % pub-id)) user-mentions))))]
    (let [retweets (mc/find-maps @db "tweets" {:created_at {$gt (t/date-time 2014 8 1)
                                                            $lt (t/date-time 2014 9 1)}
                                               :retweeted_status {$ne nil}
                                               :user.screen_name {$nin news-accounts}})]
      (time (doall (pmap transact-published retweets)))))



  ;; export replies
  (letfn [(transact-published [{:keys [user _id created_at entities id_str in_reply_to_status_id_str] :as tweet}]
            (let [uid (:_id (mc/find-one-as-map @db "users" {:id (:id user)}))
                  hids (if-not (empty? (:hashtags entities))
                         (->> (:hashtags entities)
                              (pmap #(:_id (mc/find-one-as-map @db "hashtags" (:text (:text %)))))
                              (into #{})
                              vec)
                         nil)
                  user-mentions (if-not (empty (:user_mentions entities))
                                  (->> (:user_mentions entities)
                                       (pmap #(:_id (mc/find-one-as-map @db "users" {:id (:id %)}))))
                                  #{})
                  pub-id (store-published uid _id nil :reply hids created_at)
                  source-tid (:_id (mc/find-one-as-map @db "published" {:tweet (:_id (mc/find-one-as-map @db "tweets" {:id_str in_reply_to_status_id_str}))}))]
              (doall (pmap #(when-not (nil? %) (store-mention % pub-id)) user-mentions))))]
    (let [retweets (mc/find-maps @db "tweets" {:created_at {$gt (t/date-time 2014 8 1)
                                                            $lt (t/date-time 2014 9 1)}
                                               :in_reply_to_status_id_str {$ne nil}
                                               :user.screen_name {$nin news-accounts}})]
      (time (doall (pmap transact-published retweets)))))



  ;; export shares
  (letfn [(transact-published [{:keys [user _id created_at entities id_str in_reply_to_status_id_str] :as tweet}]
            (let [uid (:_id (mc/find-one-as-map @db "users" {:id (:id user)}))
                  hids (if-not (empty? (:hashtags entities))
                         (->> (:hashtags entities)
                              (pmap #(:_id (mc/find-one-as-map @db "hashtags" (:text (:text %)))))
                              (into #{})
                              vec)
                         nil)]
              (store-published uid _id nil :share hids created_at)))]
    (time
     (doall
      (->> (mc/find-maps @db "origins" {:source nil

             (take 10)
             (pmap get-url-id)
             #_(pmap #(let [url-id (get-url-id %)]
                        (store-publications (get-uid %)
                                         (:_id %)
                                         url-id
                                         (get-type % url-id)
