                 (store-mention uid _id)
                 nil)))
           user_mentions)))
-     (mc/find-maps @db "published"))))
+     (mc/find-maps @db "publications"))))
+
+
+  (->> (mc/find-maps @db "publications")
+       (map :hashtags)
+       flatten
+       (into #{})
+       second
+       (mc/find-map-by-id @db "hashtags")
+       aprint)
+
 
 
 

             (take 10)
             (pmap get-url-id)
             #_(pmap #(let [url-id (get-url-id %)]
                        (store-published (get-uid %)
                                         (:_id %)
                                         url-id
                                         (get-type % url-id)

                (if-not (empty? (:urls entities))
                  :source
                  :unrelated))))
          (transact-publications [{:keys [user _id entities created_at] :as tweet}]
            (let [uid (:_id (mc/find-one-as-map @db "users" {:id (:id user)}))
                  url-id (if-not (empty? (:urls entities))
                           (:_id (mc/find-one-as-map @db "urls" (:tweet _id)))
                           nil)
                  hids (if-not (empty? (:hashtags entities))
                         (->> (:hashtags entities)
                              (pmap #(:_id (mc/find-one-as-map @db "hashtags" {:text (:text %)})))
                              (into #{})
                              vec)
                         nil)
                  t-type (dispatch-type tweet)]
              (store-publications uid _id url-id t-type hids created_at)))]
    (let [tweets (mc/find-maps @db "tweets"
                               {:created_at {$gt (t/date-time 2014 8 1)
                                             $lt (t/date-time 2014 9 1)}
                                :user.screen_name {$in news-accounts}})]
      (time
       (doall
        (pmap transact-publications tweets)))))

  ;; export retweets
  (letfn [(transact-publications [{:keys [user _id created_at entities id_str retweeted_status] :as tweet}]
            (let [uid (:_id (mc/find-one-as-map @db "users" {:id (:id user)}))
                  hids (if-not (empty? (:hashtags entities))
                         (->> (:hashtags entities)
                              (pmap #(:_id (mc/find-one-as-map @db "hashtags" {:text (:text %)})))
                              (into #{})
                              vec)
                         nil)]
              (store-publications uid _id nil :retweet hids created_at)))]
    (let [retweets (mc/find-maps @db "tweets" {:created_at {$gt (t/date-time 2014 8 1)
                                                            $lt (t/date-time 2014 9 1)}
                                               :retweeted_status {$ne nil}
                                               :user.screen_name {$nin news-accounts}})]
      (time (doall (pmap transact-publications retweets)))))



  ;; export replies
  (letfn [(transact-publications [{:keys [user _id created_at entities id_str in_reply_to_status_id_str] :as tweet}]
            (let [uid (:_id (mc/find-one-as-map @db "users" {:id (:id user)}))
                  hids (if-not (empty? (:hashtags entities))
                         (->> (:hashtags entities)
                              (pmap #(:_id (mc/find-one-as-map @db "hashtags" {:text (:text %)})))
                              (into #{})
                              vec)
                         nil)]
              (store-publications uid _id nil :reply hids created_at)))]
    (let [retweets (mc/find-maps @db "tweets" {:created_at {$gt (t/date-time 2014 8 1)
                                                            $lt (t/date-time 2014 9 1)}
                                               :in_reply_to_status_id_str {$ne nil}
                                               :user.screen_name {$nin news-accounts}})]
      (time (doall (pmap transact-publications retweets)))))



  ;; export shares
  (letfn [(transact-publications [{:keys [user _id created_at entities id_str in_reply_to_status_id_str] :as tweet}]
            (let [uid (:_id (mc/find-one-as-map @db "users" {:id (:id user)}))
                  hids (if-not (empty? (:hashtags entities))
                         (->> (:hashtags entities)
                              (pmap #(:_id (mc/find-one-as-map @db "hashtags" {:text (:text %)})))
                              (into #{})
                              vec)
                         nil)]
              (store-publications uid _id nil :share hids created_at)))]
    (time
     (doall
      (->> (mc/find-maps @db "origins" {:source nil
