 
 
 (defn get-tweets-from-date [month day]
-  (->> (mc/find
-        (:db @mongo-state)
-        "tweets"
-        {:created_at
-         {$gt (t/date-time 2014 month day 0 0 0 0)
-          $lte (t/date-time 2014 month day 23 59 59 999)}
-         })
-       seq
-       (pmap #(from-db-object % true))))
+  (mc/find-maps (:db @mongo-state) "tweets"
+                {:created_at {$gt (t/date-time 2014 month day 0 0 0 0)
+                              $lte (t/date-time 2014 month day 23 59 59 999)}}))
 
 
 (defn get-hashtag-frequencies [coll]

      (:_id x)
      (update-in x [:created_at] #(f/parse (:custom-formatter @mongo-state) (:created_at %))))))

  (->> (mc/find (:db @mongo-state) "articles")
       seq
       (map #(from-db-object % true)))

  (let [raw-html (slurp "http://www.tagesschau.de/multimedia/bilder/mh17-bergung-120.html")]
    (type raw-html)
    (-> (enlive/html-resource (java.io.StringReader. raw-html))
        clojure.pprint/pprint))

  (take 10 (sort-by val > (-> (mc/find (:db @mongo-state) "articles")
                              seq
                              last
                              (from-db-object true)
                              :html
                              (clojure.string/split #"\s")
                              (into #{})
                              frequencies)))

  (mc/ensure-index (:db @mongo-state) "articles" (array-map :ts 1))



  (time
   (loop [round 10]
     (when (> round 0)

       (recur (dec round)))))

  (->> (mc/find-maps (:db @mongo-state) "urls" {:ts {$gt (t/date-time 2014 7 26 14)}})
       (map :article)
       frequencies
       (sort-by val >)
       (take 10)
       )

  (->> (mc/find-maps (:db @mongo-state) "articles" {:ts {$gt (t/today)}} [:title])
       (take 50)
       (map #(vec [(:title %) (find-source (:_id %))]))
       (remove #(empty? (second %)))
      clojure.pprint/pprint)


  )

                                                    {:created_at {$gt (t/date-time 2014 7 1)}}]})))


(defn compute-tweet-diffusion [tweet-id parents]
  (let [tweet (mc/find-one-as-map (:db @mongo-state) "tweets" {:id_str tweet-id})
        neighbor-tweets (->> (mc/find-maps
                              (:db @mongo-state)
                              "tweets"
                              {$and [{$or [{"retweeted_status.id_str" tweet-id}
                                           {"in_reply_to_status_id_str" tweet-id}]}
                                     {:created_at {$gt (t/date-time 2014 7 1)}}]}
                              [:text :user.screen_name :id_str :in_reply_to_status_id_str :retweeted_status.id_str])
                             (pmap #(assoc % :parents (into parents [(:_id tweet)]))))
        neightbar-ids (pmap :id_str neighbor-tweets)]
    (merge neighbor-tweets (pmap #(compute-tweet-diffusion % (into parents [(:_id tweet)])) neightbar-ids))))


(defn get-news-diffusion []
