          (map f)
          (map g))))
 
-;(extract-weibo "weibo")
+(defn extract-douban
+  [source-address]
+  (let [m (mc/find-maps source-address)
+        f #(select-keys % [:_id :comment])
+        g #(assoc {} :mid (:_id %) :text (:comment %) :level (Integer. 0) :source "douban")]
+    (->> m
+         (map f)
+         (map g))))
 
-
-
-
-
-
-(mc/insert-batch "xuetest" (extract-tieba "tieba"))
-
-;(extract-tieba "star_baidutieba_contents")
-
-
-
+(defn extract-youku
+  [source-address]
+  (let [m (mc/find-maps source-address)
+        f #(select-keys % [:_id :text])]
+    (map #(assoc % :level (Integer. 0) :source "youku") (map f m))
+  ))
 
 
 


(flatten [[{:a 2} {:b 3}] [{:c 4} {:d 5}]])



  (:import [com.mongodb MongoOptions ServerAddress WriteConcern];the following two is for mongo use
           org.bson.types.ObjectId))

(mg/connect! {:host "192.168.3.53" :port 7017})

#_(mg/connect!)

(mg/set-db! (mg/get-db "test"))

#_(mg/set-db! (mg/get-db "star"))

(declare extract-text extract-tieba extract-tianya extract-weibo extract-douban extract-youku)

(defn insert-by-part
  [collection data]
  (let [parts (partition-all 500 data)]
    (map #(mc/insert-batch collection %) parts)))

(defn integrate-text
  [& {:as source}]
