   (:import [com.mongodb MongoOptions ServerAddress WriteConcern];the following two is for mongo use
            org.bson.types.ObjectId))
 
-#_(mg/connect! {:host "192.168.3.53" :port 7017})
+(mg/connect! {:host "192.168.3.53" :port 7017})
 
-(mg/connect!)
+#_(mg/connect!)
 
-#_(mg/set-db! (mg/get-db "test"))
+(mg/set-db! (mg/get-db "test"))
 
-(mg/set-db! (mg/get-db "star"))
+#_(mg/set-db! (mg/get-db "star"))
 
 (declare extract-text extract-tieba extract-tianya extract-weibo extract-douban extract-youku)
+
+(defn insert-by-part
+  [collection data]
+  (let [parts (partition-all 500 data)]
+    (map #(mc/insert-batch collection %) parts)))
 
 (defn integrate-text
   [& {:as source}]


(flatten [[{:a 2} {:b 3}] [{:c 4} {:d 5}]])



         (map f)
         (map g))))

(defn extract-douban
  [source-address]
  (let [m (mc/find-maps source-address)
        f #(select-keys % [:_id :comment])
        g #(assoc {} :mid (:_id %) :text (:comment %) :level (Integer. 0) :source "douban")]
    (->> m
         (map f)
         (map g))))

(defn extract-youku
  [source-address]
  (let [m (mc/find-maps source-address)
        f #(select-keys % [:_id :text])]
    (map #(assoc % :level (Integer. 0) :source "youku") (map f m))
  ))



