 
 
 
-(mg/connect! {:host "192.168.3.53" :port 7017})
+;(mg/connect! {:host "192.168.3.53" :port 7017})
+
+(mg/connect!)
 
 (mg/set-db! (mg/get-db "edu"))
+
+(defn correct-nil
+  [string]
+  (if (or (= "" string) (nil? string))
+    "1970-1-1"
+    string))
 
 
 (defn parse-date
   [string]
   (let [fmt (f/formatter (t/default-time-zone) "yyyy-MM-dd HH:mm:ss" "yyyy-MM-dd HH:mm" "yyyy-MM-dd" "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'")]
-    (->> string
+    (->> (correct-nil string)
          (f/parse fmt))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;link main;;;;;;;;;;;;;;;;;;;;;;;;;;
 
-(defn link-tieba
-  [db entry]
-  (mc/find-one-as-map db {:url (get entry :url)}))
-
-(->> (mc/find-one-as-map "baidu_tieba_contents" {:_id (ObjectId. "5326cc6aeb78a9650033b8e1")})
-     (link-tieba "baidu_tieba_main"))
+(defn get-key
+  [from-col to-col from-key to-key]
+  (mc/ensure-index from-col {from-key 1})
+  (let [col (mc/find-maps to-col)]
+    (for [entry col]
+      (let [match (mc/find-one-as-map from-col {from-key (get entry to-key)})]
+        (assoc entry :keyword (:keywords match))))))
+
+(defn write-key-db
+  [links]
+  (for [link links]
+    (let [[result-db & origin] link]
+      (->> (apply get-key origin)
+           (insert-by-part result-db)))))
+
+#_(write-key-db [["xuetesttieba" "baidu_tieba_main" "baidu_tieba_contents" :url :url]
+               ["xuetesttianya" "tianya_search" "tianya_content" :url :url]])
+
+
 
 
 

  (let [mini (:minireps entry)
        fuser #(str (:user_name %))
        fdate #(parse-date (:time %))
        floor "m"
        ftext #(:text %)
        fmap #(assoc {} :user (fuser %) :date (fdate %) :floor floor :text (ftext %))
        minimaps (map fmap mini)
        majormap {:user (:name (:author entry)) :date (parse-date (:postTime entry)) :floor (:floor (:content entry)) :text (:text entry)}
        allmaps (conj minimaps majormap)
        idmaps (map #(assoc % :mid (:_id entry)) allmaps)]
    idmaps
    ))





(defn extract-tianya
  [source-address]
  (let [m (mc/find-maps source-address)
        f #(select-keys % [:_id :content])
        g #(assoc {} :mid (:_id %) :text (:content %) :level (Integer. 0) :source "tianya")]
    (->> m
         (map f)
         (map g)
         )))

(defn extract-tianya
  [entry]
  )

;(insert-by-part "xuetesttieba" (apply concat (map #(extract-tieba %) (mc/find-maps "baidu_tieba_contents"))))

(defn extract-weibo
  [source-address]
  (let [m (mc/find-maps source-address)
        f #(select-keys % [:_id :text])
        g #(assoc {} :mid (:_id %) :text (:text %) :level (Integer. 0) :source "weibo")]
    (->> m
         (map f)
         (map g))))

(defn extract-douban
  [source-address]
  (let [m (mc/find-maps source-address)
        f #(select-keys % [:_id :comment])
        g #(assoc {} :mid (:_id %) :text (:comment %) :level (Integer. 0) :source "douban")]
    (->> m
         (map f)
         (map g))))

(defn extract-youku
  [source-address]
  (let [m (mc/find-maps source-address)
        f #(select-keys % [:_id :text])]
    (map #(assoc % :level (Integer. 0) :source "youku") (map f m))
  ))





;(insert-by-part "word_count" (word-seg (mc/find-maps "xuetest") :text :source :mid))





