   ; http://lethain.com/entry/2009/nov/15/reading-file-in-clojure/
   (count (map line-func (read-lines filename))))
 
-(defn collect-pages-redis
-  [filename]
+(defn collect-page
+  "Parse a N-TRIPLES statement and store the [page entity] pair if match"
+  [statement]
+  (let [[_ entity page] (re-find *page-pattern* statement)]
+    (when entity
+      (redis/set entity page))))
+
+(defn collect-entities-to-pages
+  "Parse a N-TRIPLES file to extract and store entity - page relationships"
+  [filename server-params]
   (redis/with-server
-    {:host "127.0.0.1" :port 6379 :db 0}
-    (let [store-func #(redis/set %1 %2)
-          process-line #(collect-page store-func %)]
-      (process-file filename process-line))))
+    server-params
+    (redis/select *pages-db*)
+    (process-file filename collect-page)))
 
-;
-; Utilities to parse a complete Wikimedia XML dump to extract sentence that
-; contain token annotated with a wiki link that point to a page that matches a
-; named entity with a type among the classes of the DBPedia ontology:
-; Person, Organisation, Place, ...
-;
+(defn join-page-type
+  "Parse a N-TRIPLES statement and store the [page type] pair if match"
+  [statement]
+  (let [[_ entity type_] (re-find *type-pattern* statement)]
+    (when entity
+      (redis/select *pages-db*)
+      (when-let [page (redis/get entity)]
+        (redis/select *types-db*)
+        (redis/set page type_)))))
 
-; TODO: rewrite this using http://github.com/marktriggs/xml-picker-seq since
-; using a zipper keeps all the parsed elements in memory which is not suitable
-; for large XML chunks
+(defn collect-pages-to-types
+  "Parse a N-TRIPLES file to extract types to be joined with page db"
+  [filename server-params]
+  (redis/with-server
+    server-params
+    (process-file filename join-page-type)))
+;;
+;; Utilities to parse a complete Wikimedia XML dump to extract sentence that
+;; contain token annotated with a wiki link that point to a page that matches a
+;; named entity with a type among the classes of the DBPedia ontology:
+;; Person, Organisation, Place, ...
+;;
+
+;; TODO: rewrite this using http://github.com/marktriggs/xml-picker-seq since
+;; using a zipper keeps all the parsed elements in memory which is not suitable
+;; for large XML chunks
 
 (defn parse-xml
   "Zipable XML content from any common source"

  [xml] (zfx/xml-> xml :page :revision :text zfx/text))


; sample timed run
; (time (dorun (collect-text (parse-xml "chunk-0001.xml"))))


  (:use
     [clojure.contrib.duck-streams :only (read-lines)]))

;;
;; Utilities to parse DBPedia N-TRIPLES dump to load them in a redis DB to
;; perform fast look up of wikilink => entity type
;;

;; RE to parse a line of a N-TRIPLES RDF stream to find a type relationship
;; http://www.w3.org/TR/rdf-testcases/#ntriples
(def *type-pattern*
  #"<([^<]+?)> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <([^<]+?)> \.")
(def *page-pattern*
  #"<([^<]+?)> <http://xmlns.com/foaf/0.1/page> <([^<]+?)> \.")

(def *pages-db* "entities-pages")
(def *types-db* "pages-types")

(defn process-file [filename line-func]
  ; TODO: add start line and end line (max)
