-;   Copyright (c) Olivier Grisel, 2009
-;   The use and distribution terms for this software are covered by the
-;   Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
-;   which can be found in the file epl-v10.html at the root of this distribution.
-;   By using this software in any fashion, you are agreeing to be bound by
-;   the terms of this license.
-;   You must not remove this notice, or any other, from this software.
+;;   Copyright (c) Olivier Grisel, 2009
+;;   The use and distribution terms for this software are covered by the
+;;   Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
+;;   which can be found in the file epl-v10.html at the root of this distribution.
+;;   By using this software in any fashion, you are agreeing to be bound by
+;;   the terms of this license.
+;;   You must not remove this notice, or any other, from this software.
 
-; corpusmaker - Clojure tools to build training dataset for machine learning
-; based NLP algorithms out of Wikimedia dumps
+;; corpusmaker - Clojure tools to build training dataset for machine learning
+;; based NLP algorithms out of Wikimedia dumps
 
 (ns corpusmaker
   (:require

  ; http://lethain.com/entry/2009/nov/15/reading-file-in-clojure/
  (count (map line-func (read-lines filename))))

(defn collect-pages-redis
  [filename]
  (redis/with-server
    {:host "127.0.0.1" :port 6379 :db 0}
    (let [store-func #(redis/set %1 %2)
          process-line #(collect-page store-func %)]
      (process-file filename process-line))))

;
; Utilities to parse a complete Wikimedia XML dump to extract sentence that
; contain token annotated with a wiki link that point to a page that matches a
; named entity with a type among the classes of the DBPedia ontology:
; Person, Organisation, Place, ...
;

; TODO: rewrite this using http://github.com/marktriggs/xml-picker-seq since
; using a zipper keeps all the parsed elements in memory which is not suitable
; for large XML chunks

(defn parse-xml
  "Zipable XML content from any common source"

  [xml] (zfx/xml-> xml :page :revision :text zfx/text))


;; sample timed run
;; (time (dorun (collect-text (parse-xml "chunk-0001.xml"))))

