   [xml] (zfx/xml-> xml :page :revision :text zfx/text))
 
 
-; sample timed run
-; (time (dorun (collect-text (parse-xml "chunk-0001.xml"))))
+;; sample timed run
+;; (time (dorun (collect-text (parse-xml "chunk-0001.xml"))))
 

;   Copyright (c) Olivier Grisel, 2009
;   The use and distribution terms for this software are covered by the
;   Eclipse Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
;   which can be found in the file epl-v10.html at the root of this distribution.
;   By using this software in any fashion, you are agreeing to be bound by
;   the terms of this license.
;   You must not remove this notice, or any other, from this software.

; corpusmaker - Clojure tools to build training dataset for machine learning
; based NLP algorithms out of Wikimedia dumps

(ns corpusmaker
  (:require

  (:use
     [clojure.contrib.duck-streams :only (read-lines)]))

;;
;; Utilities to parse DBPedia N-TRIPLES dump to load them in a redis DB to
;; perform fast look up of wikilink => entity type
;;

;; RE to parse a line of a N-TRIPLES RDF stream to find a type relationship
;; http://www.w3.org/TR/rdf-testcases/#ntriples
(def *type-pattern*
  #"<([^<]+?)> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <([^<]+?)> \.")
(def *page-pattern*
  #"<([^<]+?)> <http://xmlns.com/foaf/0.1/page> <([^<]+?)> \.")

(def *pages-db* "entities-pages")
(def *types-db* "pages-types")

(defn process-file [filename line-func]
  ; TODO: add start line and end line (max)
