 ; some <!-- wiki comment --> inside the text body
 (def *comment* #"<!--(.*?)-->")
 
+; some <ref /> inside the text body
+(def *ref* #"(<ref(.*?)/>|<ref(.*?)>(.*?)</ref>)")
+
 ; some {{wiki directive}} inside the text body
 (def *double-curly* #"\{\{(.+?)\}\}")
 


(defn no-redirect?
  "Check that the page content does not forward to another article"
  [page-markup]
  (-> page-markup .trim (.startsWith "#REDIRECT") not))

(defn clean-markup
  "Remove wiki markup that does not hold annotation data"
  [page-markup]
  (reduce #(-> %2 (.matcher %1) (.replaceAll ""))
          page-markup
          [*comment* *double-curly* *category*]))

(defn collect-text
  "collect wikimarkup payload of a dump in seqable xml"

     [redis])
  (:use
     [clojure.contrib.duck-streams :only (read-lines)]))

;; Parsing big dumps can be really slow if the program spends time handling
;; types dynamically
(set! *warn-on-reflection* true)

;;
;; Utilities to parse DBPedia N-TRIPLES dump to load them in a redis DB to
