   "Replace consecutive whitespace characters with a single space."
   [text]
   (replace text #"\s{2,}" " "))
+
+(defn tokenize
+  "Split s into tokens delimited by whitespace."
+  [s]
+  (split s #"\s+|-"))
 
 (defn clear-tokens
   "Filter out roman numerals"

  (-> text
      (replace #"\s*\([^)]+\)" "")
      (replace #"\s*\[[^\]]+\]" "")))

(defn remove-punctuation
  "Remove punctuation from s.

                                        placeholder))
                                    tail
                                    tail-range))))
              tokens
              (map range tail-offsets (rest tail-offsets))))))
