func NewTokenizer(r io.Reader) *Tokenizer {
	input := bufio.NewReader(r)
	classifier := newDefaultClassifier()
	return &Tokenizer{
		input:      *input,
		classifier: classifier}
}
func NewTokenizer(r io.Reader) *Tokenizer {
	input := bufio.NewReader(r)
	classifier := newDefaultClassifier()
	return &Tokenizer{
		input:      *input,
		classifier: classifier}
}
func NewTokenizer ( r io . Reader ) * Tokenizer { input := bufio . NewReader ( r ) classifier := newDefaultClassifier ( ) return & Tokenizer { input : * input , classifier : classifier } }
// NewTokenizer creates a new tokenizer from an input stream.
NewTokenizer creates a new tokenizer from an input stream .