public function tokenize($str)
    {
        // split the string in tokens and create documents to be
        // classified
        $tokens = $this->tok->tokenize($str);
        $docs = array();
        foreach ($tokens as $offset=>$tok) {
            $docs[] = new WordDocument($tokens,$offset,5);
        }

        // classify each token as an EOW or O
        $tags = array();
        foreach ($docs as $doc) {
            $tags[] = $this->classifier->classify(self::$classSet, $doc);
        }

        // merge O and EOW into real tokens
        $realtokens = array();
        $currentToken = array();
        foreach ($tokens as $offset=>$tok) {
            $currentToken[] = $tok;
            if ($tags[$offset] == self::EOW) {
                $realtokens[] = implode($this->sep,$currentToken);
                $currentToken = array();
            }
        }

        // return real tokens
        return $realtokens;
    }