public function tokenize($code, $filename = null)
    {
        if (function_exists('mb_internal_encoding') && ((int) ini_get('mbstring.func_overload')) & 2) {
            $encoding = mb_internal_encoding();
            mb_internal_encoding('ASCII');
        }

        $this->code = str_replace(array("\r\n", "\r"), "\n", $code);
        $this->source = '';
        $this->filename = $filename;
        $this->cursor = 0;
        $this->lineno = 1;
        $this->end = strlen($this->code);
        $this->tokens = array();
        $this->state = self::STATE_DATA;
        $this->states = array();
        $this->brackets = array();
        $this->position = -1;

        preg_match_all(self::REGEX_CHAR, $this->code, $this->positions, PREG_OFFSET_CAPTURE);

        while ($this->cursor < $this->end) {
            switch ($this->state) {
                case self::STATE_DATA:
                    $this->lexData();
                    break;

                case self::STATE_OUTPUT:
                    $this->lexOutput();
                    break;

                case self::STATE_DIRECTIVE:
                    $this->lexDirective();
                    break;
            }
        }

        if ($this->state != self::STATE_DATA) {
            $this->addCode(' ?>');
            $this->popState();
        }

        if (!empty($this->brackets)) {
            list($expect, $lineno) = array_pop($this->brackets);
            throw new SyntaxErrorException(sprintf('Unclosed "%s" at line %d in file %s', $expect, $lineno, $this->filename));
        }

        if (isset($encoding)) {
            mb_internal_encoding($encoding);
        }

        return new TokenStream(token_get_all($this->source));
    }