protected function processLine($lineNum, $fragment)
    {
        $offset = 0;

        while ($offset < strlen($fragment)) {
            if (($token = $this->match(substr($fragment, $offset), $lineNum)) === false) {
                throw new \Exception('Parse error at line ' . $lineNum . ' offset ' . $offset);
            }
            $this->tokens[] = $token;
            $offset += $token->getLength();
        }
    }