public function main()
    {
        $this->setup();
        while (!$this->eos()) {
            $p = $this->pos();
            $state = $this->stateName();

            $this->loadTransitions();
            list($nextPatternData, $nextPatternIndex, $nextPatternMatches) = $this->nextStartData();
            list($endIndex, $endMatches) = $this->nextEndData();

            if (($nextPatternIndex <= $endIndex || $endIndex === -1) && $nextPatternIndex !== -1) {
                // we're pushing a new state
                if ($p < $nextPatternIndex) {
                    $this->recordRange($p, $nextPatternIndex);
                }
                $newPos = $nextPatternIndex;
                $this->pos($newPos);
                $tok = $nextPatternData[0];
                if (isset($this->overrides[$tok])) {
                    // call override
                    $ret = call_user_func($this->overrides[$tok], $nextPatternMatches);
                    if ($ret === true) {
                        break;
                    }
                    if ($this->stateName() === $state && $this->pos() <= $newPos) {
                        throw new Exception('Override failed to either advance the pointer or change the state');
                    }
                } else {
                    // no override
                    $this->posShift(strlen($nextPatternMatches[0]));
                    $this->pushState($nextPatternData);
                    $this->record($nextPatternMatches[0]);
                    if ($nextPatternData[2] === null) {
                        // state was a full pattern, so pop now
                        $this->popState();
                    }
                }
            } elseif ($endIndex !== -1) {
                // we're at the end of a state, record what's left and pop it
                $to = $endIndex + strlen($endMatches[0]);
                $this->recordRange($this->pos(), $to);
                $this->pos($to);
                $this->popState();
            } else {
                // no more matches, consume the rest of the stirng and break
                $this->record($this->rest());
                $this->terminate();
                break;
            }
            if ($this->stateName() === $state && $this->pos() <= $p) {
                throw new Exception('Failed to advance pointer in state' . $this->stateName());
            }
        }

        // unterminated states will have left some tokens open, we need to
        // close these so there's just the root node on the stack
        assert(count($this->tokenTreeStack) >= 1);
        while (count($this->tokenTreeStack) > 1) {
            $this->popState();
        }

        return $this->tokenTreeStack[0];
    }