public function getTokens()
    {
        fseek($this->stream, 0);
        $this->buffer = new StreamBuffer($this->stream, static::BUFFER_SIZE, $this->minLength);
        $this->buffer->read();

        /** @var Token $last */
        $last = null;

        while (!$this->buffer->isEof()) {
            foreach ($this->state->match($this->buffer) as $token) {
                if ($token[0] == Token::T_BOM) {
                    $this->changeEncoding($token[1]);
                }

                $this->state = $this->state->getNextState($token[0]);

                // merge tokens together to condense T_CONTENT tokens
                if ($token[0] == Token::T_CONTENT) {
                    if (!is_null($last)) {
                        $last[1] .= $token[1];
                        $last[3] = strlen($last[1]);
                    } else {
                        $last = $token;
                    }
                } else {
                    if (!is_null($last)) {
                        yield $last;
                        $last = null;
                    }
                    yield $token;
                }

                $this->buffer->move($token[3]);
                $this->buffer->read();
            }
        }

        if (!is_null($last)) {
            yield $last;
        }

        fclose($this->stream);
    }