private function initTokenize() 
    {
        $this->_tokenize = function ($input, $finalize) {
            // If the input is a string, continuously emit tokens through the callback until the end
            if (!isset($this->input))
                $this->input = "";
            $this->input .= $input;
            $tokens = [];
            $error = "";
            $this->input = $this->tokenizeToEnd(function ($e, $t) use (&$tokens,&$error) {
                if (isset($e)) {
                    $error = $e;
                }
                array_push($tokens, $t);
            }, $finalize);
            if ($error) throw $error;
            return $tokens;
        };
    }
private function initTokenize() 
    {
        $this->_tokenize = function ($input, $finalize) {
            // If the input is a string, continuously emit tokens through the callback until the end
            if (!isset($this->input))
                $this->input = "";
            $this->input .= $input;
            $tokens = [];
            $error = "";
            $this->input = $this->tokenizeToEnd(function ($e, $t) use (&$tokens,&$error) {
                if (isset($e)) {
                    $error = $e;
                }
                array_push($tokens, $t);
            }, $finalize);
            if ($error) throw $error;
            return $tokens;
        };
    }
private function initTokenize ( ) { $ this -> _tokenize = function ( $ input , $ finalize ) { // If the input is a string, continuously emit tokens through the callback until the end if ( ! isset ( $ this -> input ) ) $ this -> input = "" ; $ this -> input .= $ input ; $ tokens = [ ] ; $ error = "" ; $ this -> input = $ this -> tokenizeToEnd ( function ( $ e , $ t ) use ( & $ tokens , & $ error ) { if ( isset ( $ e ) ) { $ error = $ e ; } array_push ( $ tokens , $ t ) ; } , $ finalize ) ; if ( $ error ) throw $ error ; return $ tokens ; } ; }
When handling tokenize as a variable, we can hotswap its functionality when dealing with various serializations
When handling tokenize as a variable we can hotswap its functionality when dealing with various serializations