lib/SqlFormatter.php
protected static function tokenize($string)
    {
        self::init();

        $tokens = array();

        // Used for debugging if there is an error while tokenizing the string
        $original_length = strlen($string);

        // Used to make sure the string keeps shrinking on each iteration
        $old_string_len = strlen($string) + 1;

        $token = null;

        $current_length = strlen($string);

        // Keep processing the string until it is empty
        while ($current_length) {
            // If the string stopped shrinking, there was a problem
            if ($old_string_len <= $current_length) {
                $tokens[] = array(
                    self::TOKEN_VALUE=>$string,
                    self::TOKEN_TYPE=>self::TOKEN_TYPE_ERROR
                );

                return $tokens;
            }
            $old_string_len =  $current_length;

            // Determine if we can use caching
            if ($current_length >= self::$max_cachekey_size) {
                $cacheKey = substr($string,0,self::$max_cachekey_size);
            } else {
                $cacheKey = false;
            }

            // See if the token is already cached
            if ($cacheKey && isset(self::$token_cache[$cacheKey])) {
                // Retrieve from cache
                $token = self::$token_cache[$cacheKey];
                $token_length = strlen($token[self::TOKEN_VALUE]);
                self::$cache_hits++;
            } else {
                // Get the next token and the token type
                $token = self::getNextToken($string, $token);
                $token_length = strlen($token[self::TOKEN_VALUE]);
                self::$cache_misses++;

                // If the token is shorter than the max length, store it in cache
                if ($cacheKey && $token_length < self::$max_cachekey_size) {
                    self::$token_cache[$cacheKey] = $token;
                }
            }

            $tokens[] = $token;

            // Advance the string
            $string = substr($string, $token_length);

            $current_length -= $token_length;
        }

        return $tokens;
    }
protected static function tokenize($string)
    {
        self::init();

        $tokens = array();

        // Used for debugging if there is an error while tokenizing the string
        $original_length = strlen($string);

        // Used to make sure the string keeps shrinking on each iteration
        $old_string_len = strlen($string) + 1;

        $token = null;

        $current_length = strlen($string);

        // Keep processing the string until it is empty
        while ($current_length) {
            // If the string stopped shrinking, there was a problem
            if ($old_string_len <= $current_length) {
                $tokens[] = array(
                    self::TOKEN_VALUE=>$string,
                    self::TOKEN_TYPE=>self::TOKEN_TYPE_ERROR
                );

                return $tokens;
            }
            $old_string_len =  $current_length;

            // Determine if we can use caching
            if ($current_length >= self::$max_cachekey_size) {
                $cacheKey = substr($string,0,self::$max_cachekey_size);
            } else {
                $cacheKey = false;
            }

            // See if the token is already cached
            if ($cacheKey && isset(self::$token_cache[$cacheKey])) {
                // Retrieve from cache
                $token = self::$token_cache[$cacheKey];
                $token_length = strlen($token[self::TOKEN_VALUE]);
                self::$cache_hits++;
            } else {
                // Get the next token and the token type
                $token = self::getNextToken($string, $token);
                $token_length = strlen($token[self::TOKEN_VALUE]);
                self::$cache_misses++;

                // If the token is shorter than the max length, store it in cache
                if ($cacheKey && $token_length < self::$max_cachekey_size) {
                    self::$token_cache[$cacheKey] = $token;
                }
            }

            $tokens[] = $token;

            // Advance the string
            $string = substr($string, $token_length);

            $current_length -= $token_length;
        }

        return $tokens;
    }
protected static function tokenize ( $ string ) { self :: init ( ) ; $ tokens = array ( ) ; // Used for debugging if there is an error while tokenizing the string $ original_length = strlen ( $ string ) ; // Used to make sure the string keeps shrinking on each iteration $ old_string_len = strlen ( $ string ) + 1 ; $ token = null ; $ current_length = strlen ( $ string ) ; // Keep processing the string until it is empty while ( $ current_length ) { // If the string stopped shrinking, there was a problem if ( $ old_string_len <= $ current_length ) { $ tokens [ ] = array ( self :: TOKEN_VALUE => $ string , self :: TOKEN_TYPE => self :: TOKEN_TYPE_ERROR ) ; return $ tokens ; } $ old_string_len = $ current_length ; // Determine if we can use caching if ( $ current_length >= self :: $ max_cachekey_size ) { $ cacheKey = substr ( $ string , 0 , self :: $ max_cachekey_size ) ; } else { $ cacheKey = false ; } // See if the token is already cached if ( $ cacheKey && isset ( self :: $ token_cache [ $ cacheKey ] ) ) { // Retrieve from cache $ token = self :: $ token_cache [ $ cacheKey ] ; $ token_length = strlen ( $ token [ self :: TOKEN_VALUE ] ) ; self :: $ cache_hits ++ ; } else { // Get the next token and the token type $ token = self :: getNextToken ( $ string , $ token ) ; $ token_length = strlen ( $ token [ self :: TOKEN_VALUE ] ) ; self :: $ cache_misses ++ ; // If the token is shorter than the max length, store it in cache if ( $ cacheKey && $ token_length < self :: $ max_cachekey_size ) { self :: $ token_cache [ $ cacheKey ] = $ token ; } } $ tokens [ ] = $ token ; // Advance the string $ string = substr ( $ string , $ token_length ) ; $ current_length -= $ token_length ; } return $ tokens ; }
Takes a SQL string and breaks it into tokens.
Each token is an associative array with type and value.

@param String $string The SQL string

@return Array An array of tokens.
Takes a SQL string and breaks it into tokens . Each token is an associative array with type and value .