function(rules, flag) {
    flag = flag ? "g" + flag : "g";
    this.rules = rules;

    this.regExps = {};
    this.matchMappings = {};
    for ( var key in this.rules) {
        var rule = this.rules[key];
        var state = rule;
        var ruleRegExps = [];
        var matchTotal = 0;
        var mapping = this.matchMappings[key] = {};

        for ( var i = 0; i < state.length; i++) {

            if (state[i].regex instanceof RegExp)
                state[i].regex = state[i].regex.toString().slice(1, -1);

            // Count number of matching groups. 2 extra groups from the full match
            // And the catch-all on the end (used to force a match);
            var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2;

            // Replace any backreferences and offset appropriately.
            var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) {
                return "\\" + (parseInt(digit, 10) + matchTotal + 1);
            });

            if (matchcount > 1 && state[i].token.length !== matchcount-1)
                throw new Error("Matching groups and length of the token array don't match in rule #" + i + " of state " + key);

            mapping[matchTotal] = {
                rule: i,
                len: matchcount
            };
            matchTotal += matchcount;

            ruleRegExps.push(adjustedregex);
        }

        this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", flag);
    }
}
function(rules, flag) {
    flag = flag ? "g" + flag : "g";
    this.rules = rules;

    this.regExps = {};
    this.matchMappings = {};
    for ( var key in this.rules) {
        var rule = this.rules[key];
        var state = rule;
        var ruleRegExps = [];
        var matchTotal = 0;
        var mapping = this.matchMappings[key] = {};

        for ( var i = 0; i < state.length; i++) {

            if (state[i].regex instanceof RegExp)
                state[i].regex = state[i].regex.toString().slice(1, -1);

            
            
            var matchcount = new RegExp("(?:(" + state[i].regex + ")|(.))").exec("a").length - 2;

            
            var adjustedregex = state[i].regex.replace(/\\([0-9]+)/g, function (match, digit) {
                return "\\" + (parseInt(digit, 10) + matchTotal + 1);
            });

            if (matchcount > 1 && state[i].token.length !== matchcount-1)
                throw new Error("Matching groups and length of the token array don't match in rule #" + i + " of state " + key);

            mapping[matchTotal] = {
                rule: i,
                len: matchcount
            };
            matchTotal += matchcount;

            ruleRegExps.push(adjustedregex);
        }

        this.regExps[key] = new RegExp("(?:(" + ruleRegExps.join(")|(") + ")|(.))", flag);
    }
}
function ( rules , flag ) { flag = flag ? "g" + flag : "g" ; this . rules = rules ; this . regExps = { } ; this . matchMappings = { } ; for ( var key in this . rules ) { var rule = this . rules [ key ] ; var state = rule ; var ruleRegExps = [ ] ; var matchTotal = 0 ; var mapping = this . matchMappings [ key ] = { } ; for ( var i = 0 ; i < state . length ; i ++ ) { if ( state [ i ] . regex instanceof RegExp ) state [ i ] . regex = state [ i ] . regex . toString ( ) . slice ( 1 , - 1 ) ; // Count number of matching groups. 2 extra groups from the full match // And the catch-all on the end (used to force a match); var matchcount = new RegExp ( "(?:(" + state [ i ] . regex + ")|(.))" ) . exec ( "a" ) . length - 2 ; // Replace any backreferences and offset appropriately. var adjustedregex = state [ i ] . regex . replace ( / \\([0-9]+) / g , function ( match , digit ) { return "\\" + ( parseInt ( digit , 10 ) + matchTotal + 1 ) ; } ) ; if ( matchcount > 1 && state [ i ] . token . length !== matchcount - 1 ) throw new Error ( "Matching groups and length of the token array don't match in rule #" + i + " of state " + key ) ; mapping [ matchTotal ] = { rule : i , len : matchcount } ; matchTotal += matchcount ; ruleRegExps . push ( adjustedregex ) ; } this . regExps [ key ] = new RegExp ( "(?:(" + ruleRegExps . join ( ")|(" ) + ")|(.))" , flag ) ; } }
class Tokenizer

This class takes a set of highlighting rules, and creates a tokenizer out of them. For more information, see [the wiki on extending highlighters](https://github.com/ajaxorg/ace/wiki/Creating-or-Extending-an-Edit-Mode#wiki-extendingTheHighlighter).

 
new Tokenizer(rules, flag)
- rules (Object): The highlighting rules
- flag (String): Any additional regular expression flags to pass (like "i" for case insensitive)

Constructs a new tokenizer based on the given rules and flags.
class Tokenizer