local f=string.find local A=string.sub local Z=table.insert local function C(f,A,Z)if type(A)~=Z then error("\097\114\103\117\109\101\110\116 "..(f..(" \109\117\115\116 \098\101 "..Z)),2)end end local O={}local b="\094\091\037\043\037-\093\063\037\100\043\037\046\063\037\100\042\091\101\069\093\091\037\043\037-\093\063\037\100\043"local I="\094\091\037\043\037-\093\063\037\100\042\037\046\037\100\043\091\101\069\093\091\037\043\037-\093\063\037\100\043"local h="\094\091\037\043\037-\093\063\037\100\043\037\046\063\037\100\042"local L="\094\091\037\043\037-\093\063\037\100\042\037\046\037\100\043"local W="\094\048\120\091\037\100\097-\102\065-\070\093\043"local G="\094\037\100\043\037\046\063\037\100\042\091\101\069\093\091\037\043\037-\093\063\037\100\043"local o="\094\037\100\042\037\046\037\100\043\091\101\069\093\091\037\043\037-\093\063\037\100\043"local R="\094\037\100\043\037\046\063\037\100\042"local m="\094\037\100\042\037\046\037\100\043"local q="\094\091\037\097\095\093\091\037\119\095\093\042"local l="\094\037\115\043"local F="\094\040\091\'\"\093\041\037\049"local j="\094\040\091\'\"\093\041\040\092\042\041\037\050\037\049"local P="\094\040\091\'\"\093\041\046-\091\094\092\093\040\092\042\041\037\050\037\049"local d="\094\'\'"local i="\094\'\040\092\042\041\037\049\'"local e="\094\'\046-\091\094\092\093\040\092\042\041\037\049\'"local M="\094\035\046-\091\094\092\093\n"local V,s,T,z,D local function U(f)return f,f end local function n(f,A)if A and A.number then f=tonumber(f)end return"\110\117\109\098\101\114",f end local function N(f,A)if A and A.string then f=f:sub(2,-2)end return"\115\116\114\105\110\103",f end local function K(f,A,Z)if A and A.string then local A=3 if Z[3]then A=A+Z[3]:len()end f=f:sub(A,-A)if f:sub(1,1)=="\n"then f=f:sub(2)end end return"\115\116\114\105\110\103",f end local function H(f,A)if A and A.string then f=f:sub(2,-2)end return"\099\104\097\114",f end local function v(f)return"\099\111\109\109\101\110\116",f end local function k(f)return"\115\112\097\099\101",f end local function B(f)return"\112\114\101\112\114\111",f end local function J(f)return"\105\100\101\110",f end local function S(f)if z[f]then return"\107\101\121\119\111\114\100",f else return"\105\100\101\110",f end end local function t(f)if D[f]then return"\107\101\121\119\111\114\100",f else return"\105\100\101\110",f end end function O.scan(Z,C,G,o)local R=type(Z)~="\115\116\114\105\110\103"and Z G=G or{space=true}o=o or{number=true;string=true}if G then if G.space then G[k]=true end if G.comments then G[v]=true end end if not C then if not V then V={{l;k},{W;n},{q,J},{b;n},{I;n},{h;n},{L,n},{F,N},{j,N},{P,N},{"\094\046";U}}end C=V end local m=0 local d=R and R:read()local i=R and 0 or#Z local e=1 local M local s local T=true local function z(b)local I=type(b)if s then local f=s[M]if f then M=M+1 return f[1],f[2]else s=nil end end if I=="\115\116\114\105\110\103"then local C,O=f(Z,b,e)if C then local f=A(Z,C,O)e=O+1 return"",f else e=i+1 return"",""end elseif I=="\116\097\098\108\101"then M=1 s=b return"",""elseif I~="\110\105\108"then return m,e else if T then if not R then m=1 end T=false end if e>i then if R then if not d then return end Z=d m=m+1 d=R:read()if d then Z=Z.."\n"end e,i=1,#Z else return end end for C,b in ipairs(C)do local I=b[1]local h=b[2]local L={f(Z,I,e)}local W,q=L[1],L[2]if W then local f=A(Z,W,q)e=q+1 local C,b if not(G and G[h])then O.finished=e>i C,b=h(f,o,L)end if not R and f:find("\n")then local A,Z=f:gsub("\n",{})m=m+Z end if C then return C,b else return z()end end end end end return z end local function a(f)return type(f)=="\115\116\114\105\110\103"end function O.insert(f,A,C)if not A then return end local O if a(A)and a(C)then O={{A;C}}elseif type(A)=="\102\117\110\099\116\105\111\110"then O={}for f,A in A()do Z(O,{f,A})end else O=A end f(O)end function O.getline(f)local A,Z=f("\046-\n")return Z end function O.lineno(f)return f(0)end function O.getrest(f)local A,Z=f("\046\043")return Z end function O.get_keywords()if not z then z={["\097\110\100"]=true,["\098\114\101\097\107"]=true;["\100\111"]=true,["\101\108\115\101"]=true,["\101\108\115\101\105\102"]=true,["\101\110\100"]=true,["\102\097\108\115\101"]=true,["\102\111\114"]=true;["\102\117\110\099\116\105\111\110"]=true;["\105\102"]=true;["\105\110"]=true;["\108\111\099\097\108"]=true,["\110\105\108"]=true,["\110\111\116"]=true;["\111\114"]=true;["\114\101\112\101\097\116"]=true,["\114\101\116\117\114\110"]=true,["\116\104\101\110"]=true,["\116\114\117\101"]=true;["\117\110\116\105\108"]=true,["\119\104\105\108\101"]=true}end return z end function O.lua(f,A,Z)A=A or{space=true;comments=true}O.get_keywords()if not s then s={{l;k},{W,n},{q;S},{G,n};{o,n},{R,n},{m,n};{F;N};{j,N};{P,N},{"\094\037-\037-\037\091\040\061\042\041\037\091\046-\037\093\037\049\037\093",v},{"\094\037-\037-\046-\n";v},{"\094\037\091\040\061\042\041\037\091\046-\037\093\037\049\037\093";K},{"\094\061\061",U},{"\094~\061",U};{"\094\060\061",U},{"\094\062\061";U},{"\094\037\046\037\046\037\046",U},{"\094\037\046\037\046";U},{"\094\046";U}}end return O.scan(f,s,A,Z)end function O.cpp(f,A,Z)A=A or{space=true;comments=true}if not D then D={class=true,["\098\114\101\097\107"]=true,["\100\111"]=true;sizeof=true;["\101\108\115\101"]=true,["\099\111\110\116\105\110\117\101"]=true;struct=true;["\102\097\108\115\101"]=true,["\102\111\114"]=true,public=true;void=true;private=true,protected=true;goto=true;["\105\102"]=true;static=true,const=true;typedef=true,enum=true;char=true,int=true;bool=true;long=true;float=true;["\116\114\117\101"]=true;delete=true;double=true,["\119\104\105\108\101"]=true;new=true;namespace=true,try=true;catch=true;switch=true,case=true;extern=true;["\114\101\116\117\114\110"]=true,default=true;unsigned=true,signed=true;union=true;volatile=true;register=true;short=true}end if not T then T={{l,k};{M;B},{W,n};{q;t},{G,n},{o,n};{R,n};{m;n},{d;H};{i;H};{e,H};{F;N};{j,N};{P;N},{"\094\047\047\046-\n",v};{"\094\047\037\042\046-\037\042\047";v};{"\094\061\061",U};{"\094\033\061";U},{"\094\060\061";U},{"\094\062\061";U};{"\094-\062",U},{"\094\038\038";U};{"\094\124\124";U},{"\094\037\043\037\043",U},{"\094\037-\037-",U},{"\094\037\043\061",U},{"\094\037-\061",U};{"\094\037\042\061",U};{"\094\047\061",U},{"\094\124\061",U},{"\094\037\094\061",U};{"\094\058\058",U},{"\094\046";U}}end return O.scan(f,T,A,Z)end function O.get_separated_list(f,A,C)A=A or"\041"C=C or"\044"local O={}local b=1 local I={}local function h(f,A,C)C=C or A Z(f,{A,C})end local L if A=="\n"then L=function(f,A)return f=="\115\112\097\099\101"and A:find("\n")end else L=function(f)return f==A end end local W,G while true do W,G=f()if not W then return nil,"\069\079\083"end if L(W,G)and b==1 then Z(O,I)break elseif W=="\040"then b=b+1 h(I,"\040")elseif W=="\041"then b=b-1 if b==0 then Z(O,I)break else h(I,"\041")end elseif W==C and b==1 then Z(O,I)I={}else h(I,W,G)end end return O,{W,G}end function O.skipws(f)local A,Z=f()while A=="\115\112\097\099\101"do A,Z=f()end return A,Z end local Q=O.skipws function O.expecting(f,A,Z)C(1,f,"\102\117\110\099\116\105\111\110")C(2,A,"\115\116\114\105\110\103")local O,b if Z then O,b=f()else O,b=Q(f)end if O~=A then error("\101\120\112\101\099\116\105\110\103 "..A,2)end return b end return O
--- Lexical scanner for creating a sequence of tokens from text.
-- `lexer.scan(s)` returns an iterator over all tokens found in the
-- string `s`. This iterator returns two values, a token type string
-- (such as 'string' for quoted string, 'iden' for identifier) and the value of the
-- token.
--
-- Versions specialized for Lua and C are available; these also handle block comments
-- and classify keywords as 'keyword' tokens. For example:
--
--    > s = 'for i=1,n do'
--    > for t,v in lexer.lua(s)  do print(t,v) end
--    keyword for
--    iden    i
--    =       =
--    number  1
--    ,       ,
--    iden    n
--    keyword do
--
-- See the Guide for further @{06-data.md.Lexical_Scanning|discussion}
-- @module pl.lexer

local strfind = string.find
local strsub = string.sub
local append = table.insert


local function assert_arg(idx,val,tp)
    if type(val) ~= tp then
        error("argument "..idx.." must be "..tp, 2)
    end
end

local lexer = {}

local NUMBER1  = '^[%+%-]?%d+%.?%d*[eE][%+%-]?%d+'
local NUMBER1a = '^[%+%-]?%d*%.%d+[eE][%+%-]?%d+'
local NUMBER2  = '^[%+%-]?%d+%.?%d*'
local NUMBER2a = '^[%+%-]?%d*%.%d+'
local NUMBER3  = '^0x[%da-fA-F]+'
local NUMBER4  = '^%d+%.?%d*[eE][%+%-]?%d+'
local NUMBER4a = '^%d*%.%d+[eE][%+%-]?%d+'
local NUMBER5  = '^%d+%.?%d*'
local NUMBER5a = '^%d*%.%d+'
local IDEN = '^[%a_][%w_]*'
local WSPACE = '^%s+'
local STRING1 = "^(['\"])%1" -- empty string
local STRING2 = [[^(['"])(\*)%2%1]]
local STRING3 = [[^(['"]).-[^\](\*)%2%1]]
local CHAR1 = "^''"
local CHAR2 = [[^'(\*)%1']]
local CHAR3 = [[^'.-[^\](\*)%1']]
local PREPRO = '^#.-[^\\]\n'

local plain_matches,lua_matches,cpp_matches,lua_keyword,cpp_keyword

local function tdump(tok)
    return tok,tok
end

local function ndump(tok,options)
    if options and options.number then
        tok = tonumber(tok)
    end
    return "number",tok
end

-- regular strings, single or double quotes; usually we want them
-- without the quotes
local function sdump(tok,options)
    if options and options.string then
        tok = tok:sub(2,-2)
    end
    return "string",tok
end

-- long Lua strings need extra work to get rid of the quotes
local function sdump_l(tok,options,findres)
    if options and options.string then
        local quotelen = 3
        if findres[3] then
            quotelen = quotelen + findres[3]:len()
        end
        tok = tok:sub(quotelen, -quotelen)
        if tok:sub(1, 1) == "\n" then
            tok = tok:sub(2)
        end
    end
    return "string",tok
end

local function chdump(tok,options)
    if options and options.string then
        tok = tok:sub(2,-2)
    end
    return "char",tok
end

local function cdump(tok)
    return "comment",tok
end

local function wsdump (tok)
    return "space",tok
end

local function pdump (tok)
    return "prepro",tok
end

local function plain_vdump(tok)
    return "iden",tok
end

local function lua_vdump(tok)
    if lua_keyword[tok] then
        return "keyword",tok
    else
        return "iden",tok
    end
end

local function cpp_vdump(tok)
    if cpp_keyword[tok] then
        return "keyword",tok
    else
        return "iden",tok
    end
end

--- create a plain token iterator from a string or file-like object.
-- @tparam string|file s a string or a file-like object with `:read()` method returning lines.
-- @tab matches an optional match table - array of token descriptions.
-- A token is described by a `{pattern, action}` pair, where `pattern` should match
-- token body and `action` is a function called when a token of described type is found.
-- @tab[opt] filter a table of token types to exclude, by default `{space=true}`
-- @tab[opt] options a table of options; by default, `{number=true,string=true}`,
-- which means convert numbers and strip string quotes.
function lexer.scan(s,matches,filter,options)
    local file = type(s) ~= 'string' and s
    filter = filter or {space=true}
    options = options or {number=true,string=true}
    if filter then
        if filter.space then filter[wsdump] = true end
        if filter.comments then
            filter[cdump] = true
        end
    end
    if not matches then
        if not plain_matches then
            plain_matches = {
                {WSPACE,wsdump},
                {NUMBER3,ndump},
                {IDEN,plain_vdump},
                {NUMBER1,ndump},
                {NUMBER1a,ndump},
                {NUMBER2,ndump},
                {NUMBER2a,ndump},
                {STRING1,sdump},
                {STRING2,sdump},
                {STRING3,sdump},
                {'^.',tdump}
            }
        end
        matches = plain_matches
    end

    local line_nr = 0
    local next_line = file and file:read()
    local sz = file and 0 or #s
    local idx = 1

    local tlist_i
    local tlist

    local first_hit = true

    local function iter(res)
        local tp = type(res)

        if tlist then -- returning the inserted token list
            local cur = tlist[tlist_i]
            if cur then
                tlist_i = tlist_i + 1
                return cur[1], cur[2]
            else
                tlist = nil
            end
        end

        if tp == 'string' then -- search up to some special pattern
            local i1,i2 = strfind(s,res,idx)
            if i1 then
                local tok = strsub(s,i1,i2)
                idx = i2 + 1
                return '', tok
            else
                idx = sz + 1
                return '', ''
            end

        elseif tp == 'table' then -- insert a token list
            tlist_i = 1
            tlist = res
            return '', ''

        elseif tp ~= 'nil' then -- return position
            return line_nr, idx

        else -- look for next token
            if first_hit then
                if not file then line_nr = 1 end
                first_hit = false
            end

            if idx > sz then
                if file then
                    if not next_line then
                      return -- past the end of file, done
                    end
                    s = next_line
                    line_nr = line_nr + 1
                    next_line = file:read()
                    if next_line then
                        s = s .. '\n'
                    end
                    idx, sz = 1, #s
                else
                    return -- past the end of input, done
                end
            end

            for _,m in ipairs(matches) do
                local pat = m[1]
                local fun = m[2]
                local findres = {strfind(s,pat,idx)}
                local i1, i2 = findres[1], findres[2]
                if i1 then
                    local tok = strsub(s,i1,i2)
                    idx = i2 + 1
                    local ret1, ret2
                    if not (filter and filter[fun]) then
                        lexer.finished = idx > sz
                        ret1, ret2 = fun(tok, options, findres)
                    end
                    if not file and tok:find("\n") then
                        -- Update line number.
                        local _, newlines = tok:gsub("\n", {})
                        line_nr = line_nr + newlines
                    end
                    if ret1 then
                        return ret1, ret2 -- found a match
                    else
                        return iter() -- tail-call to try again
                    end
                end
            end
        end
    end

    return iter
end

local function isstring (s)
    return type(s) == 'string'
end

--- insert tokens into a stream.
-- @param tok a token stream
-- @param a1 a string is the type, a table is a token list and
-- a function is assumed to be a token-like iterator (returns type & value)
-- @string a2 a string is the value
function lexer.insert (tok,a1,a2)
    if not a1 then return end
    local ts
    if isstring(a1) and isstring(a2) then
        ts = {{a1,a2}}
    elseif type(a1) == 'function' then
        ts = {}
        for t,v in a1() do
            append(ts,{t,v})
        end
    else
        ts = a1
    end
    tok(ts)
end

--- get everything in a stream upto a newline.
-- @param tok a token stream
-- @return a string
function lexer.getline (tok)
    local _,v = tok('.-\n')
    return v
end

--- get current line number.
-- @param tok a token stream
-- @return the line number.
-- if the input source is a file-like object,
-- also return the column.
function lexer.lineno (tok)
    return tok(0)
end

--- get the rest of the stream.
-- @param tok a token stream
-- @return a string
function lexer.getrest (tok)
    local _,v = tok('.+')
    return v
end

--- get the Lua keywords as a set-like table.
-- So `res["and"]` etc would be `true`.
-- @return a table
function lexer.get_keywords ()
    if not lua_keyword then
        lua_keyword = {
            ["and"] = true, ["break"] = true,  ["do"] = true,
            ["else"] = true, ["elseif"] = true, ["end"] = true,
            ["false"] = true, ["for"] = true, ["function"] = true,
            ["if"] = true, ["in"] = true,  ["local"] = true, ["nil"] = true,
            ["not"] = true, ["or"] = true, ["repeat"] = true,
            ["return"] = true, ["then"] = true, ["true"] = true,
            ["until"] = true,  ["while"] = true
        }
    end
    return lua_keyword
end

--- create a Lua token iterator from a string or file-like object.
-- Will return the token type and value.
-- @string s the string
-- @tab[opt] filter a table of token types to exclude, by default `{space=true,comments=true}`
-- @tab[opt] options a table of options; by default, `{number=true,string=true}`,
-- which means convert numbers and strip string quotes.
function lexer.lua(s,filter,options)
    filter = filter or {space=true,comments=true}
    lexer.get_keywords()
    if not lua_matches then
        lua_matches = {
            {WSPACE,wsdump},
            {NUMBER3,ndump},
            {IDEN,lua_vdump},
            {NUMBER4,ndump},
            {NUMBER4a,ndump},
            {NUMBER5,ndump},
            {NUMBER5a,ndump},
            {STRING1,sdump},
            {STRING2,sdump},
            {STRING3,sdump},
            {'^%-%-%[(=*)%[.-%]%1%]',cdump},
            {'^%-%-.-\n',cdump},
            {'^%[(=*)%[.-%]%1%]',sdump_l},
            {'^==',tdump},
            {'^~=',tdump},
            {'^<=',tdump},
            {'^>=',tdump},
            {'^%.%.%.',tdump},
            {'^%.%.',tdump},
            {'^.',tdump}
        }
    end
    return lexer.scan(s,lua_matches,filter,options)
end

--- create a C/C++ token iterator from a string or file-like object.
-- Will return the token type type and value.
-- @string s the string
-- @tab[opt] filter a table of token types to exclude, by default `{space=true,comments=true}`
-- @tab[opt] options a table of options; by default, `{number=true,string=true}`,
-- which means convert numbers and strip string quotes.
function lexer.cpp(s,filter,options)
    filter = filter or {space=true,comments=true}
    if not cpp_keyword then
        cpp_keyword = {
            ["class"] = true, ["break"] = true,  ["do"] = true, ["sizeof"] = true,
            ["else"] = true, ["continue"] = true, ["struct"] = true,
            ["false"] = true, ["for"] = true, ["public"] = true, ["void"] = true,
            ["private"] = true, ["protected"] = true, ["goto"] = true,
            ["if"] = true, ["static"] = true,  ["const"] = true, ["typedef"] = true,
            ["enum"] = true, ["char"] = true, ["int"] = true, ["bool"] = true,
            ["long"] = true, ["float"] = true, ["true"] = true, ["delete"] = true,
            ["double"] = true,  ["while"] = true, ["new"] = true,
            ["namespace"] = true, ["try"] = true, ["catch"] = true,
            ["switch"] = true, ["case"] = true, ["extern"] = true,
            ["return"] = true,["default"] = true,['unsigned']  = true,['signed'] = true,
            ["union"] =  true, ["volatile"] = true, ["register"] = true,["short"] = true,
        }
    end
    if not cpp_matches then
        cpp_matches = {
            {WSPACE,wsdump},
            {PREPRO,pdump},
            {NUMBER3,ndump},
            {IDEN,cpp_vdump},
            {NUMBER4,ndump},
            {NUMBER4a,ndump},
            {NUMBER5,ndump},
            {NUMBER5a,ndump},
            {CHAR1,chdump},
            {CHAR2,chdump},
            {CHAR3,chdump},
            {STRING1,sdump},
            {STRING2,sdump},
            {STRING3,sdump},
            {'^//.-\n',cdump},
            {'^/%*.-%*/',cdump},
            {'^==',tdump},
            {'^!=',tdump},
            {'^<=',tdump},
            {'^>=',tdump},
            {'^->',tdump},
            {'^&&',tdump},
            {'^||',tdump},
            {'^%+%+',tdump},
            {'^%-%-',tdump},
            {'^%+=',tdump},
            {'^%-=',tdump},
            {'^%*=',tdump},
            {'^/=',tdump},
            {'^|=',tdump},
            {'^%^=',tdump},
            {'^::',tdump},
            {'^.',tdump}
        }
    end
    return lexer.scan(s,cpp_matches,filter,options)
end

--- get a list of parameters separated by a delimiter from a stream.
-- @param tok the token stream
-- @string[opt=')'] endtoken end of list. Can be '\n'
-- @string[opt=','] delim separator
-- @return a list of token lists.
function lexer.get_separated_list(tok,endtoken,delim)
    endtoken = endtoken or ')'
    delim = delim or ','
    local parm_values = {}
    local level = 1 -- used to count ( and )
    local tl = {}
    local function tappend (tl,t,val)
        val = val or t
        append(tl,{t,val})
    end
    local is_end
    if endtoken == '\n' then
        is_end = function(t,val)
            return t == 'space' and val:find '\n'
        end
    else
        is_end = function (t)
            return t == endtoken
        end
    end
    local token,value
    while true do
        token,value=tok()
        if not token then return nil,'EOS' end -- end of stream is an error!
        if is_end(token,value) and level == 1 then
            append(parm_values,tl)
            break
        elseif token == '(' then
            level = level + 1
            tappend(tl,'(')
        elseif token == ')' then
            level = level - 1
            if level == 0 then -- finished with parm list
                append(parm_values,tl)
                break
            else
                tappend(tl,')')
            end
        elseif token == delim and level == 1 then
            append(parm_values,tl) -- a new parm
            tl = {}
        else
            tappend(tl,token,value)
        end
    end
    return parm_values,{token,value}
end

--- get the next non-space token from the stream.
-- @param tok the token stream.
function lexer.skipws (tok)
    local t,v = tok()
    while t == 'space' do
        t,v = tok()
    end
    return t,v
end

local skipws = lexer.skipws

--- get the next token, which must be of the expected type.
-- Throws an error if this type does not match!
-- @param tok the token stream
-- @string expected_type the token type
-- @bool no_skip_ws whether we should skip whitespace
function lexer.expecting (tok,expected_type,no_skip_ws)
    assert_arg(1,tok,'function')
    assert_arg(2,expected_type,'string')
    local t,v
    if no_skip_ws then
        t,v = tok()
    else
        t,v = skipws(tok)
    end
    if t ~= expected_type then error ("expecting "..expected_type,2) end
    return v
end

return lexer
