--[[ This file is part of parser.lua - table based parsing Copyright (C) 2019 Soni L. This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . --]] local parser = require "parser" local caseno = 0 local function case() caseno = caseno + 1 return caseno end do -- basic check local case = case() local defs = {} local count = 0 local state, err = parser.parse(defs, function() assert(count == 0, "should be called only once"); count = count + 1 return nil end) assert(state) end -- basic check do -- trim left spaces local defs = {} defs.self = defs defs[' '] = "whitespace" defs['\n'] = "whitespace" defs['\r'] = "whitespace" defs['\t'] = "whitespace" defs['\f'] = "whitespace" defs['\v'] = "whitespace" defs.whitespace = "self" defs[parser.FALLBACK] = function(state, token) state[#state + 1] = token if #state > 20 then state[1] = table.concat(state) for i=#state, 2, -1 do state[i] = nil end end return "start" end defs.start = {} defs.start.self = defs.start defs.start[parser.FALLBACK] = function(state, token) state[#state + 1] = token if #state > 20 then state[1] = table.concat(state) for i=#state, 2, -1 do state[i] = nil end end return "self" end for k,v in ipairs({"hello", " hello", "\t \v \n\r hello"}) do local state, err = parser.parse(defs, v) local case = case() if not state then print(case, err) else assert(table.concat(state) == "hello") end end end -- trim left spaces do -- lua tokens local luatokens = require "luatokens" local tokens = luatokens.defs local state, err, etoken, estate = parser.parse(tokens, [["hello world"]]) local case = case() if not state then print(case, "---- IN TOKENS ----") print(case, err, etoken) for i,v in pairs(estate) do print(case, i, v) end print(case, "---- OUT TOKENS ----") else assert(state[1] == luatokens.tokens.TK_STRING) assert(state[2] == "hello world") assert(state[3] == nil) assert(state.line == 1 or not state.line) end end -- lua tokens do -- more lua tokens local luatokens = require "luatokens" local tokens = luatokens.defs local state, err, etoken, estate = parser.parse(tokens, [["\a\b\f\n\r\t\v\\\"\'\z \x41\65\ "]]) local case = case() if not state then print(case, "---- IN TOKENS ----") print(case, err, etoken) for i,v in pairs(estate) do print(case, i, v) end print(case, "---- OUT TOKENS ----") else assert(state[1] == luatokens.tokens.TK_STRING) assert(state[2] == "\7\8\12\10\13\9\11\92\34\39\65\65\10") assert(state[3] == nil) assert(state.line == 2) end end -- lua tokens do -- even more lua tokens local luatokens = require "luatokens" local tokens = luatokens.defs local state, err, etoken, estate = parser.parse(tokens, [["\u{000000000000000000000000000000000000000000000000000000000000041}"]]) local case = case() if not state then print(case, "---- IN TOKENS ----") print(case, err, etoken) for i,v in pairs(estate) do print(case, i, v) end print(case, "---- OUT TOKENS ----") else assert(state[1] == luatokens.tokens.TK_STRING) assert(state[2] == "A") assert(state[3] == nil) assert(state.line == 1 or not state.line) end end -- lua tokens do -- even more lua tokens local luatokens = require "luatokens" local tokens = luatokens.defs local state, err, etoken, estate = parser.parse(tokens, [["\u{7F}""\u{80}""\u{7FF}""\u{800}""\u{FFFF}""\u{10000}""\u{1FFFFF}""\u{200000}""\u{3FFFFFF}""\u{4000000}""\u{7FFFFFFF}"]]) local case = case() if not state then print(case, "---- IN TOKENS ----") print(case, err, etoken) for i,v in pairs(estate) do print(case, i, v) end print(case, "---- OUT TOKENS ----") else assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "\127") assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "\194\128") assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "\223\191") assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "\224\160\128") assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "\239\191\191") assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "\240\144\128\128") assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "\247\191\191\191") assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "\248\136\128\128\128") assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "\251\191\191\191\191") assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "\252\132\128\128\128\128") assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "\253\191\191\191\191\191") assert(table.remove(state, 1) == nil) assert(state.line == 1 or not state.line) end end -- lua tokens do -- simple lua tokens local luatokens = require "luatokens" local tokens = luatokens.defs local state, err, etoken, estate = parser.parse(tokens, [[[""]]) local case = case() if not state then print(case, "---- IN TOKENS ----") print(case, err, etoken) for i,v in pairs(estate) do print(case, i, v) end print(case, "---- OUT TOKENS ----") else assert(table.remove(state, 1) == "[") assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "") assert(table.remove(state, 1) == nil) assert(state.line == 1 or not state.line) end end -- lua tokens do -- simple long string local luatokens = require "luatokens" local tokens = luatokens.defs local state, err, etoken, estate = parser.parse(tokens, [=[[[]]]=]) local case = case() if not state then print(case, "---- IN TOKENS ----") print(case, err, etoken) for i,v in pairs(estate) do print(case, i, v) end print(case, "---- OUT TOKENS ----") else assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "") assert(table.remove(state, 1) == nil) assert(state.line == 1 or not state.line) end end -- long string do -- long string with depth 1 local luatokens = require "luatokens" local tokens = luatokens.defs local state, err, etoken, estate = parser.parse(tokens, [==[[=[]=]]==]) local case = case() if not state then print(case, "---- IN TOKENS ----") print(case, err, etoken) for i,v in pairs(estate) do print(case, i, v) end print(case, "---- OUT TOKENS ----") else assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "") assert(table.remove(state, 1) == nil) assert(state.line == 1 or not state.line) end end -- long string do -- long string with "nested" long string local luatokens = require "luatokens" local tokens = luatokens.defs local state, err, etoken, estate = parser.parse(tokens, [==[[=[[[]]]=]]==]) local case = case() if not state then print(case, "---- IN TOKENS ----") print(case, err, etoken) for i,v in pairs(estate) do print(case, i, v) end print(case, "---- OUT TOKENS ----") else assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "[[]]") assert(table.remove(state, 1) == nil) assert(state.line == 1 or not state.line) end end -- long string do -- long string edge cases local luatokens = require "luatokens" local tokens = luatokens.defs local state, err, etoken, estate = parser.parse(tokens, "[==[]=]==][==[]]==][=[] ]=][[\n]][[\n ]]") local case = case() if not state then print(case, "---- IN TOKENS ----") print(case, err, etoken) for i,v in pairs(estate) do print(case, i, v) end print(case, "---- OUT TOKENS ----") else assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "]=") assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "]") assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "] ") assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == "") assert(table.remove(state, 1) == luatokens.tokens.TK_STRING) assert(table.remove(state, 1) == " ") assert(table.remove(state, 1) == nil) assert(state.line == 3 or not state.line) end end -- long string do -- keywords local luatokens = require "luatokens" local tokens = luatokens.defs local state, err, etoken, estate = parser.parse(tokens, [[ and break do else elseif end false for function goto if in local nil not or repeat return then true until while]]) local case = case() if not state then print(case, "---- IN TOKENS ----") print(case, err, etoken) for i,v in pairs(estate) do print(case, i, v) end print(case, "---- OUT TOKENS ----") else assert(table.remove(state, 1) == luatokens.tokens.TK_AND) assert(table.remove(state, 1) == luatokens.tokens.TK_BREAK) assert(table.remove(state, 1) == luatokens.tokens.TK_DO) assert(table.remove(state, 1) == luatokens.tokens.TK_ELSE) assert(table.remove(state, 1) == luatokens.tokens.TK_ELSEIF) assert(table.remove(state, 1) == luatokens.tokens.TK_END) assert(table.remove(state, 1) == luatokens.tokens.TK_FALSE) assert(table.remove(state, 1) == luatokens.tokens.TK_FOR) assert(table.remove(state, 1) == luatokens.tokens.TK_FUNCTION) assert(table.remove(state, 1) == luatokens.tokens.TK_GOTO) assert(table.remove(state, 1) == luatokens.tokens.TK_IF) assert(table.remove(state, 1) == luatokens.tokens.TK_IN) assert(table.remove(state, 1) == luatokens.tokens.TK_LOCAL) assert(table.remove(state, 1) == luatokens.tokens.TK_NIL) assert(table.remove(state, 1) == luatokens.tokens.TK_NOT) assert(table.remove(state, 1) == luatokens.tokens.TK_OR) assert(table.remove(state, 1) == luatokens.tokens.TK_REPEAT) assert(table.remove(state, 1) == luatokens.tokens.TK_RETURN) assert(table.remove(state, 1) == luatokens.tokens.TK_THEN) assert(table.remove(state, 1) == luatokens.tokens.TK_TRUE) assert(table.remove(state, 1) == luatokens.tokens.TK_UNTIL) assert(table.remove(state, 1) == luatokens.tokens.TK_WHILE) assert(table.remove(state, 1) == nil) assert(state.line == 4) end end -- keywords do -- "other tokens" local luatokens = require "luatokens" local tokens = luatokens.defs local state, err, etoken, estate = parser.parse(tokens, [[ + - * / % ^ # & ~ | << >> // == ~= <= >= < > = ( ) { } [ ] :: ; : , . .. ...]]) local case = case() if not state then print(case, "---- IN TOKENS ----") print(case, err, etoken) for i,v in pairs(estate) do print(case, i, v) end print(case, "---- OUT TOKENS ----") else assert(table.remove(state, 1) == "+") assert(table.remove(state, 1) == "-") assert(table.remove(state, 1) == "*") assert(table.remove(state, 1) == "/") assert(table.remove(state, 1) == "%") assert(table.remove(state, 1) == "^") assert(table.remove(state, 1) == "#") assert(table.remove(state, 1) == "&") assert(table.remove(state, 1) == "~") assert(table.remove(state, 1) == "|") assert(table.remove(state, 1) == luatokens.tokens.TK_SHL) assert(table.remove(state, 1) == luatokens.tokens.TK_SHR) assert(table.remove(state, 1) == luatokens.tokens.TK_IDIV) assert(table.remove(state, 1) == luatokens.tokens.TK_EQ) assert(table.remove(state, 1) == luatokens.tokens.TK_NE) assert(table.remove(state, 1) == luatokens.tokens.TK_LE) assert(table.remove(state, 1) == luatokens.tokens.TK_GE) assert(table.remove(state, 1) == "<") assert(table.remove(state, 1) == ">") assert(table.remove(state, 1) == "=") assert(table.remove(state, 1) == "(") assert(table.remove(state, 1) == ")") assert(table.remove(state, 1) == "{") assert(table.remove(state, 1) == "}") assert(table.remove(state, 1) == "[") assert(table.remove(state, 1) == "]") assert(table.remove(state, 1) == luatokens.tokens.TK_DBCOLON) assert(table.remove(state, 1) == ";") assert(table.remove(state, 1) == ":") assert(table.remove(state, 1) == ",") assert(table.remove(state, 1) == ".") assert(table.remove(state, 1) == luatokens.tokens.TK_CONCAT) assert(table.remove(state, 1) == luatokens.tokens.TK_DOTS) assert(table.remove(state, 1) == nil) assert(state.line == 5) end end -- "other tokens" do -- long comments local luatokens = require "luatokens" local tokens = luatokens.defs local state, err, etoken, estate = parser.parse(tokens, [==[--[[ --]]]==]) local case = case() if not state then print(case, "---- IN TOKENS ----") print(case, err, etoken) for i,v in pairs(estate) do print(case, i, v) end print(case, "---- OUT TOKENS ----") else assert(table.remove(state, 1) == nil) assert(state.line == 3) end end -- long comments do -- numbers local luatokens = require "luatokens" local tokens = luatokens.defs local state, err, etoken, estate = parser.parse(tokens, [[ 3 345 0xff 0xBEBADA 3.0 3.1416 314.16e-2 0.31416E1 34e1 0x0.1E 0xA23p-4 0X1.921FB54442D18P+1]]) local case = case() if not state then print(case, "---- IN TOKENS ----") print(case, err, etoken) for i,v in pairs(estate) do print(case, i, v) end print(case, "---- OUT TOKENS ----") else -- integers assert(table.remove(state, 1) == luatokens.tokens.TK_INT) assert(table.remove(state, 1) == 3) assert(table.remove(state, 1) == luatokens.tokens.TK_INT) assert(table.remove(state, 1) == 345) assert(table.remove(state, 1) == luatokens.tokens.TK_INT) assert(table.remove(state, 1) == 0xff) assert(table.remove(state, 1) == luatokens.tokens.TK_INT) assert(table.remove(state, 1) == 0xBEBADA) -- floats assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) assert(table.remove(state, 1) == 3.0) assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) assert(table.remove(state, 1) == 3.1416) assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) assert(table.remove(state, 1) == 314.16e-2) assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) assert(table.remove(state, 1) == 0.31416E1) assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) assert(table.remove(state, 1) == 34e1) assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) assert(table.remove(state, 1) == 0.1171875) assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) assert(table.remove(state, 1) == 162.1875) assert(table.remove(state, 1) == luatokens.tokens.TK_FLT) assert(table.remove(state, 1) == math.pi) assert(table.remove(state, 1) == nil) assert(state.line == 3) end end -- numbers do -- FUCK local luatokens = require "luatokens" local luatokens_file = io.open("./luatokens.lua", "r") local tokens = luatokens.defs local state, err, etoken, estate = parser.parse(tokens, function() return luatokens_file:read(8192) end) local case = case() if not state then print(case, "---- IN TOKENS ----") print(case, err, etoken) for i,v in pairs(estate) do v = luatokens.reverse_keywords[v] or luatokens.reverse_tokens[v] or v print(case, i, v) end print(case, "---- OUT TOKENS ----") end end -- FUCK