It parses itself!

This commit is contained in:
SoniEx2 2019-04-09 06:56:37 -03:00
parent 893ca6ca0d
commit fefc2f3ed7
3 changed files with 111 additions and 8 deletions

View File

@ -581,11 +581,14 @@ defs["."] = setmetatable({
if token ~= "." then if token ~= "." then
if rule ~= "digit" then if rule ~= "digit" then
state[#state+1] = "." state[#state+1] = "."
else
error("NYI") -- TODO digit handling
end end
end end
end, end,
digit = function(state, token, rule)
state[#state+1] = TK_FLT
state[COLLECT] = {".", coalesce=31}
return "in_decimal"
end,
["."] = setmetatable({ ["."] = setmetatable({
[-1] = function(state, token, rule) [-1] = function(state, token, rule)
if token ~= "." then if token ~= "." then
@ -595,17 +598,82 @@ defs["."] = setmetatable({
["."] = function(state, token) ["."] = function(state, token)
state[#state+1] = TK_DOTS state[#state+1] = TK_DOTS
return "self" -- actually goes into defs return "self" -- actually goes into defs
end end,
}, {__index=defs}) }, {__index=defs})
}, {__index=defs}) }, {__index=defs})
function defs.digit(state, token) function defs.digit(state, token)
-- TODO state[COLLECT] = {token, coalesce=31}
if token == "0" then
return "in_zero"
else
return "in_integer"
end
end end
defs.in_digit = { defs.in_integer = setmetatable(selfify({
-- TODO hexdigit = "alpha",
} alpha = false,
['e'] = "exp",
['E'] = "exp",
exp = function(state, token)
collect_fallback(state, token)
return "in_exp"
end,
['.'] = function(state, token)
collect_fallback(state, token)
return "in_decimal"
end,
digit = function(state, token)
collect_fallback(state, token)
return "in_digit"
end,
}, "in_digit"), {__index=defs})
defs.in_zero = setmetatable({
['x'] = function(state, token)
collect_fallback(state, token)
return "in_hex"
end,
}, {__index=defs.in_integer})
defs.in_decimal = setmetatable(selfify({
['.'] = false,
}, "in_digit"), {__index=defs.in_integer})
defs.in_expnum = setmetatable(selfify({
exp = false,
}, "in_digit"), {__index=defs.in_decimal})
defs.in_subexp = setmetatable({
in_expnum = defs.in_expnum,
digit = function(state, token)
collect_fallback(state, token)
return "in_expnum"
end,
}, {__index=defs.base})
defs.in_exp = setmetatable({
["+"] = "sign",
["-"] = "sign",
sign = function(state, token)
collect_fallback(state, token)
return "in_subexp"
end,
}, {__index=defs.in_subexp})
defs.in_hex = setmetatable({
in_decimal = "in_hex_fraction",
hexdigit = 'digit',
['e'] = 'hexdigit',
['E'] = 'hexdigit',
['p'] = 'exp',
['P'] = 'exp',
}, {__index=defs.in_integer})
defs.in_hex_fraction = setmetatable({
['.'] = false,
}, {__index=defs.in_hex})
function defs.simpletoken(state, token) function defs.simpletoken(state, token)
state[#state+1] = token state[#state+1] = token

35
printtokens.lua Normal file
View File

@ -0,0 +1,35 @@
--[[
This file is part of luatokens.lua - pure-Lua Lua tokenizer
Copyright (C) 2019 Soni L.
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
--]]
local parser = require "parser"
do
local filename = arg[1]
local luatokens = require "luatokens"
local file = io.open(filename, "r")
local tokens = luatokens.defs
local state, err, etoken, estate = parser.parse(tokens, function() return file:read(8192) end)
if state then
for i,v in ipairs(state) do
v = luatokens.reverse_keywords[v] or luatokens.reverse_tokens[v] or v
print(i, v) -- TODO formatting
end
else
print("Parse error")
end
end

View File

@ -405,7 +405,7 @@ do -- long comments
end end
end -- long comments end -- long comments
while false do -- FUCK do -- FUCK
local luatokens = require "luatokens" local luatokens = require "luatokens"
local luatokens_file = io.open("./luatokens.lua", "r") local luatokens_file = io.open("./luatokens.lua", "r")
local tokens = luatokens.defs local tokens = luatokens.defs