Compare commits

...

2 Commits

Author SHA1 Message Date
SoniEx2 886df79e9f Tests pass in Lua 5.1/5.2 now too 2019-04-09 16:37:54 -03:00
SoniEx2 b352a06fa3 Fix number parsing (not compatible with 5.1) 2019-04-09 16:35:05 -03:00
2 changed files with 66 additions and 5 deletions

View File

@ -616,6 +616,7 @@ defs.in_integer = setmetatable(selfify({
alpha = false,
['e'] = "exp",
['E'] = "exp",
[parser.EOZ] = "self", -- defs
exp = function(state, token)
collect_fallback(state, token)
return "in_exp"
@ -628,10 +629,20 @@ defs.in_integer = setmetatable(selfify({
collect_fallback(state, token)
return "in_digit"
end,
[-1] = function(state, token, rule)
-- TODO figure out best order for these checks
if rule == "digit" or token == "." or rule == "hexdigit" or rule == "into_hex" or rule == "exp" then return end
state[#state+1] = state[STATE].numtype
state[#state+1] = tonumber(table.concat(state[COLLECT])) -- TODO maybe not the best option
state[COLLECT] = nil
end,
numtype = TK_INT
}, "in_digit"), {__index=defs})
defs.in_zero = setmetatable({
['x'] = function(state, token)
['x'] = "into_hex",
['X'] = "into_hex",
into_hex = function(state, token)
collect_fallback(state, token)
return "in_hex"
end,
@ -639,6 +650,7 @@ defs.in_zero = setmetatable({
defs.in_decimal = setmetatable(selfify({
['.'] = false,
numtype = TK_FLT
}, "in_digit"), {__index=defs.in_integer})
defs.in_expnum = setmetatable(selfify({
@ -654,6 +666,7 @@ defs.in_subexp = setmetatable({
}, {__index=defs.base})
defs.in_exp = setmetatable({
in_subexp = defs.in_subexp,
["+"] = "sign",
["-"] = "sign",
sign = function(state, token)
@ -662,18 +675,19 @@ defs.in_exp = setmetatable({
end,
}, {__index=defs.in_subexp})
defs.in_hex = setmetatable({
defs.in_hex = setmetatable(selfify({
in_decimal = "in_hex_fraction",
hexdigit = 'digit',
['e'] = 'hexdigit',
['E'] = 'hexdigit',
['p'] = 'exp',
['P'] = 'exp',
}, {__index=defs.in_integer})
}, "in_digit"), {__index=defs.in_integer})
defs.in_hex_fraction = setmetatable({
defs.in_hex_fraction = setmetatable(selfify({
['.'] = false,
}, {__index=defs.in_hex})
numtype = TK_FLT
}, "in_digit"), {__index=defs.in_hex})
function defs.simpletoken(state, token)
state[#state+1] = token

View File

@ -405,6 +405,53 @@ do -- long comments
end
end -- long comments
do -- numbers
local luatokens = require "luatokens"
local tokens = luatokens.defs
local state, err, etoken, estate = parser.parse(tokens, [[
3 345 0xff 0xBEBADA
3.0 3.1416 314.16e-2 0.31416E1 34e1
0x0.1E 0xA23p-4 0X1.921FB54442D18P+1]])
local case = case()
if not state then
print(case, "---- IN TOKENS ----")
print(case, err, etoken)
for i,v in pairs(estate) do
print(case, i, v)
end
print(case, "---- OUT TOKENS ----")
else
-- integers
assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
assert(table.remove(state, 1) == 3)
assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
assert(table.remove(state, 1) == 345)
assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
assert(table.remove(state, 1) == 0xff)
assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
assert(table.remove(state, 1) == 0xBEBADA)
-- floats
assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
assert(table.remove(state, 1) == 3.0)
assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
assert(table.remove(state, 1) == 3.1416)
assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
assert(table.remove(state, 1) == 314.16e-2)
assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
assert(table.remove(state, 1) == 0.31416E1)
assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
assert(table.remove(state, 1) == 34e1)
assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
assert(table.remove(state, 1) == 0.1171875)
assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
assert(table.remove(state, 1) == 162.1875)
assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
assert(table.remove(state, 1) == math.pi)
assert(table.remove(state, 1) == nil)
assert(state.line == 3)
end
end -- numbers
do -- FUCK
local luatokens = require "luatokens"
local luatokens_file = io.open("./luatokens.lua", "r")