Fix number parsing (not compatible with 5.1)

luatokens.lua

@@ -616,6 +616,7 @@ defs.in_integer = setmetatable(selfify({
     alpha = false,
     ['e'] = "exp",
     ['E'] = "exp",
+    [parser.EOZ] = "self", -- defs
     exp = function(state, token)
         collect_fallback(state, token)
         return "in_exp"
@@ -628,10 +629,20 @@ defs.in_integer = setmetatable(selfify({
         collect_fallback(state, token)
         return "in_digit"
     end,
+    [-1] = function(state, token, rule)
+        -- TODO figure out best order for these checks
+        if rule == "digit" or token == "." or rule == "hexdigit" or rule == "into_hex" or rule == "exp" then return end
+        state[#state+1] = state[STATE].numtype
+        state[#state+1] = tonumber(table.concat(state[COLLECT])) -- TODO maybe not the best option
+        state[COLLECT] = nil
+    end,
+    numtype = TK_INT
 }, "in_digit"), {__index=defs})
 
 defs.in_zero = setmetatable({
-    ['x'] = function(state, token)
+    ['x'] = "into_hex",
+    ['X'] = "into_hex",
+    into_hex = function(state, token)
         collect_fallback(state, token)
         return "in_hex"
     end,
@@ -639,6 +650,7 @@ defs.in_zero = setmetatable({
 
 defs.in_decimal = setmetatable(selfify({
     ['.'] = false,
+    numtype = TK_FLT
 }, "in_digit"), {__index=defs.in_integer})
 
 defs.in_expnum = setmetatable(selfify({
@@ -654,6 +666,7 @@ defs.in_subexp = setmetatable({
 }, {__index=defs.base})
 
 defs.in_exp = setmetatable({
+    in_subexp = defs.in_subexp,
     ["+"] = "sign",
     ["-"] = "sign",
     sign = function(state, token)
@@ -662,18 +675,19 @@ defs.in_exp = setmetatable({
     end,
 }, {__index=defs.in_subexp})
 
-defs.in_hex = setmetatable({
+defs.in_hex = setmetatable(selfify({
     in_decimal = "in_hex_fraction",
     hexdigit = 'digit',
     ['e'] = 'hexdigit',
     ['E'] = 'hexdigit',
     ['p'] = 'exp',
     ['P'] = 'exp',
-}, {__index=defs.in_integer})
+}, "in_digit"), {__index=defs.in_integer})
 
-defs.in_hex_fraction = setmetatable({
+defs.in_hex_fraction = setmetatable(selfify({
     ['.'] = false,
-}, {__index=defs.in_hex})
+    numtype = TK_FLT
+}, "in_digit"), {__index=defs.in_hex})
 
 function defs.simpletoken(state, token)
     state[#state+1] = token

test.lua

@@ -405,6 +405,53 @@ do -- long comments
     end
 end -- long comments
 
+do -- numbers
+    local luatokens = require "luatokens"
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, [[
+     3   345   0xff   0xBEBADA
+     3.0     3.1416     314.16e-2     0.31416E1     34e1
+     0x0.1E  0xA23p-4   0X1.921FB54442D18P+1]])
+    local case = case()
+    if not state then
+        print(case, "---- IN  TOKENS ----")
+        print(case, err, etoken)
+        for i,v in pairs(estate) do
+            print(case, i, v)
+        end
+        print(case, "---- OUT TOKENS ----")
+    else
+        -- integers
+        assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
+        assert(table.remove(state, 1) == 3)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
+        assert(table.remove(state, 1) == 345)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
+        assert(table.remove(state, 1) == 0xff)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
+        assert(table.remove(state, 1) == 0xBEBADA)
+        -- floats
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == 3.0)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == 3.1416)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == 314.16e-2)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == 0.31416E1)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == 34e1)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == 0x0.1E)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == 0xA23p-4)
+        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
+        assert(table.remove(state, 1) == 0x1.921FB54442D18P+1)
+        assert(table.remove(state, 1) == nil)
+        assert(state.line == 3)
+    end
+end -- numbers
+
 do -- FUCK
     local luatokens = require "luatokens"
     local luatokens_file = io.open("./luatokens.lua", "r")