luatokens.lua

@@ -616,7 +616,6 @@ defs.in_integer = setmetatable(selfify({
     alpha = false,
     ['e'] = "exp",
     ['E'] = "exp",
-    [parser.EOZ] = "self", -- defs
     exp = function(state, token)
         collect_fallback(state, token)
         return "in_exp"
@@ -629,20 +628,10 @@ defs.in_integer = setmetatable(selfify({
         collect_fallback(state, token)
         return "in_digit"
     end,
-    [-1] = function(state, token, rule)
-        -- TODO figure out best order for these checks
-        if rule == "digit" or token == "." or rule == "hexdigit" or rule == "into_hex" or rule == "exp" then return end
-        state[#state+1] = state[STATE].numtype
-        state[#state+1] = tonumber(table.concat(state[COLLECT])) -- TODO maybe not the best option
-        state[COLLECT] = nil
-    end,
-    numtype = TK_INT
 }, "in_digit"), {__index=defs})
 
 defs.in_zero = setmetatable({
-    ['x'] = "into_hex",
-    ['X'] = "into_hex",
-    into_hex = function(state, token)
+    ['x'] = function(state, token)
         collect_fallback(state, token)
         return "in_hex"
     end,
@@ -650,7 +639,6 @@ defs.in_zero = setmetatable({
 
 defs.in_decimal = setmetatable(selfify({
     ['.'] = false,
-    numtype = TK_FLT
 }, "in_digit"), {__index=defs.in_integer})
 
 defs.in_expnum = setmetatable(selfify({
@@ -666,7 +654,6 @@ defs.in_subexp = setmetatable({
 }, {__index=defs.base})
 
 defs.in_exp = setmetatable({
-    in_subexp = defs.in_subexp,
     ["+"] = "sign",
     ["-"] = "sign",
     sign = function(state, token)
@@ -675,19 +662,18 @@ defs.in_exp = setmetatable({
     end,
 }, {__index=defs.in_subexp})
 
-defs.in_hex = setmetatable(selfify({
+defs.in_hex = setmetatable({
     in_decimal = "in_hex_fraction",
     hexdigit = 'digit',
     ['e'] = 'hexdigit',
     ['E'] = 'hexdigit',
     ['p'] = 'exp',
     ['P'] = 'exp',
-}, "in_digit"), {__index=defs.in_integer})
+}, {__index=defs.in_integer})
 
-defs.in_hex_fraction = setmetatable(selfify({
+defs.in_hex_fraction = setmetatable({
     ['.'] = false,
-    numtype = TK_FLT
-}, "in_digit"), {__index=defs.in_hex})
+}, {__index=defs.in_hex})
 
 function defs.simpletoken(state, token)
     state[#state+1] = token

test.lua

@@ -405,53 +405,6 @@ do -- long comments
     end
 end -- long comments
 
-do -- numbers
-    local luatokens = require "luatokens"
-    local tokens = luatokens.defs
-    local state, err, etoken, estate = parser.parse(tokens, [[
-     3   345   0xff   0xBEBADA
-     3.0     3.1416     314.16e-2     0.31416E1     34e1
-     0x0.1E  0xA23p-4   0X1.921FB54442D18P+1]])
-    local case = case()
-    if not state then
-        print(case, "---- IN  TOKENS ----")
-        print(case, err, etoken)
-        for i,v in pairs(estate) do
-            print(case, i, v)
-        end
-        print(case, "---- OUT TOKENS ----")
-    else
-        -- integers
-        assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
-        assert(table.remove(state, 1) == 3)
-        assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
-        assert(table.remove(state, 1) == 345)
-        assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
-        assert(table.remove(state, 1) == 0xff)
-        assert(table.remove(state, 1) == luatokens.tokens.TK_INT)
-        assert(table.remove(state, 1) == 0xBEBADA)
-        -- floats
-        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
-        assert(table.remove(state, 1) == 3.0)
-        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
-        assert(table.remove(state, 1) == 3.1416)
-        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
-        assert(table.remove(state, 1) == 314.16e-2)
-        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
-        assert(table.remove(state, 1) == 0.31416E1)
-        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
-        assert(table.remove(state, 1) == 34e1)
-        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
-        assert(table.remove(state, 1) == 0.1171875)
-        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
-        assert(table.remove(state, 1) == 162.1875)
-        assert(table.remove(state, 1) == luatokens.tokens.TK_FLT)
-        assert(table.remove(state, 1) == math.pi)
-        assert(table.remove(state, 1) == nil)
-        assert(state.line == 3)
-    end
-end -- numbers
-
 do -- FUCK
     local luatokens = require "luatokens"
     local luatokens_file = io.open("./luatokens.lua", "r")