It parses itself!

luatokens.lua

@@ -581,11 +581,14 @@ defs["."] = setmetatable({
         if token ~= "." then
             if rule ~= "digit" then
                 state[#state+1] = "."
-            else
-                error("NYI") -- TODO digit handling
             end
         end
     end,
+    digit = function(state, token, rule)
+        state[#state+1] = TK_FLT
+        state[COLLECT] = {".", coalesce=31}
+        return "in_decimal"
+    end,
     ["."] = setmetatable({
         [-1] = function(state, token, rule)
             if token ~= "." then
@@ -595,17 +598,82 @@ defs["."] = setmetatable({
         ["."] = function(state, token)
             state[#state+1] = TK_DOTS
             return "self" -- actually goes into defs
-        end
+        end,
     }, {__index=defs})
 }, {__index=defs})
 
 function defs.digit(state, token)
-    -- TODO
+    state[COLLECT] = {token, coalesce=31}
+    if token == "0" then
+        return "in_zero"
+    else
+        return "in_integer"
+    end
 end
 
-defs.in_digit = {
-    -- TODO
-}
+defs.in_integer = setmetatable(selfify({
+    hexdigit = "alpha",
+    alpha = false,
+    ['e'] = "exp",
+    ['E'] = "exp",
+    exp = function(state, token)
+        collect_fallback(state, token)
+        return "in_exp"
+    end,
+    ['.'] = function(state, token)
+        collect_fallback(state, token)
+        return "in_decimal"
+    end,
+    digit = function(state, token)
+        collect_fallback(state, token)
+        return "in_digit"
+    end,
+}, "in_digit"), {__index=defs})
+
+defs.in_zero = setmetatable({
+    ['x'] = function(state, token)
+        collect_fallback(state, token)
+        return "in_hex"
+    end,
+}, {__index=defs.in_integer})
+
+defs.in_decimal = setmetatable(selfify({
+    ['.'] = false,
+}, "in_digit"), {__index=defs.in_integer})
+
+defs.in_expnum = setmetatable(selfify({
+    exp = false,
+}, "in_digit"), {__index=defs.in_decimal})
+
+defs.in_subexp = setmetatable({
+    in_expnum = defs.in_expnum,
+    digit = function(state, token)
+        collect_fallback(state, token)
+        return "in_expnum"
+    end,
+}, {__index=defs.base})
+
+defs.in_exp = setmetatable({
+    ["+"] = "sign",
+    ["-"] = "sign",
+    sign = function(state, token)
+        collect_fallback(state, token)
+        return "in_subexp"
+    end,
+}, {__index=defs.in_subexp})
+
+defs.in_hex = setmetatable({
+    in_decimal = "in_hex_fraction",
+    hexdigit = 'digit',
+    ['e'] = 'hexdigit',
+    ['E'] = 'hexdigit',
+    ['p'] = 'exp',
+    ['P'] = 'exp',
+}, {__index=defs.in_integer})
+
+defs.in_hex_fraction = setmetatable({
+    ['.'] = false,
+}, {__index=defs.in_hex})
 
 function defs.simpletoken(state, token)
     state[#state+1] = token

printtokens.lua

@@ -0,0 +1,35 @@
+--[[
+    This file is part of luatokens.lua - pure-Lua Lua tokenizer
+    Copyright (C) 2019  Soni L.
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+--]]
+
+local parser = require "parser"
+
+do
+    local filename = arg[1]
+    local luatokens = require "luatokens"
+    local file = io.open(filename, "r")
+    local tokens = luatokens.defs
+    local state, err, etoken, estate = parser.parse(tokens, function() return file:read(8192) end)
+    if state then
+        for i,v in ipairs(state) do
+            v = luatokens.reverse_keywords[v] or luatokens.reverse_tokens[v] or v
+            print(i, v) -- TODO formatting
+        end
+    else
+        print("Parse error")
+    end
+end

test.lua

@@ -405,7 +405,7 @@ do -- long comments
     end
 end -- long comments
 
-while false do -- FUCK
+do -- FUCK
     local luatokens = require "luatokens"
     local luatokens_file = io.open("./luatokens.lua", "r")
     local tokens = luatokens.defs