luatokens.lua

@@ -386,8 +386,7 @@ end
 do local tlongstring = {}
     defs.longstring = tlongstring
     do local tllongstring_proper = selfify({[""] = "self", ["]"] = function(state, token) state.longstring_close = 0 return "maybe_end" end})
-        tllongstring_proper[1] = false -- placeholder for newline handling
-        tllongstring_proper[2] = collect_fallback
+        tllongstring_proper[1] = collect_fallback
 
         do local tllmaybe_end = selfify({defs = defs}, "maybe_end")
             tllongstring_proper.maybe_end = tllmaybe_end
@@ -485,34 +484,38 @@ do local tmaybe_comment = setmetatable({["-"] = "comment"}, {__index=defs})
         mknewline(tmcomment, 1, defs)
         mknewline(tmcomment.comment_proper, 1, defs)
         tmcomment.comment_proper[""] = "self"
-        do local tllongcomment_proper = selfify({[""] = "self", ["]"] = function(state, token) state.longcomment_close = 0 return "maybe_end" end})
-            tmcomment.longcomment = tllongcomment_proper
-            do local tllmaybe_end = selfify({defs = defs}, "maybe_end")
-                tllongcomment_proper.maybe_end = tllmaybe_end
-                tllmaybe_end.longcomment_proper = tllongcomment_proper
-                tllmaybe_end["="] = function(state, token)
-                    state.longcomment_close = state.longcomment_close + 1
-                    return "maybe_end"
-                end
-                tllmaybe_end["]"] = function(state, token)
-                    if state.longcomment_close == state.longcomment_count then
-                        state.longcomment_close = nil
-                        state.longcomment_count = nil
-                        return "defs"
-                    else
-                        state.longcomment_close = 0
+        do local tlongcomment = {}
+            tmcomment.longcomment = tlongcomment
+            do local tllongcomment_proper = selfify({[""] = "self", ["]"] = function(state, token) state.longcomment_close = 0 return "maybe_end" end})
+                do local tllmaybe_end = selfify({comment = tcomment}, "maybe_end")
+                    tllongcomment_proper.maybe_end = tllmaybe_end
+                    tllmaybe_end = tllongcomment_proper
+                    tllmaybe_end["="] = function(state, token)
+                        state.longcomment_close = state.longcomment_close + 1
                         return "maybe_end"
                     end
-                end
-                tllmaybe_end[""] = "longcomment_proper"
-                tllmaybe_end[-1] = function(state, token, rule)
-                    if not rule then
-                        state.longcomment_close = nil
+                    tllmaybe_end["]"] = function(state, token)
+                        if state.longcomment_close == state.longcomment_count then
+                            state.longcomment_close = nil
+                            state.longcomment_count = nil
+                            return "defs"
+                        else
+                            state.longcomment_close = 0
+                            return "maybe_end"
+                        end
+                    end
+                    tllmaybe_end[""] = "longcomment_proper"
+                    tllmaybe_end[-1] = function(state, token, rule)
+                        if not rule then
+                            state.longcomment_close = nil
+                        end
                     end
                 end
-            end
 
-            mknewline(tllongcomment_proper, 1, tllongcomment_proper)
+                tlongcomment.longcomment_proper = tllongcomment_proper
+                mknewline(tlongcomment, 1, tllongcomment_proper)
+                setmetatable(tlongcomment, {__index=tllongcomment_proper})
+            end
         end
 
         tmcomment.maybe_longcomment = setmetatable({
@@ -533,7 +536,7 @@ do local tmaybe_comment = setmetatable({["-"] = "comment"}, {__index=defs})
                     return "longcomment_count"
                 elseif token == "[" then
                     state.longcomment_count = 0
-                    return "longcomment"
+                    return "longstring"
                 end
             end,
         }, {__index=tmcomment})

test.lua

@@ -260,7 +260,7 @@ end -- long string
 do -- long string edge cases
     local luatokens = require "luatokens"
     local tokens = luatokens.defs
-    local state, err, etoken, estate = parser.parse(tokens, "[==[]=]==][==[]]==][=[] ]=][[\n]][[\n ]]")
+    local state, err, etoken, estate = parser.parse(tokens, "[==[]=]==][==[]]==][=[] ]=]")
     local case = case()
     if not state then
         print(case, "---- IN  TOKENS ----")
@@ -276,12 +276,8 @@ do -- long string edge cases
         assert(table.remove(state, 1) == "]")
         assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
         assert(table.remove(state, 1) == "] ")
-        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
-        assert(table.remove(state, 1) == "")
-        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
-        assert(table.remove(state, 1) == " ")
         assert(table.remove(state, 1) == nil)
-        assert(state.line == 3 or not state.line)
+        assert(state.line == 1 or not state.line)
     end
 end -- long string
 
@@ -389,7 +385,6 @@ do -- long comments
     local luatokens = require "luatokens"
     local tokens = luatokens.defs
     local state, err, etoken, estate = parser.parse(tokens, [==[--[[
-      
     --]]]==])
     local case = case()
     if not state then
@@ -401,7 +396,7 @@ do -- long comments
         print(case, "---- OUT TOKENS ----")
     else
         assert(table.remove(state, 1) == nil)
-        assert(state.line == 3)
+        assert(state.line == 2)
     end
 end -- long comments