Fix issues with newline handling in long strings

luatokens.lua

@@ -386,7 +386,8 @@ end
 do local tlongstring = {}
     defs.longstring = tlongstring
     do local tllongstring_proper = selfify({[""] = "self", ["]"] = function(state, token) state.longstring_close = 0 return "maybe_end" end})
-        tllongstring_proper[1] = collect_fallback
+        tllongstring_proper[1] = false -- placeholder for newline handling
+        tllongstring_proper[2] = collect_fallback
 
         do local tllmaybe_end = selfify({defs = defs}, "maybe_end")
             tllongstring_proper.maybe_end = tllmaybe_end
@@ -484,38 +485,34 @@ do local tmaybe_comment = setmetatable({["-"] = "comment"}, {__index=defs})
         mknewline(tmcomment, 1, defs)
         mknewline(tmcomment.comment_proper, 1, defs)
         tmcomment.comment_proper[""] = "self"
-        do local tlongcomment = {}
-            tmcomment.longcomment = tlongcomment
-            do local tllongcomment_proper = selfify({[""] = "self", ["]"] = function(state, token) state.longcomment_close = 0 return "maybe_end" end})
-                do local tllmaybe_end = selfify({comment = tcomment}, "maybe_end")
-                    tllongcomment_proper.maybe_end = tllmaybe_end
-                    tllmaybe_end = tllongcomment_proper
-                    tllmaybe_end["="] = function(state, token)
-                        state.longcomment_close = state.longcomment_close + 1
+        do local tllongcomment_proper = selfify({[""] = "self", ["]"] = function(state, token) state.longcomment_close = 0 return "maybe_end" end})
+            tmcomment.longcomment = tllongcomment_proper
+            do local tllmaybe_end = selfify({defs = defs}, "maybe_end")
+                tllongcomment_proper.maybe_end = tllmaybe_end
+                tllmaybe_end.longcomment_proper = tllongcomment_proper
+                tllmaybe_end["="] = function(state, token)
+                    state.longcomment_close = state.longcomment_close + 1
+                    return "maybe_end"
+                end
+                tllmaybe_end["]"] = function(state, token)
+                    if state.longcomment_close == state.longcomment_count then
+                        state.longcomment_close = nil
+                        state.longcomment_count = nil
+                        return "defs"
+                    else
+                        state.longcomment_close = 0
                         return "maybe_end"
                     end
-                    tllmaybe_end["]"] = function(state, token)
-                        if state.longcomment_close == state.longcomment_count then
-                            state.longcomment_close = nil
-                            state.longcomment_count = nil
-                            return "defs"
-                        else
-                            state.longcomment_close = 0
-                            return "maybe_end"
-                        end
-                    end
-                    tllmaybe_end[""] = "longcomment_proper"
-                    tllmaybe_end[-1] = function(state, token, rule)
-                        if not rule then
-                            state.longcomment_close = nil
-                        end
+                end
+                tllmaybe_end[""] = "longcomment_proper"
+                tllmaybe_end[-1] = function(state, token, rule)
+                    if not rule then
+                        state.longcomment_close = nil
                     end
                 end
-
-                tlongcomment.longcomment_proper = tllongcomment_proper
-                mknewline(tlongcomment, 1, tllongcomment_proper)
-                setmetatable(tlongcomment, {__index=tllongcomment_proper})
             end
+
+            mknewline(tllongcomment_proper, 1, tllongcomment_proper)
         end
 
         tmcomment.maybe_longcomment = setmetatable({
@@ -536,7 +533,7 @@ do local tmaybe_comment = setmetatable({["-"] = "comment"}, {__index=defs})
                     return "longcomment_count"
                 elseif token == "[" then
                     state.longcomment_count = 0
-                    return "longstring"
+                    return "longcomment"
                 end
             end,
         }, {__index=tmcomment})

test.lua

@@ -260,7 +260,7 @@ end -- long string
 do -- long string edge cases
     local luatokens = require "luatokens"
     local tokens = luatokens.defs
-    local state, err, etoken, estate = parser.parse(tokens, "[==[]=]==][==[]]==][=[] ]=]")
+    local state, err, etoken, estate = parser.parse(tokens, "[==[]=]==][==[]]==][=[] ]=][[\n]][[\n ]]")
     local case = case()
     if not state then
         print(case, "---- IN  TOKENS ----")
@@ -276,8 +276,12 @@ do -- long string edge cases
         assert(table.remove(state, 1) == "]")
         assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
         assert(table.remove(state, 1) == "] ")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == "")
+        assert(table.remove(state, 1) == luatokens.tokens.TK_STRING)
+        assert(table.remove(state, 1) == " ")
         assert(table.remove(state, 1) == nil)
-        assert(state.line == 1 or not state.line)
+        assert(state.line == 3 or not state.line)
     end
 end -- long string
 
@@ -385,6 +389,7 @@ do -- long comments
     local luatokens = require "luatokens"
     local tokens = luatokens.defs
     local state, err, etoken, estate = parser.parse(tokens, [==[--[[
+      
     --]]]==])
     local case = case()
     if not state then
@@ -396,7 +401,7 @@ do -- long comments
         print(case, "---- OUT TOKENS ----")
     else
         assert(table.remove(state, 1) == nil)
-        assert(state.line == 2)
+        assert(state.line == 3)
     end
 end -- long comments