From 659a23db78b993256b075fad33403eb2196b3d11 Mon Sep 17 00:00:00 2001 From: Dave Lucia Date: Tue, 10 Feb 2026 20:05:12 -0800 Subject: [PATCH 1/3] feat: Add \z escape sequence to skip whitespace in strings Implements the Lua 5.3 \z escape sequence which skips all following whitespace characters (spaces, tabs, newlines) in string literals. This allows splitting long string literals across multiple lines without including the whitespace in the final string value. Co-Authored-By: Claude Sonnet 4.5 --- lib/lua/lexer.ex | 34 ++++++++++++++++++++++++++++++++++ test/lua/lexer_test.exs | 16 +++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/lib/lua/lexer.ex b/lib/lua/lexer.ex index b899dea..e55f330 100644 --- a/lib/lua/lexer.ex +++ b/lib/lua/lexer.ex @@ -334,6 +334,12 @@ defmodule Lua.Lexer do do_tokenize(rest, [token | acc], pos) end + # \z escape: skip all following whitespace + defp scan_string(<>, str_acc, acc, pos, start_pos, quote) do + {remaining, new_pos} = skip_whitespace_in_string(rest, advance_column(pos, 2)) + scan_string(remaining, str_acc, acc, new_pos, start_pos, quote) + end + defp scan_string(<>, str_acc, acc, pos, start_pos, quote) do # Escape sequence case escape_char(esc) do @@ -371,6 +377,34 @@ defmodule Lua.Lexer do defp escape_char(?'), do: {:ok, ?'} defp escape_char(_), do: :error + # Helper for \z escape: skip all whitespace characters + defp skip_whitespace_in_string(<>, pos) do + skip_whitespace_in_string(rest, advance_column(pos, 1)) + end + + defp skip_whitespace_in_string(<>, pos) do + skip_whitespace_in_string(rest, advance_column(pos, 1)) + end + + defp skip_whitespace_in_string(<>, pos) do + new_pos = %{line: pos.line + 1, column: 1, byte_offset: pos.byte_offset + 1} + skip_whitespace_in_string(rest, new_pos) + end + + defp skip_whitespace_in_string(<>, pos) do + new_pos = %{line: pos.line + 1, column: 1, byte_offset: pos.byte_offset + 2} + skip_whitespace_in_string(rest, new_pos) + end + + defp skip_whitespace_in_string(<>, pos) do + new_pos = %{line: pos.line + 1, column: 1, byte_offset: pos.byte_offset + 1} + skip_whitespace_in_string(rest, new_pos) + end + + defp skip_whitespace_in_string(rest, pos) do + {rest, pos} + end + # Scan long bracket for level: [[ or [=[ or [==[ etc. defp scan_long_bracket(rest, equals) do case rest do diff --git a/test/lua/lexer_test.exs b/test/lua/lexer_test.exs index 205d3be..f850d37 100644 --- a/test/lua/lexer_test.exs +++ b/test/lua/lexer_test.exs @@ -213,6 +213,21 @@ defmodule Lua.LexerTest do assert {:error, {:unclosed_long_string, _}} = Lexer.tokenize("[[hello") assert {:error, {:unclosed_long_string, _}} = Lexer.tokenize("[=[test") end + + test "handles \\z escape sequence (skip whitespace)" do + # \z skips all following whitespace including newlines + assert {:ok, [{:string, "abcdef", _}, {:eof, _}]} = Lexer.tokenize("\"abc\\z \n def\"") + + # Multiple spaces and tabs - \z skips them all + assert {:ok, [{:string, "helloworld", _}, {:eof, _}]} = + Lexer.tokenize("\"hello\\z \t world\"") + + # Multiple newlines + assert {:ok, [{:string, "abc", _}, {:eof, _}]} = Lexer.tokenize("\"abc\\z \n\n\n\"") + + # With CRLF + assert {:ok, [{:string, "test", _}, {:eof, _}]} = Lexer.tokenize("\"test\\z\r\n\"") + end end describe "operators" do @@ -553,7 +568,6 @@ defmodule Lua.LexerTest do test "handles invalid escape sequences in strings" do # Invalid escape sequences should be included as-is assert {:ok, [{:string, "\\x", _}, {:eof, _}]} = Lexer.tokenize(~s("\\x")) - assert {:ok, [{:string, "\\z", _}, {:eof, _}]} = Lexer.tokenize(~s("\\z")) assert {:ok, [{:string, "\\1", _}, {:eof, _}]} = Lexer.tokenize(~s("\\1")) end From 35ef10c027092611651d1b6020ce13f8b75ccc5d Mon Sep 17 00:00:00 2001 From: Dave Lucia Date: Tue, 10 Feb 2026 20:07:28 -0800 Subject: [PATCH 2/3] test: Add unit test for table constructor semicolon syntax Adds test verifying that semicolons can be used as field separators in table constructors, as required by Lua 5.3 test suite. Co-Authored-By: Claude Sonnet 4.5 --- test/lua_test.exs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/lua_test.exs b/test/lua_test.exs index 2a29e9c..6b13e0f 100644 --- a/test/lua_test.exs +++ b/test/lua_test.exs @@ -23,6 +23,22 @@ defmodule LuaTest do assert {["nested"], _lua} = lua |> Lua.set!([:a, :b, :c], "nested") |> Lua.eval!("return a.b.c") end + + test "table constructors with semicolons", %{lua: lua} do + # Can retrieve values from tables with explicit fields using semicolons + code = """ + t = {1, 2; n=2} + return t[1], t[2], t.n + """ + assert {[1, 2, 2], _lua} = Lua.eval!(lua, code) + + # Mixed commas and semicolons + code = """ + t = {1; 2, 3} + return t[1], t[2], t[3] + """ + assert {[1, 2, 3], _lua} = Lua.eval!(lua, code) + end end describe "inspect" do From 8329dc55634ff137a0344796e7771e4fd22d0468 Mon Sep 17 00:00:00 2001 From: Dave Lucia Date: Tue, 10 Feb 2026 20:23:34 -0800 Subject: [PATCH 3/3] format: Add blank lines in test --- test/lua_test.exs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/lua_test.exs b/test/lua_test.exs index 6b13e0f..09e995a 100644 --- a/test/lua_test.exs +++ b/test/lua_test.exs @@ -30,6 +30,7 @@ defmodule LuaTest do t = {1, 2; n=2} return t[1], t[2], t.n """ + assert {[1, 2, 2], _lua} = Lua.eval!(lua, code) # Mixed commas and semicolons @@ -37,6 +38,7 @@ defmodule LuaTest do t = {1; 2, 3} return t[1], t[2], t[3] """ + assert {[1, 2, 3], _lua} = Lua.eval!(lua, code) end end