From fdc954330ae48c9aa36c07c7ff1354f2354f50a6 Mon Sep 17 00:00:00 2001 From: He-Pin Date: Sat, 28 Feb 2026 08:27:59 +0800 Subject: [PATCH] chore: sync more tests and support ignoring --- build.mill | 6 +- build.sbt | 2 +- sjsonnet/src-js/sjsonnet/Platform.scala | 76 ++++++++++++++----- sjsonnet/src-native/sjsonnet/Platform.scala | 76 ++++++++++++++----- sjsonnet/src/sjsonnet/Importer.scala | 2 + .../test/resources/go_test_suite/.sync_ignore | 3 + .../test/resources/test_suite/.sync_ignore | 7 ++ .../error.parse.deep_array_nesting.jsonnet | 1 + ...or.parse.deep_array_nesting.jsonnet.golden | 1 + .../test/resources/test_suite/stdlib.jsonnet | 2 + .../src-jvm-native/sjsonnet/FileTests.scala | 31 +++----- .../test/src/sjsonnet/ParseYamlTests.scala | 9 +++ sync_test_suites.sh | 49 +++++++++++- 13 files changed, 196 insertions(+), 69 deletions(-) create mode 100644 sjsonnet/test/resources/go_test_suite/.sync_ignore create mode 100644 sjsonnet/test/resources/test_suite/.sync_ignore create mode 100644 sjsonnet/test/resources/test_suite/error.parse.deep_array_nesting.jsonnet create mode 100644 sjsonnet/test/resources/test_suite/error.parse.deep_array_nesting.jsonnet.golden diff --git a/build.mill b/build.mill index b1231892..681cf003 100644 --- a/build.mill +++ b/build.mill @@ -158,7 +158,7 @@ object sjsonnet extends VersionFileModule { def mvnDeps = super.mvnDeps() ++ Seq( - mvn"org.virtuslab::scala-yaml::0.3.0" + mvn"org.virtuslab::scala-yaml::0.3.1" ) def nodeJsArgs: List[String] = List("--stack-size=" + stackSizekBytes) @@ -265,7 +265,7 @@ object sjsonnet extends VersionFileModule { def mvnDeps = super.mvnDeps() ++ Seq( mvn"com.github.lolgab::scala-native-crypto::0.3.0", - mvn"org.virtuslab::scala-yaml::0.3.0" + mvn"org.virtuslab::scala-yaml::0.3.1" ) def releaseMode = ReleaseMode.ReleaseFull @@ -301,7 +301,7 @@ object sjsonnet extends VersionFileModule { def mvnDeps = super.mvnDeps() ++ Seq( mvn"org.tukaani:xz::1.11", mvn"at.yawk.lz4:lz4-java::1.10.3", - mvn"org.yaml:snakeyaml::2.4", + mvn"org.yaml:snakeyaml::2.6", mvn"com.google.re2j:re2j:1.8" ) diff --git a/build.sbt b/build.sbt index 01d27420..92513d55 100644 --- a/build.sbt +++ b/build.sbt @@ -27,7 +27,7 @@ lazy val main = (project in file("sjsonnet")) "at.yawk.lz4" % "lz4-java" % "1.10.3", "org.scala-lang.modules" %% "scala-collection-compat" % "2.14.0", "org.tukaani" % "xz" % "1.11", - "org.yaml" % "snakeyaml" % "2.5", + "org.yaml" % "snakeyaml" % "2.6", "com.google.re2j" % "re2j" % "1.8" ), libraryDependencies ++= Seq( diff --git a/sjsonnet/src-js/sjsonnet/Platform.scala b/sjsonnet/src-js/sjsonnet/Platform.scala index fb486536..25c6bf37 100644 --- a/sjsonnet/src-js/sjsonnet/Platform.scala +++ b/sjsonnet/src-js/sjsonnet/Platform.scala @@ -42,35 +42,69 @@ object Platform { Error.fail("Unsupported YAML node type: " + node.getClass.getSimpleName) } - private val docSplitPattern = Pattern.compile("(^|\n)---\\s*(\n|$)") - def yamlToJson(s: String): ujson.Value = { - val docs = docSplitPattern.split(s, -1) - docs.size match { + // Split YAML multi-document stream manually, similar to SnakeYAML's loadAll + // since parseManyYamls doesn't handle all cases correctly + val documents = splitYamlDocuments(s) + + documents.size match { case 0 => ujson.Null - case 1 => - docs.head.asNode match { - case Right(n) => - nodeToJson(n) - case Left(e) if docs.head.trim.isEmpty => - ujson.Null - case Left(e) => - Error.fail("Error converting YAML to JSON: " + e.getMessage) - } + case 1 => parseSingleDocument(documents.head) case _ => - val buf = new mutable.ArrayBuffer[ujson.Value](docs.size) - for (doc <- docs) { - doc.asNode match { - case Right(n) => buf += nodeToJson(n) - case Left(e) if docs.head.trim.isEmpty => - case Left(e) => - Error.fail("Error converting YAML to JSON: " + e.getMessage) - } + val buf = new mutable.ArrayBuffer[ujson.Value](documents.size) + for (doc <- documents) { + buf += parseSingleDocument(doc) } ujson.Arr(buf) } } + private def splitYamlDocuments(s: String): List[String] = { + if (s.trim.isEmpty) return Nil + + // Split on document separator "---" at line start + // But only if it's followed by whitespace or end of line + val lines = s.split("\n", -1).toList + val documents = mutable.ArrayBuffer[String]() + val currentDoc = mutable.ArrayBuffer[String]() + var isFirstDoc = true + + for (line <- lines) { + val trimmed = line.trim + // Check if this line starts with "---" and is followed by whitespace or end + if (trimmed.startsWith("---") && (trimmed.length == 3 || trimmed.charAt(3).isWhitespace)) { + // Save previous document if not empty + if (currentDoc.nonEmpty || !isFirstDoc) { + documents += currentDoc.mkString("\n") + } + currentDoc.clear() + isFirstDoc = false + } else { + currentDoc += line + } + } + + // Add last document + if (currentDoc.nonEmpty || documents.nonEmpty) { + documents += currentDoc.mkString("\n") + } + + documents.toList + } + + private def parseSingleDocument(doc: String): ujson.Value = { + val trimmed = doc.trim + if (trimmed.isEmpty) { + ujson.Null + } else { + // Use parseYaml for single document + parseYaml(trimmed) match { + case Right(node) => nodeToJson(node) + case Left(e) => Error.fail("Error converting YAML to JSON: " + e.getMessage) + } + } + } + def md5(s: String): String = { throw new Exception("MD5 not implemented in Scala.js") } diff --git a/sjsonnet/src-native/sjsonnet/Platform.scala b/sjsonnet/src-native/sjsonnet/Platform.scala index a484597a..eb3030a7 100644 --- a/sjsonnet/src-native/sjsonnet/Platform.scala +++ b/sjsonnet/src-native/sjsonnet/Platform.scala @@ -77,35 +77,69 @@ object Platform { Error.fail("Unsupported YAML node type: " + node.getClass.getSimpleName) } - private val docSplitPattern = Pattern.compile("(?m)^---\\s*$") - def yamlToJson(s: String): ujson.Value = { - val docs = docSplitPattern.split(s, -1) - docs.length match { + // Split YAML multi-document stream manually, similar to SnakeYAML's loadAll + // since parseManyYamls doesn't handle all cases correctly + val documents = splitYamlDocuments(s) + + documents.size match { case 0 => ujson.Null - case 1 => - docs.head.asNode match { - case Right(n) => - nodeToJson(n) - case Left(e) if docs.head.trim.isEmpty => - ujson.Null - case Left(e) => - Error.fail("Error converting YAML to JSON: " + e.getMessage) - } + case 1 => parseSingleDocument(documents.head) case _ => - val buf = new mutable.ArrayBuffer[ujson.Value](docs.length) - for (doc <- docs) { - doc.asNode match { - case Right(n) => buf += nodeToJson(n) - case Left(e) if doc.isEmpty => - case Left(e) => - Error.fail("Error converting YAML to JSON: " + e.getMessage) - } + val buf = new mutable.ArrayBuffer[ujson.Value](documents.size) + for (doc <- documents) { + buf += parseSingleDocument(doc) } ujson.Arr(buf) } } + private def splitYamlDocuments(s: String): List[String] = { + if (s.trim.isEmpty) return Nil + + // Split on document separator "---" at line start + // But only if it's followed by whitespace or end of line + val lines = s.split("\n", -1).toList + val documents = mutable.ArrayBuffer[String]() + val currentDoc = mutable.ArrayBuffer[String]() + var isFirstDoc = true + + for (line <- lines) { + val trimmed = line.trim + // Check if this line starts with "---" and is followed by whitespace or end + if (trimmed.startsWith("---") && (trimmed.length == 3 || trimmed.charAt(3).isWhitespace)) { + // Save previous document if not empty + if (currentDoc.nonEmpty || !isFirstDoc) { + documents += currentDoc.mkString("\n") + } + currentDoc.clear() + isFirstDoc = false + } else { + currentDoc += line + } + } + + // Add last document + if (currentDoc.nonEmpty || documents.nonEmpty) { + documents += currentDoc.mkString("\n") + } + + documents.toList + } + + private def parseSingleDocument(doc: String): ujson.Value = { + val trimmed = doc.trim + if (trimmed.isEmpty) { + ujson.Null + } else { + // Use parseYaml for single document + parseYaml(trimmed) match { + case Right(node) => nodeToJson(node) + case Left(e) => Error.fail("Error converting YAML to JSON: " + e.getMessage) + } + } + } + private def computeHash(algorithm: String, s: String) = { java.security.MessageDigest .getInstance(algorithm) diff --git a/sjsonnet/src/sjsonnet/Importer.scala b/sjsonnet/src/sjsonnet/Importer.scala index 0cb6d59a..debd3b64 100644 --- a/sjsonnet/src/sjsonnet/Importer.scala +++ b/sjsonnet/src/sjsonnet/Importer.scala @@ -225,6 +225,8 @@ class CachedResolver( case e: ParseError if e.offset >= 0 => val pos = new Position(new FileScope(path), e.offset) Left(new ParseError(e.getMessage).addFrame(pos)) + case e: ParseError => + Left(e) } parsed.flatMap { case (e, fs) => process(e, fs) } } diff --git a/sjsonnet/test/resources/go_test_suite/.sync_ignore b/sjsonnet/test/resources/go_test_suite/.sync_ignore new file mode 100644 index 00000000..e6917210 --- /dev/null +++ b/sjsonnet/test/resources/go_test_suite/.sync_ignore @@ -0,0 +1,3 @@ +# Files to ignore when syncing from google/go-jsonnet (Go test suite). +# One .jsonnet filename per line. Lines starting with '#' are comments. +# Corresponding .golden files are also automatically ignored. diff --git a/sjsonnet/test/resources/test_suite/.sync_ignore b/sjsonnet/test/resources/test_suite/.sync_ignore new file mode 100644 index 00000000..5276faca --- /dev/null +++ b/sjsonnet/test/resources/test_suite/.sync_ignore @@ -0,0 +1,7 @@ +# Files to ignore when syncing from google/jsonnet (C++ test suite). +# One .jsonnet filename per line. Lines starting with '#' are comments. +# Corresponding .golden files are also automatically ignored. +# +# error.std_parseYaml1.jsonnet: sjsonnet uses SnakeYAML/scala-yaml which +# successfully parses 'a: b:' as {"a":"b:"}, unlike RapidYAML which errors. +error.std_parseYaml1.jsonnet diff --git a/sjsonnet/test/resources/test_suite/error.parse.deep_array_nesting.jsonnet b/sjsonnet/test/resources/test_suite/error.parse.deep_array_nesting.jsonnet new file mode 100644 index 00000000..93d2c577 --- /dev/null +++ b/sjsonnet/test/resources/test_suite/error.parse.deep_array_nesting.jsonnet @@ -0,0 +1 @@ +[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[[1]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]]] diff --git a/sjsonnet/test/resources/test_suite/error.parse.deep_array_nesting.jsonnet.golden b/sjsonnet/test/resources/test_suite/error.parse.deep_array_nesting.jsonnet.golden new file mode 100644 index 00000000..4d5e4b50 --- /dev/null +++ b/sjsonnet/test/resources/test_suite/error.parse.deep_array_nesting.jsonnet.golden @@ -0,0 +1 @@ +sjsonnet.ParseError: Parsing exceeded maximum recursion depth of 1000 \ No newline at end of file diff --git a/sjsonnet/test/resources/test_suite/stdlib.jsonnet b/sjsonnet/test/resources/test_suite/stdlib.jsonnet index 5d2ed924..0a854c3a 100644 --- a/sjsonnet/test/resources/test_suite/stdlib.jsonnet +++ b/sjsonnet/test/resources/test_suite/stdlib.jsonnet @@ -1481,6 +1481,8 @@ std.assertEqual(std.parseJson('{"a": {"b": ["c", 42]}}'), { a: { b: ['c', 42] } // Empty input evaluates to `null`. // Regression test for https://github.com/google/jsonnet/issues/1270 std.assertEqual(std.parseYaml(' \n'), null) && +// Stream input, regression test for https://github.com/google/jsonnet/issues/1148 +std.assertEqual(std.parseYaml('1\n---'), [1, null]) && std.assertEqual(std.parseYaml('{}'), {}) && std.assertEqual(std.parseYaml('[]'), []) && std.assertEqual( diff --git a/sjsonnet/test/src-jvm-native/sjsonnet/FileTests.scala b/sjsonnet/test/src-jvm-native/sjsonnet/FileTests.scala index 7af3a7bd..0160a9ea 100644 --- a/sjsonnet/test/src-jvm-native/sjsonnet/FileTests.scala +++ b/sjsonnet/test/src-jvm-native/sjsonnet/FileTests.scala @@ -3,25 +3,18 @@ package sjsonnet import utest.* object FileTests extends BaseFileTests { - val testDataSkippedTests: Set[String] = { - val nativeOnly = - if (isScalaNative) - Set( - // These tests are skipped in Scala Native because we can't catch the stack overflow and recover. - "error.obj_recursive_manifest.jsonnet", - "error.recursive_object_non_term.jsonnet", - "error.recursive_import.jsonnet", - "error.recursive_function_nonterm.jsonnet", - "error.function_infinite_default.jsonnet", - "error.obj_recursive.jsonnet" - ) - else Set.empty[String] - nativeOnly ++ Set( - // ParseError is thrown directly (not caught by Interpreter.interpret) for deeply nested arrays, - // causing the test framework to crash rather than producing a Left error result. - "error.parse.deep_array_nesting.jsonnet" - ) - } + val testDataSkippedTests: Set[String] = + if (isScalaNative) + Set( + // These tests are skipped in Scala Native because we can't catch the stack overflow and recover. + "error.obj_recursive_manifest.jsonnet", + "error.recursive_object_non_term.jsonnet", + "error.recursive_import.jsonnet", + "error.recursive_function_nonterm.jsonnet", + "error.function_infinite_default.jsonnet", + "error.obj_recursive.jsonnet" + ) + else Set.empty[String] val goTestDataSkippedTests: Set[String] = Set( // We support base64 of unicode strings diff --git a/sjsonnet/test/src/sjsonnet/ParseYamlTests.scala b/sjsonnet/test/src/sjsonnet/ParseYamlTests.scala index 02d7c798..fccd82d8 100644 --- a/sjsonnet/test/src/sjsonnet/ParseYamlTests.scala +++ b/sjsonnet/test/src/sjsonnet/ParseYamlTests.scala @@ -35,5 +35,14 @@ object ParseYamlTests extends TestSuite { """{"a":"\"\\n\n\r\f\b\t""" + "\u263A" + """","l":9}""" // <\n> <\r> <\f> <\b> <\t> ) } + test { + // Regression test for https://github.com/google/jsonnet/issues/1148 + // Empty document after --- should be treated as null + eval("std.parseYaml('1\\n---')") ==> ujson.Value("""[1,null]""") + } + test { + // Test that trailing empty document with whitespace is handled + eval("std.parseYaml('1\\n---\\n')") ==> ujson.Value("""[1,null]""") + } } } diff --git a/sync_test_suites.sh b/sync_test_suites.sh index 2dbe61df..e0c2a909 100755 --- a/sync_test_suites.sh +++ b/sync_test_suites.sh @@ -5,7 +5,9 @@ # This script copies .jsonnet source files and .golden files from upstream repos # into sjsonnet's test resource directories. Only *.jsonnet and *.golden files are # synced. Lint-related files (*.linter.*) and formatter-related files (*.fmt.*) -# from google/jsonnet are excluded. A .jsonnet file is only synced if it has a +# from google/jsonnet are excluded. Files listed in per-suite .sync_ignore files +# are also excluded (one .jsonnet filename per line, comments start with '#'). +# A .jsonnet file is only synced if it has a # corresponding valid .golden file (upstream or already present locally), so that # Scala tests always find a matching .jsonnet.golden for each .jsonnet file. # For new .jsonnet files that have no golden file after syncing, @@ -54,7 +56,19 @@ sync_test_files() { return fi - echo " Syncing $suite_name..." + # Load .sync_ignore file from target directory (if it exists) + local ignore_file="$target_dir/.sync_ignore" + local ignore_stems_file + ignore_stems_file=$(mktemp) + if [ -f "$ignore_file" ]; then + # Strip comments and blank lines, extract stems (remove .jsonnet extension) + grep -v '^\s*#' "$ignore_file" | grep -v '^\s*$' | sed 's/\.jsonnet$//' > "$ignore_stems_file" + local ignore_count + ignore_count=$(wc -l < "$ignore_stems_file" | tr -d ' ') + echo " Syncing $suite_name... ($ignore_count file(s) in .sync_ignore)" + else + echo " Syncing $suite_name..." + fi local before_jsonnet_count before_jsonnet_count=$(find "$target_dir" -maxdepth 1 -name '*.jsonnet' 2>/dev/null | wc -l | tr -d ' ') @@ -86,7 +100,12 @@ sync_test_files() { continue fi # Extract stem: a.jsonnet.golden -> a - echo "${basename%.jsonnet.golden}" >> "$golden_stems_file" + local stem="${basename%.jsonnet.golden}" + # Skip stems listed in .sync_ignore + if grep -Fqx "$stem" "$ignore_stems_file" 2>/dev/null; then + continue + fi + echo "$stem" >> "$golden_stems_file" done # Check *.golden files (that are not *.jsonnet.golden, skip directories) @@ -102,6 +121,10 @@ sync_test_files() { fi # Extract stem: a.golden -> a local stem="${basename%.golden}" + # Skip stems listed in .sync_ignore + if grep -qx "$stem" "$ignore_stems_file" 2>/dev/null; then + continue + fi # Only count if corresponding .jsonnet exists upstream if [ -f "$source_dir/${stem}.jsonnet" ]; then echo "$stem" >> "$golden_stems_file" @@ -136,8 +159,13 @@ sync_test_files() { # Extract stem: a.jsonnet -> a local stem="${basename%.jsonnet}" + # Skip files listed in .sync_ignore + if grep -Fqx "$stem" "$ignore_stems_file" 2>/dev/null; then + continue + fi + # Skip .jsonnet files that have no valid golden file (upstream or local) - if ! grep -qx "$stem" "$golden_stems_file" 2>/dev/null; then + if ! grep -Fqx "$stem" "$golden_stems_file" 2>/dev/null; then skipped_no_golden=$((skipped_no_golden + 1)) continue fi @@ -172,6 +200,12 @@ sync_test_files() { continue fi + # Skip files listed in .sync_ignore + local stem="${basename%.jsonnet.golden}" + if grep -Fqx "$stem" "$ignore_stems_file" 2>/dev/null; then + continue + fi + local dest_file="$target_dir/$basename" # Only copy new golden files, never overwrite existing ones @@ -202,6 +236,12 @@ sync_test_files() { # Derive the corresponding .jsonnet filename: a.golden -> a.jsonnet local stem="${basename%.golden}" + + # Skip files listed in .sync_ignore + if grep -Fqx "$stem" "$ignore_stems_file" 2>/dev/null; then + continue + fi + local jsonnet_file="$source_dir/${stem}.jsonnet" # Only sync if the corresponding .jsonnet file exists upstream @@ -225,6 +265,7 @@ sync_test_files() { echo " .golden: Before=$before_golden_count, After=$after_golden_count (New=$new_golden, Updated=$updated_golden)" rm -f "$golden_stems_file" + rm -f "$ignore_stems_file" } # Sync C++ test suite (google/jsonnet test_suite -> sjsonnet test_suite)