From 429ee1314fd5ed1e14fb598b0b1f6cc004605280 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 11 Feb 2026 13:21:11 +0100 Subject: [PATCH 01/17] basic setup for fixing #2633 by wrapping the character buffer under the JsonReader. idea via @DavyLandman --- src/org/rascalmpl/library/lang/json/IO.java | 4 + .../lang/json/internal/JsonValueReader.java | 91 +++++++++++++++++-- 2 files changed, 86 insertions(+), 9 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index f359b3a92d..b1ea8a9bad 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -10,8 +10,10 @@ *******************************************************************************/ package org.rascalmpl.library.lang.json; +import java.io.FilterReader; import java.io.IOException; import java.io.OutputStreamWriter; +import java.io.Reader; import java.io.StringReader; import java.io.StringWriter; import java.nio.charset.Charset; @@ -162,4 +164,6 @@ public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFor throw RuntimeExceptionFactory.io(e); } } + + } diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 5af18d864d..dc7f8f727f 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -23,7 +23,9 @@ package org.rascalmpl.library.lang.json.internal; import java.io.EOFException; +import java.io.FilterReader; import java.io.IOException; +import java.io.Reader; import java.io.StringReader; import java.lang.invoke.MethodHandles; import java.lang.invoke.VarHandle; @@ -91,14 +93,23 @@ public class JsonValueReader { private final class ExpectedTypeDispatcher implements ITypeVisitor { private final JsonReader in; + private final OriginTrackingReader tracker; + private int offset = 0; private int lastPos = 0; private int lastLimit = 0; - private boolean stopTracking = true; /* origin tracking is turned off while we work on debugging the offsets on another PR */ + private boolean stopTracking = false; + + private ExpectedTypeDispatcher(JsonReader in) { + this.in = in; + this.stopTracking = true; + this.tracker = null; + } - private ExpectedTypeDispatcher(JsonReader in, boolean noTracking) { + public ExpectedTypeDispatcher(JsonReader in, OriginTrackingReader tracker, boolean disableTracking) { this.in = in; - this.stopTracking = true; // noTracking; NB origin tracking is turned off while we work on debugging the offsets on another PR + this.tracker = tracker; + this.stopTracking = disableTracking; } @Override @@ -465,11 +476,10 @@ private int getPos() { var internalPos = (int) posHandler.get(in); var internalLimit = getLimit(); + var baseOffset = tracker.getLimitOffset(); if (internalPos < lastPos) { - // so we detected we are in trouble, but we do not have enough information for a solution here. - // TODO: fix this code in another PR by wrapping the CharacterReader. - offset = offset + (lastLimit - lastPos) + internalPos /* gson copies the tail of the buffer to the front */; + offset = baseOffset + internalPos; } else { // the offset advances by the number of parsed characters @@ -860,7 +870,7 @@ public IValue visitNumber(Type type) throws IOException { if (numberString.contains("r")) { return vf.rational(numberString); } - if (numberString.matches(".*[\\.eE].*")) { + if (numberString.matches(".*[\.eE].*")) { return vf.real(numberString); } else { @@ -1067,7 +1077,8 @@ public JsonValueReader setParsers(IFunction parsers) { } /** - * Read and validate a Json stream as an IValue + * Read and validate a Json stream as an IValue. This version does not support accurate error messages + * or origin tracking. * * @param in json stream * @param expected type to validate against (recursively) @@ -1075,7 +1086,8 @@ public JsonValueReader setParsers(IFunction parsers) { * @throws IOException when either a parse error or a validation error occurs */ public IValue read(JsonReader in, Type expected) throws IOException { - var dispatch = new ExpectedTypeDispatcher(in, disableTracking); + assert !trackOrigins : "use the other read method to track Json origins"; + var dispatch = new ExpectedTypeDispatcher(in); try { var result = expected.accept(dispatch); @@ -1090,4 +1102,65 @@ public IValue read(JsonReader in, Type expected) throws IOException { throw dispatch.parseErrorHere(e.getMessage()); } } + + /** + * Read and validate a Json stream as an IValue. This version supports accurate error messages + * and origin tracking. + * + * @param in json stream + * @param expected type to validate against (recursively) + * @return an IValue of the expected type + * @throws IOException when either a parse error or a validation error occurs + */ + public IValue read(Reader in, Type expected) throws IOException { + try (OriginTrackingReader wrappedIn = new OriginTrackingReader(in); JsonReader jsonIn = new JsonReader(wrappedIn)) { + var dispatch = new ExpectedTypeDispatcher(jsonIn, wrappedIn, disableTracking); + + try { + var result = expected.accept(dispatch); + if (result == null) { + throw new JsonParseException( + "null occurred outside an optionality context and without a registered representation."); + } + return result; + } catch (EOFException | JsonParseException | NumberFormatException | MalformedJsonException | IllegalStateException | NullPointerException e) { + throw dispatch.parseErrorHere(e.getMessage()); + } + } + } + + /** + * This wraps a normal reader to make it possible for a client to detect accurate + * character offsets (> buffersize) in a large file, even if the underlying stream is buffered. + * + * This implementation is tightly coupled (semantically) with the internals of JsonReader. It provides + * just enough information, together with internal private fields of JsonReader, to compute Rascal-required + * offsets. We get only the character offset in the file, at the start of each streamed buffer contents. + * That should be just enough information to recompute the actual offset of every Json element, using the + * current position in the buffer. + */ + public static class OriginTrackingReader extends FilterReader { + private int offset = 0; + private int limit = 0; + + protected OriginTrackingReader(Reader in) { + super(in); + } + + @Override + public int read(char[] cbuf, int off, int len) throws IOException { + // we've read until here before we were reset to 0. + offset += limit; + + // get the new limit + limit = in.read(cbuf, off, len); + + // and return it. + return limit; + } + + public int getLimitOffset() { + return offset; + } + } } From fcac23c2120d073fd53626f7270fa53b35cd5eda Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 11 Feb 2026 13:25:02 +0100 Subject: [PATCH 02/17] cleaning up a bit --- .../lang/json/internal/JsonValueReader.java | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index dc7f8f727f..e44dc549fb 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -97,7 +97,6 @@ private final class ExpectedTypeDispatcher implements ITypeVisitor Date: Wed, 11 Feb 2026 14:07:03 +0100 Subject: [PATCH 03/17] this works. but now a better way to detect a read --- src/org/rascalmpl/library/lang/json/IO.java | 49 +++++++++++++------ .../lang/json/internal/JsonValueReader.java | 4 -- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index b1ea8a9bad..b83b54a61e 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -63,22 +63,41 @@ public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, parsers = null; } - try (JsonReader in = new JsonReader(URIResolverRegistry.getInstance().getCharacterReader(loc))) { - in.setLenient(lenient.getValue()); - return new JsonValueReader(values, store, monitor, loc) - .setCalendarFormat(dateTimeFormat.getValue()) - .setParsers(parsers) - .setNulls(unreify(nulls)) - .setExplicitConstructorNames(explicitConstructorNames.getValue()) - .setExplicitDataTypes(explicitDataTypes.getValue()) - .setTrackOrigins(trackOrigins.getValue()) - .read(in, start); - } - catch (IOException e) { - throw RuntimeExceptionFactory.io(e); + if (trackOrigins.getValue()) { + try (Reader in = URIResolverRegistry.getInstance().getCharacterReader(loc)) { + return new JsonValueReader(values, store, monitor, loc) + .setCalendarFormat(dateTimeFormat.getValue()) + .setParsers(parsers) + .setNulls(unreify(nulls)) + .setExplicitConstructorNames(explicitConstructorNames.getValue()) + .setExplicitDataTypes(explicitDataTypes.getValue()) + .setTrackOrigins(trackOrigins.getValue()) + .read(in, start); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + catch (NullPointerException e) { + throw RuntimeExceptionFactory.io("NPE in error handling code"); + } } - catch (NullPointerException e) { - throw RuntimeExceptionFactory.io("NPE in error handling code"); + else { + try (JsonReader in = new JsonReader(URIResolverRegistry.getInstance().getCharacterReader(loc))) { + in.setLenient(lenient.getValue()); + return new JsonValueReader(values, store, monitor, loc) + .setCalendarFormat(dateTimeFormat.getValue()) + .setParsers(parsers) + .setNulls(unreify(nulls)) + .setExplicitConstructorNames(explicitConstructorNames.getValue()) + .setExplicitDataTypes(explicitDataTypes.getValue()) + .read(in, start); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } + catch (NullPointerException e) { + throw RuntimeExceptionFactory.io("NPE in error handling code"); + } } } diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index e44dc549fb..a90fea924b 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -1027,10 +1027,6 @@ public JsonValueReader setNulls(Map nulls) { public JsonValueReader setTrackOrigins(boolean trackOrigins) { this.trackOrigins = trackOrigins; - if (trackOrigins) { - monitor.warning("The origin tracking feature of the JSON parser is temporarily disabled.", src); - this.trackOrigins = false; - } return this; } From 6b0804ee52c7e8ccbd0a968a227a025d2b201082 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 11 Feb 2026 14:15:26 +0100 Subject: [PATCH 04/17] knowing when to start from the new tracker position is important --- .../lang/json/internal/JsonValueReader.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index a90fea924b..f91bd8b2ad 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -96,6 +96,7 @@ private final class ExpectedTypeDispatcher implements ITypeVisitor Date: Thu, 12 Feb 2026 09:44:05 +0100 Subject: [PATCH 05/17] cleaning up parameters and error reporting --- .../exceptions/RuntimeExceptionFactory.java | 8 ++ src/org/rascalmpl/library/lang/json/IO.java | 87 +++++++------------ src/org/rascalmpl/library/lang/json/IO.rsc | 9 +- .../lang/json/internal/JsonValueReader.java | 52 ++++++++--- 4 files changed, 88 insertions(+), 68 deletions(-) diff --git a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java index 335e46da2e..0d150a558f 100644 --- a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java +++ b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java @@ -98,6 +98,7 @@ public class RuntimeExceptionFactory { // NotImplemented public static final Type ParseError = TF.constructor(TS, Exception, "ParseError", TF.sourceLocationType(), "location"); + public static final Type NoOffsetParseError = TF.constructor(TS, Exception, "NoOffsetParseError", TF.sourceLocationType(), "location", TF.integerType(), "line", TF.integerType(), "column"); public static final Type PathNotFound = TF.constructor(TS,Exception,"PathNotFound",TF.sourceLocationType(), "location"); @@ -684,6 +685,11 @@ public static Throw jsonParseError(ISourceLocation loc, String cause, String pat .asWithKeywordParameters().setParameter("path", VF.string(path))); } + public static Throw jsonParseError(ISourceLocation file, int line, int col, String cause, String path) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'jsonParseError'"); + } + public static Throw parseError(ISourceLocation loc, AbstractAST ast, StackTrace trace) { return new Throw(VF.constructor(ParseError, loc), ast != null ? ast.getLocation() : null, trace); @@ -793,4 +799,6 @@ public static Throw parseErrorRecovery(IValue trigger, ISourceLocation loc) { public static Throw parseErrorRecoveryNoSuchField(String name, ISourceLocation loc) { return new Throw(VF.constructor(ParseErrorRecovery, VF.constructor(NoSuchField, VF.string(name)), loc)); } + + } diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index b83b54a61e..c9f98152ff 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -10,7 +10,6 @@ *******************************************************************************/ package org.rascalmpl.library.lang.json; -import java.io.FilterReader; import java.io.IOException; import java.io.OutputStreamWriter; import java.io.Reader; @@ -40,7 +39,6 @@ import io.usethesource.vallang.type.Type; import io.usethesource.vallang.type.TypeStore; -import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonWriter; public class IO { @@ -53,82 +51,55 @@ public IO(IRascalValueFactory values, IRascalMonitor monitor) { this.monitor = monitor; } - public IValue readJSON(IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, - IFunction parsers, IMap nulls, IBool explicitConstructorNames, IBool explicitDataTypes) { + private IValue doReadJSON(Reader in, + IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, + IFunction parsers, IMap nulls, IBool explicitConstructorNames, IBool explicitDataTypes) throws IOException { + TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); - + if (parsers.getType() instanceof ReifiedType && parsers.getType().getTypeParameters().getFieldType(0).isTop()) { // ignore the default parser parsers = null; } - if (trackOrigins.getValue()) { - try (Reader in = URIResolverRegistry.getInstance().getCharacterReader(loc)) { - return new JsonValueReader(values, store, monitor, loc) - .setCalendarFormat(dateTimeFormat.getValue()) - .setParsers(parsers) - .setNulls(unreify(nulls)) - .setExplicitConstructorNames(explicitConstructorNames.getValue()) - .setExplicitDataTypes(explicitDataTypes.getValue()) - .setTrackOrigins(trackOrigins.getValue()) - .read(in, start); - } - catch (IOException e) { - throw RuntimeExceptionFactory.io(e); - } - catch (NullPointerException e) { - throw RuntimeExceptionFactory.io("NPE in error handling code"); - } - } - else { - try (JsonReader in = new JsonReader(URIResolverRegistry.getInstance().getCharacterReader(loc))) { - in.setLenient(lenient.getValue()); - return new JsonValueReader(values, store, monitor, loc) + try { + return new JsonValueReader(values, store, monitor, loc) .setCalendarFormat(dateTimeFormat.getValue()) - .setParsers(parsers) + .setLenient(lenient.getValue()) + .setParsers(parsers) .setNulls(unreify(nulls)) .setExplicitConstructorNames(explicitConstructorNames.getValue()) .setExplicitDataTypes(explicitDataTypes.getValue()) + .setTrackOrigins(trackOrigins.getValue()) .read(in, start); - } - catch (IOException e) { - throw RuntimeExceptionFactory.io(e); - } - catch (NullPointerException e) { - throw RuntimeExceptionFactory.io("NPE in error handling code"); - } + } + catch (NullPointerException e) { + throw RuntimeExceptionFactory.io("NPE in error handling code"); } } + + public IValue readJSON( + IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, + IFunction parsers, IMap nulls, IBool explicitConstructorNames, IBool explicitDataTypes) { - private Map unreify(IMap nulls) { - var tr = new TypeReifier(values); - return nulls.stream().map(t -> (ITuple) t) - .collect(Collectors.toMap(t -> tr.valueToType((IConstructor) t.get(0)), t -> t.get(1))); + try (Reader in = URIResolverRegistry.getInstance().getCharacterReader(loc)) { + return doReadJSON(in, type, loc, dateTimeFormat, lenient, trackOrigins, parsers, nulls, explicitConstructorNames, explicitDataTypes); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e); + } } public IValue parseJSON(IValue type, IString src, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IFunction parsers, IMap nulls, IBool explicitConstructorNames, IBool explicitDataTypes) { - TypeStore store = new TypeStore(); - Type start = new TypeReifier(values).valueToType((IConstructor) type, store); - - try (JsonReader in = new JsonReader(new StringReader(src.getValue()))) { - in.setLenient(lenient.getValue()); - return new JsonValueReader(values, store, monitor,null) - .setCalendarFormat(dateTimeFormat.getValue()) - .setParsers(parsers) - .setNulls(unreify(nulls)) - .setTrackOrigins(trackOrigins.getValue()) - .setExplicitConstructorNames(explicitConstructorNames.getValue()) - .setExplicitDataTypes(explicitDataTypes.getValue()) - .read(in, start); + + try (Reader in = new StringReader(src.getValue())) { + return doReadJSON(in, type, null, dateTimeFormat, lenient, trackOrigins, parsers, nulls, explicitConstructorNames, explicitDataTypes); } catch (IOException e) { throw RuntimeExceptionFactory.io(e); } - catch (NullPointerException e) { - throw RuntimeExceptionFactory.io("NPE"); - } } public void writeJSON(ISourceLocation loc, IValue value, IBool unpackedLocations, IString dateTimeFormat, @@ -184,5 +155,9 @@ public IString asJSON(IValue value, IBool unpackedLocations, IString dateTimeFor } } - + private Map unreify(IMap nulls) { + var tr = new TypeReifier(values); + return nulls.stream().map(t -> (ITuple) t) + .collect(Collectors.toMap(t -> tr.valueToType((IConstructor) t.get(0)), t -> t.get(1))); + } } diff --git a/src/org/rascalmpl/library/lang/json/IO.rsc b/src/org/rascalmpl/library/lang/json/IO.rsc index 8c71c94ed7..2198c8e28d 100644 --- a/src/org/rascalmpl/library/lang/json/IO.rsc +++ b/src/org/rascalmpl/library/lang/json/IO.rsc @@ -74,7 +74,14 @@ import Exception; * `cause` is a factual diagnosis of what was expected at that position, versus what was found. * `path` is a path query string into the JSON value from the root down to the leaf where the error was detected. } -data RuntimeException(str cause="", str path=""); +@benefits{ +* ((NoOffsetParseError)) is for when accurate offset tracking is turned off. Typically this is _on_ +even if `trackOrigins=false`, when we call the json parsers from Rascal. +} +data RuntimeException(str cause="", str path="") + = ParseError(loc location) + | NoOffsetParseError(loc location, int line, int column) + ; private str DEFAULT_DATETIME_FORMAT = "yyyy-MM-dd\'T\'HH:mm:ssZ"; diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index f91bd8b2ad..65829723cf 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -65,6 +65,7 @@ import io.usethesource.vallang.type.TypeStore; import com.google.gson.JsonParseException; +import com.google.gson.Strictness; import com.google.gson.stream.JsonReader; import com.google.gson.stream.JsonToken; import com.google.gson.stream.MalformedJsonException; @@ -82,14 +83,16 @@ public class JsonValueReader { private final IRascalMonitor monitor; private ISourceLocation src; private boolean trackOrigins = false; - private boolean disableTracking = false; + private boolean stopTracking = false; private VarHandle posHandler; private VarHandle lineHandler; private VarHandle lineStartHandler; private boolean explicitConstructorNames; private boolean explicitDataTypes; + private boolean lenient; private IFunction parsers; private Map nulls = Collections.emptyMap(); + private final class ExpectedTypeDispatcher implements ITypeVisitor { private final JsonReader in; @@ -98,18 +101,27 @@ private final class ExpectedTypeDispatcher implements ITypeVisitor Date: Thu, 12 Feb 2026 09:44:59 +0100 Subject: [PATCH 06/17] fixed broken header --- src/org/rascalmpl/library/lang/json/IO.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/IO.java b/src/org/rascalmpl/library/lang/json/IO.java index c9f98152ff..e5b91d89fd 100644 --- a/src/org/rascalmpl/library/lang/json/IO.java +++ b/src/org/rascalmpl/library/lang/json/IO.java @@ -5,8 +5,10 @@ * * Contributors: * - * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI * Mark Hills - Mark.Hills@cwi.nl (CWI) * Arnold - * Lankamp - Arnold.Lankamp@cwi.nl * Bert Lisser - Bert.Lisser@cwi.nl + * * Jurgen J. Vinju - Jurgen.Vinju@cwi.nl - CWI + * * Mark Hills - Mark.Hills@cwi.nl (CWI) + * * Arnold - Lankamp - Arnold.Lankamp@cwi.nl + * * Bert Lisser - Bert.Lisser@cwi.nl *******************************************************************************/ package org.rascalmpl.library.lang.json; @@ -54,7 +56,7 @@ public IO(IRascalValueFactory values, IRascalMonitor monitor) { private IValue doReadJSON(Reader in, IValue type, ISourceLocation loc, IString dateTimeFormat, IBool lenient, IBool trackOrigins, IFunction parsers, IMap nulls, IBool explicitConstructorNames, IBool explicitDataTypes) throws IOException { - + TypeStore store = new TypeStore(); Type start = new TypeReifier(values).valueToType((IConstructor) type, store); From db3001ccafa730200f2810a94f4aafbc027cc412 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 12 Feb 2026 10:13:26 +0100 Subject: [PATCH 07/17] testing.json is now longer to no longer need the carriage returns to trigger #2633 --- .../lang/json/internal/JsonValueReader.java | 3 +- .../tests/library/lang/json/longcomment.json | 8 ++ .../tests/library/lang/json/longstring.json | 6 ++ .../tests/library/lang/json/testing.json | 83 ++++++++++--------- 4 files changed, 58 insertions(+), 42 deletions(-) create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longcomment.json create mode 100644 src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longstring.json diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 65829723cf..b310a0f35b 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -752,7 +752,8 @@ else if (!explicitDataTypes && "_type".equals(label)) { in.endObject(); int endPos = getPos(); - assert endPos >= startPos : "as assumpion on the internals of gson"; + assert endPos >= startPos : "offset tracking messed up while stopTracking is " + stopTracking + " and trackOrigins is " + trackOrigins; + int endLine = getLine(); int endCol = getCol(); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longcomment.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longcomment.json new file mode 100644 index 0000000000..001e102dca --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longcomment.json @@ -0,0 +1,8 @@ +// 66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk +{ + "name": "this object has a very long comment of >1024 characters", + + "nested" : { + "type": "" + } +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longstring.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longstring.json new file mode 100644 index 0000000000..a7830c82ab --- /dev/null +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/longstring.json @@ -0,0 +1,6 @@ +{ + "name": "this object has a very long string of 1024 characters", + "nested" : { + "type": "66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk" + } +} \ No newline at end of file diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json index 4211534ee3..bb778c26d1 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json @@ -1,41 +1,42 @@ -{ - "name": "TESTING Adding one more character to this data causes the test to fail", - "type": "type_abcd", - "nested_abc": [ - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - }, - { - "type": "line", - "property__1": { "y": { "value": 2, "absolute": false } }, - "property__2": {} - } - ] -} +// This comment pushes the length of the file beyong the 1024 limit. +{ + "name": "TESTING Adding one more character to this data caused the test to fail", + "type": "type_abcd", + "nested_abc": [ + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + }, + { + "type": "line", + "property__1": { "y": { "value": 2, "absolute": false } }, + "property__2": {} + } + ] +} From a191a8258b85ccc9e3f9278b6535ad792f06f34f Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 12 Feb 2026 10:47:16 +0100 Subject: [PATCH 08/17] testing and working on asserts --- .../library/lang/json/internal/JsonValueReader.java | 7 ++++++- .../lang/rascal/tests/library/lang/json/JSONIOTests.rsc | 7 +++---- .../lang/rascal/tests/library/lang/json/testing.json | 2 +- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index b310a0f35b..b2451ba5fb 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -482,6 +482,8 @@ public IValue visitBool(Type type) throws IOException { * `internalPos < lastPos` will not have had the opportunity to evaluate to `true`. */ private int getPos() { + assert !(!stopTracking && posHandler == null) : "if we don't have the posHandler stopTracking should be true"; + if (stopTracking) { return 0; } @@ -986,6 +988,10 @@ public JsonValueReader(IValueFactory vf, TypeStore store, IRascalMonitor monitor this.posHandler = privateLookup.findVarHandle(JsonReader.class, "pos", int.class); this.lineHandler = privateLookup.findVarHandle(JsonReader.class, "lineNumber", int.class); this.lineStartHandler = privateLookup.findVarHandle(JsonReader.class, "lineStart", int.class); + + if (posHandler == null || lineHandler == null || lineStartHandler == null) { + stopTracking = true; + } } catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { // we disable the origin tracking if we can not get to the fields @@ -1093,7 +1099,6 @@ public JsonValueReader setLenient(boolean value) { * @throws IOException when either a parse error or a validation error occurs */ public IValue read(JsonReader in, Type expected) throws IOException { - assert !trackOrigins : "use the other read method to track Json origins"; in.setStrictness(lenient ? Strictness.LENIENT : Strictness.LEGACY_STRICT); var dispatch = new ExpectedTypeDispatcher(in); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index 4ea773c85a..0d23ce9f86 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -106,10 +106,10 @@ bool originTest(loc example) { return true; } -@ignore{awaiting fix of 2633} test bool originTracking() { - return originTest(|std:///lang/rascal/tests/library/lang/json/glossary.json|) - && originTest(|std:///lang/rascal/tests/library/lang/json/testing.json|); + files = [ l | loc l <- |std:///lang/rascal/tests/library/lang/json|.ls, l.extension == "json"]; + + return (true | it && originTest(example) | example <- files, bprintln("testing origins of ")); } value numNormalizer(int i) = i % maxLong when abs(i) > maxLong; @@ -181,7 +181,6 @@ value toDefaultValue(real r) = r - round(r) == 0 : fitDouble(r); default value toDefaultValue(value x) = x; -@ignore{awaiting fix of 2633} test bool accurateParseErrors() { ex = readFile(|std:///lang/rascal/tests/library/lang/json/glossary.json|); broken = ex[..size(ex)/2] + ex[size(ex)/2+10..]; diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json index bb778c26d1..6d758bfccf 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json @@ -1,7 +1,7 @@ // This comment pushes the length of the file beyong the 1024 limit. { "name": "TESTING Adding one more character to this data caused the test to fail", - "type": "type_abcd", + "type": "66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk", "nested_abc": [ { "type": "line", From 974101335ac4a40c0fe7b46a1d31d2cea8e8718e Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 12 Feb 2026 12:12:23 +0100 Subject: [PATCH 09/17] make sure tracking is really of when a JsonReader is passed in, to avoid NPEs and other crashes --- .../lang/json/internal/JsonValueReader.java | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index b2451ba5fb..d75a04ef09 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -981,24 +981,23 @@ public JsonValueReader(IValueFactory vf, TypeStore store, IRascalMonitor monitor setCalendarFormat("yyyy-MM-dd'T'HH:mm:ssZ"); - if (src != null) { - try { - var lookup = MethodHandles.lookup(); - var privateLookup = MethodHandles.privateLookupIn(JsonReader.class, lookup); - this.posHandler = privateLookup.findVarHandle(JsonReader.class, "pos", int.class); - this.lineHandler = privateLookup.findVarHandle(JsonReader.class, "lineNumber", int.class); - this.lineStartHandler = privateLookup.findVarHandle(JsonReader.class, "lineStart", int.class); - - if (posHandler == null || lineHandler == null || lineStartHandler == null) { - stopTracking = true; - } - } - catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { - // we disable the origin tracking if we can not get to the fields + + try { + var lookup = MethodHandles.lookup(); + var privateLookup = MethodHandles.privateLookupIn(JsonReader.class, lookup); + this.posHandler = privateLookup.findVarHandle(JsonReader.class, "pos", int.class); + this.lineHandler = privateLookup.findVarHandle(JsonReader.class, "lineNumber", int.class); + this.lineStartHandler = privateLookup.findVarHandle(JsonReader.class, "lineStart", int.class); + + if (posHandler == null) { stopTracking = true; - monitor.warning("Unable to retrieve origin information due to: " + e.getMessage(), src); } } + catch (NoSuchFieldException | SecurityException | IllegalAccessException e) { + // we disable the origin tracking if we can not get to the fields + stopTracking = true; + monitor.warning("Unable to retrieve origin information due to: " + e.getMessage(), src); + } } public JsonValueReader(IValueFactory vf, IRascalMonitor monitor, ISourceLocation src) { @@ -1100,7 +1099,10 @@ public JsonValueReader setLenient(boolean value) { */ public IValue read(JsonReader in, Type expected) throws IOException { in.setStrictness(lenient ? Strictness.LENIENT : Strictness.LEGACY_STRICT); - + + // we can't track accurately because we don't have a handle to the raw buffer under `in` + this.stopTracking = true; + var dispatch = new ExpectedTypeDispatcher(in); try { From 1a328240685a00a143188b0e2d22822ec450898e Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 12 Feb 2026 12:18:03 +0100 Subject: [PATCH 10/17] forgot to implement the degenerate parse error --- src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java | 8 +++++--- .../library/lang/json/internal/JsonValueReader.java | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java index 0d150a558f..fbb88d3653 100644 --- a/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java +++ b/src/org/rascalmpl/exceptions/RuntimeExceptionFactory.java @@ -98,6 +98,8 @@ public class RuntimeExceptionFactory { // NotImplemented public static final Type ParseError = TF.constructor(TS, Exception, "ParseError", TF.sourceLocationType(), "location"); + + // this comes from lang::json::IO public static final Type NoOffsetParseError = TF.constructor(TS, Exception, "NoOffsetParseError", TF.sourceLocationType(), "location", TF.integerType(), "line", TF.integerType(), "column"); public static final Type PathNotFound = TF.constructor(TS,Exception,"PathNotFound",TF.sourceLocationType(), "location"); @@ -686,11 +688,11 @@ public static Throw jsonParseError(ISourceLocation loc, String cause, String pat } public static Throw jsonParseError(ISourceLocation file, int line, int col, String cause, String path) { - // TODO Auto-generated method stub - throw new UnsupportedOperationException("Unimplemented method 'jsonParseError'"); + return new Throw(VF.constructor(NoOffsetParseError, file, VF.integer(line), VF.integer(col)) + .asWithKeywordParameters().setParameter("reason", VF.string(cause)) + .asWithKeywordParameters().setParameter("path", VF.string(path))); } - public static Throw parseError(ISourceLocation loc, AbstractAST ast, StackTrace trace) { return new Throw(VF.constructor(ParseError, loc), ast != null ? ast.getLocation() : null, trace); } diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index d75a04ef09..0d1fb636d3 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -1102,7 +1102,6 @@ public IValue read(JsonReader in, Type expected) throws IOException { // we can't track accurately because we don't have a handle to the raw buffer under `in` this.stopTracking = true; - var dispatch = new ExpectedTypeDispatcher(in); try { From f0356b89648602fc9845eec4e25f648fc3fb72ff Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 12 Feb 2026 12:22:14 +0100 Subject: [PATCH 11/17] cleaning up --- .../lang/json/internal/JsonValueReader.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 0d1fb636d3..636ee7eec3 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -78,15 +78,17 @@ public class JsonValueReader { private static final TypeFactory TF = TypeFactory.getInstance(); private final TypeStore store; - private final IValueFactory vf; - private ThreadLocal format; + private final IValueFactory vf; private final IRascalMonitor monitor; - private ISourceLocation src; - private boolean trackOrigins = false; - private boolean stopTracking = false; + private final ISourceLocation src; private VarHandle posHandler; private VarHandle lineHandler; private VarHandle lineStartHandler; + + /* options */ + private ThreadLocal format; + private boolean trackOrigins = false; + private boolean stopTracking = false; private boolean explicitConstructorNames; private boolean explicitDataTypes; private boolean lenient; @@ -101,7 +103,6 @@ private final class ExpectedTypeDispatcher implements ITypeVisitor Date: Thu, 12 Feb 2026 12:25:58 +0100 Subject: [PATCH 12/17] added some comments --- .../rascalmpl/library/lang/json/internal/JsonValueReader.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 636ee7eec3..326617bc7f 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -492,7 +492,10 @@ private int getPos() { var trackerCount = tracker.getReadCount(); if (readCount < trackerCount) { + // the tracker indicates that `read` has happened and so the buffer has been rewound. readCount = trackerCount; + // we learn from the tracker how far the offset is until the new first character in the buffer + // and we add the current offset since that reset to found our new current offset. offset = tracker.getLimitOffset() + internalPos; } else { @@ -504,6 +507,7 @@ private int getPos() { lastPos = internalPos; try { + // never go below 0. might happen with a parse error at the first character. return Math.max(0, offset - 1); } catch (IllegalArgumentException | SecurityException e) { From 44c9e7714f3dc1802bb878c5ac9f0587e34e13e3 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 12 Feb 2026 12:31:08 +0100 Subject: [PATCH 13/17] reset testing.json --- .../library/lang/rascal/tests/library/lang/json/testing.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json index 6d758bfccf..bb778c26d1 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/testing.json @@ -1,7 +1,7 @@ // This comment pushes the length of the file beyong the 1024 limit. { "name": "TESTING Adding one more character to this data caused the test to fail", - "type": "66XU0Cylu7Sur4Iiae/KOiEr7HnJOWgaHqwPTYusrEsX1Imws/Qxpc9EZISLCq+G60OVedHbctmJF1KY2Vx962khn5xcKZ7IpdavqI1Xi5FVz2PB4nNb2SD4zwxFK1Fz6NhS2B80vftdwtik9ya3CGgMV5d8dllRivihC+hhdY4sLAjdUXiFGSgEqhPIZYbRtFHi4kWNBJd0BJHx1gWEuimd6WDbBS0FyVsRqjqLutmKjy6pOiF0s5XHYYHkm+LjFN+eTT1xV6AKvuqslRt8q7cz2Cy2HRGTmfSunlLCei1Q/Bff/jtGEG+sYsfRZy7FQwa1vNxAGXOGUqOaYrQOrg68Y/TbVnQ2Wq/KFjQIxEEZJWq5z5MKRGZfV+EfQ2gLSjE+owPr+y/AAH4QjSehItW6qYGo0k8Vt7ptD95xNg4K/b2ebXOuJh4vOpOdIXUpRuwZfndVacYT61ozLOGzlukdqofQF5zaE1XuegssUUJNPxvT6gpVF1d3P0MoIVSJpCi4boSgM0xpAYYnC+Trl9B+zz74HIvS3oePc+Bm2CrL3Ap2rC9BreJKRljLwzJdWw/Snnb5ssZJtKI1S4EqelMEOg0vxoUQDYiGGysRHxZBsvTU0sHzKEe85m7ovR3iDnd5mnAhx58BS/YVXemd3zefcisnyAnjreW1g+fiDXG66le90C0lGvBPEg15xRPt298AjGulnWylJ3Q4pKZ367x41L134GvAGtnihuYMnPoSuWo2bakvHHDP2iexVobSK64QzlfXFLHydqNyJ/kSPztvsX5QV1kWIIHiqxGZEttOpbP4OvZDP82/5kvnI4hNv92ZcCdu1eKfgCNrd+cSYBeRXvz2W7KH+Ts2FbZDrwHNQWhvkfgBEpDvk+YwQTZFvMwenj4VA6VmYZvw5pzOiS2t3kdOJz60CZsgCqKaF7slaJV/SeG6zfi5rDkioQJZyB97Tjmed8S8etMexEjrq7q1S2Wkn7uAdDY/cEjhwuH1+YvNlECS6naT+oFu3thk", + "type": "type_abcd", "nested_abc": [ { "type": "line", From d8dec4d1218012d6a4622adcb33a4bc5e739f8d4 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 12 Feb 2026 13:18:57 +0100 Subject: [PATCH 14/17] commented read better and added fix by @DavyLandman for when off!=0 --- .../lang/json/internal/JsonValueReader.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index 326617bc7f..cf32769641 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -1160,8 +1160,11 @@ public IValue read(Reader in, Type expected) throws IOException { * current position in the buffer. */ public static class OriginTrackingReader extends FilterReader { + // offset is always pointing at the point in the file where JsonReader.pos == 0 private int offset = 0; + // limit is always pointing to the amount of no-junk characters in the underlying buffer below buffer.length private int limit = 0; + // readCount increases by 1 with every call to `read` private int readCount = 0; protected OriginTrackingReader(Reader in) { @@ -1170,16 +1173,19 @@ protected OriginTrackingReader(Reader in) { @Override public int read(char[] cbuf, int off, int len) throws IOException { - // we've read until here before we were reset to 0. - offset += limit; + // we've read until here before we were reset to starting point `off`. + // the offset of the new current 0-based buffer starts here: + offset += limit - off; - // get the new limit - limit = in.read(cbuf, off, len); + var charsRead = in.read(cbuf, off, len); + + // get the new limit (to where we've filled the buffer) + limit = charsRead + off; readCount++; - // and return it. - return limit; + // and return the number of characters read. + return charsRead; } public int getLimitOffset() { From 1f6c294a50a080a8cad358fe43bcb4538ceaa540 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 13 Feb 2026 10:17:15 +0100 Subject: [PATCH 15/17] added tests by @davylandman and fixed off-by-one with length of origin positions on objects --- .../lang/json/internal/JsonValueReader.java | 6 ++-- .../tests/library/lang/json/JSONIOTests.rsc | 35 +++++++++++++++++++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index cf32769641..a8870880cb 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -772,7 +772,7 @@ else if (!explicitDataTypes && "_type".equals(label)) { if (trackOrigins && !stopTracking) { kwParams.put(kwParams.containsKey("src") ? "rascal-src" : "src", - vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); + vf.sourceLocation(src, startPos, endPos - startPos, startLine, endLine, startCol, endCol + 1)); } return vf.constructor(cons, args, kwParams); @@ -858,7 +858,7 @@ public IValue visitNode(Type type) throws IOException { if (trackOrigins && !stopTracking) { kws.put(kws.containsKey("src") ? "rascal-src" : "src", - vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); + vf.sourceLocation(src, startPos, endPos - startPos, startLine, endLine, startCol, endCol + 1)); } IValue[] argArray = args.entrySet().stream().sorted((e, f) -> e.getKey().compareTo(f.getKey())) @@ -920,6 +920,7 @@ public IValue visitList(Type type) throws IOException { } IListWriter w = vf.listWriter(); + getPos(); in.beginArray(); while (in.hasNext()) { // here we pass label from the higher context @@ -932,6 +933,7 @@ public IValue visitList(Type type) throws IOException { } in.endArray(); + getPos(); return w.done(); } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index 0d23ce9f86..f9b3dfbd0d 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -311,5 +311,40 @@ test bool explicitDataTypes() { // here we can't be sure to get z() back, but we will get some Enum assert data4(e=Enum _) := parseJSON(#DATA4, json, explicitDataTypes=false); + return true; +} + +data X(loc src=|unkown:///|) = v1(int x=0, str s = ""); + +// BUG: off-by-one in current implementation +loc origin(X x) = x.src[length = x.src.length -1]; + +test bool jsonVerifyOriginCorrect() { + ref = v1(x=123456789); + refExpected = asJSON(ref); + t1 = [v1(s="hoi"), ref]; + writeJSON(|memory:///test.json|, t1); + v = readJSON(#list[X],|memory:///test.json|, trackOrigins=true); + return refExpected == readFile(origin(v[1])); +} + +test bool jsonVerifyOriginCorrectAcrossBufferBoundaries() { + ref = v1(x=123456789); + refExpected = asJSON(ref); + for (sSize <- [900..1024]) { + println(sSize); + t1 = [v1(s="a<}>"), ref]; + writeJSON(|memory:///test.json|, t1); + + // this throws exceptions and asserts if there are bugs with the + // origin tracker. In particular it triggers #2633 + v = readJSON(#list[X],|memory:///test.json|, trackOrigins=true); + + // checking the last element + if (refExpected != readFile(origin(v[1]))) { + println("Failed for : "); + return false; + } + } return true; } \ No newline at end of file From 37077dcb1051c2b5c49ecfeadd5a87c1b34b1acb Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 13 Feb 2026 12:01:20 +0100 Subject: [PATCH 16/17] fixing off-by-ones --- .../lang/json/internal/JsonValueReader.java | 12 ++++---- .../tests/library/lang/json/JSONIOTests.rsc | 9 ++---- .../tests/library/lang/json/glossary.json | 28 +------------------ 3 files changed, 11 insertions(+), 38 deletions(-) diff --git a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java index a8870880cb..19397d1d28 100644 --- a/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java +++ b/src/org/rascalmpl/library/lang/json/internal/JsonValueReader.java @@ -756,13 +756,14 @@ else if (!explicitDataTypes && "_type".equals(label)) { } } - in.endObject(); int endPos = getPos(); - assert endPos >= startPos : "offset tracking messed up while stopTracking is " + stopTracking + " and trackOrigins is " + trackOrigins; + assert endPos > startPos : "offset tracking messed up while stopTracking is " + stopTracking + " and trackOrigins is " + trackOrigins; int endLine = getLine(); int endCol = getCol(); + in.endObject(); + for (int i = 0; i < args.length; i++) { if (args[i] == null) { throw parseErrorHere( @@ -772,7 +773,7 @@ else if (!explicitDataTypes && "_type".equals(label)) { if (trackOrigins && !stopTracking) { kwParams.put(kwParams.containsKey("src") ? "rascal-src" : "src", - vf.sourceLocation(src, startPos, endPos - startPos, startLine, endLine, startCol, endCol + 1)); + vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); } return vf.constructor(cons, args, kwParams); @@ -851,14 +852,15 @@ public IValue visitNode(Type type) throws IOException { } } - in.endObject(); int endPos = getPos(); int endLine = getLine(); int endCol = getCol(); + in.endObject(); + if (trackOrigins && !stopTracking) { kws.put(kws.containsKey("src") ? "rascal-src" : "src", - vf.sourceLocation(src, startPos, endPos - startPos, startLine, endLine, startCol, endCol + 1)); + vf.sourceLocation(src, startPos, endPos - startPos + 1, startLine, endLine, startCol, endCol + 1)); } IValue[] argArray = args.entrySet().stream().sorted((e, f) -> e.getKey().compareTo(f.getKey())) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc index f9b3dfbd0d..ecf73e70eb 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/JSONIOTests.rsc @@ -316,16 +316,13 @@ test bool explicitDataTypes() { data X(loc src=|unkown:///|) = v1(int x=0, str s = ""); -// BUG: off-by-one in current implementation -loc origin(X x) = x.src[length = x.src.length -1]; - test bool jsonVerifyOriginCorrect() { ref = v1(x=123456789); refExpected = asJSON(ref); t1 = [v1(s="hoi"), ref]; writeJSON(|memory:///test.json|, t1); v = readJSON(#list[X],|memory:///test.json|, trackOrigins=true); - return refExpected == readFile(origin(v[1])); + return refExpected == readFile(v[1].src); } test bool jsonVerifyOriginCorrectAcrossBufferBoundaries() { @@ -341,8 +338,8 @@ test bool jsonVerifyOriginCorrectAcrossBufferBoundaries() { v = readJSON(#list[X],|memory:///test.json|, trackOrigins=true); // checking the last element - if (refExpected != readFile(origin(v[1]))) { - println("Failed for : "); + if (refExpected != readFile(v[1].src)) { + println("Failed for : "); return false; } } diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json index 804e8e54c8..9e26dfeeb6 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json @@ -1,27 +1 @@ -{ - "glossary": { - "title": "example glossary", - "line" : 2, - "GlossDiv": { - "line" : 5, - "title": "S", - "GlossList": { - "line" : 8, - "GlossEntry": { - "line" : 10, - "ID": "SGML", - "SortAs": "SGML", - "GlossTerm": "Standard Generalized Markup Language", - "Acronym": "SGML", - "Abbrev": "ISO 8879:1986", - "GlossDef": { - "line" : 17, - "para": "A meta-markup language, used to create markup languages such as DocBook.", - "GlossSeeAlso": ["GML", "XML"] - }, - "GlossSee": "markup" - } - } - } - } -} \ No newline at end of file +{} \ No newline at end of file From b5e1a04d9db327b6e1be89716cdbd2a82baf071b Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 13 Feb 2026 12:02:03 +0100 Subject: [PATCH 17/17] reset --- .../tests/library/lang/json/glossary.json | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json index 9e26dfeeb6..804e8e54c8 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json +++ b/src/org/rascalmpl/library/lang/rascal/tests/library/lang/json/glossary.json @@ -1 +1,27 @@ -{} \ No newline at end of file +{ + "glossary": { + "title": "example glossary", + "line" : 2, + "GlossDiv": { + "line" : 5, + "title": "S", + "GlossList": { + "line" : 8, + "GlossEntry": { + "line" : 10, + "ID": "SGML", + "SortAs": "SGML", + "GlossTerm": "Standard Generalized Markup Language", + "Acronym": "SGML", + "Abbrev": "ISO 8879:1986", + "GlossDef": { + "line" : 17, + "para": "A meta-markup language, used to create markup languages such as DocBook.", + "GlossSeeAlso": ["GML", "XML"] + }, + "GlossSee": "markup" + } + } + } + } +} \ No newline at end of file