From 4ac1d1bff418cbf83cd877817c0cc88635645dc0 Mon Sep 17 00:00:00 2001 From: Jules Ivanic Date: Tue, 13 Oct 2026 18:56:17 +1100 Subject: [PATCH 1/2] Parse slices with omitted bounds or a step `l[2:]`, `l[:]` and `l[-5:]` failed, `l[2:3]` parsed as the JSON path `2 : 3` and `l[:2]` as the numeric bind `:2`. Inside a subscript, a `:` now separates the bounds of a slice rather than starting a JSON path. Slices also take DuckDB's optional step, `l[1:5:2]`, where a `-` stop stands for the end of the list, `l[1:-:2]`. `RangeExpression` and `ArrayExpression` gain a step and accept missing bounds. --- .../expression/ArrayExpression.java | 21 +++- .../expression/ExpressionVisitorAdapter.java | 5 +- .../expression/RangeExpression.java | 36 ++++++- .../sf/jsqlparser/util/TablesNamesFinder.java | 11 +- .../util/deparser/ExpressionDeParser.java | 30 ++++-- .../validator/ExpressionValidator.java | 11 +- .../net/sf/jsqlparser/parser/JSqlParserCC.jjt | 100 ++++++++++++++---- .../expression/ArrayExpressionTest.java | 43 ++++++++ .../util/TablesNamesFinderTraversalTest.java | 9 +- 9 files changed, 222 insertions(+), 44 deletions(-) diff --git a/src/main/java/net/sf/jsqlparser/expression/ArrayExpression.java b/src/main/java/net/sf/jsqlparser/expression/ArrayExpression.java index e86f34cad8..74072efe34 100644 --- a/src/main/java/net/sf/jsqlparser/expression/ArrayExpression.java +++ b/src/main/java/net/sf/jsqlparser/expression/ArrayExpression.java @@ -17,6 +17,7 @@ public class ArrayExpression extends ASTNodeAccessImpl implements Expression { private Expression indexExpression; private Expression startIndexExpression; private Expression stopIndexExpression; + private Expression stepExpression; public ArrayExpression() { @@ -72,6 +73,19 @@ public void setStopIndexExpression(Expression stopIndexExpression) { this.stopIndexExpression = stopIndexExpression; } + public Expression getStepExpression() { + return stepExpression; + } + + public void setStepExpression(Expression stepExpression) { + this.stepExpression = stepExpression; + } + + public ArrayExpression withStepExpression(Expression stepExpression) { + this.setStepExpression(stepExpression); + return this; + } + @Override public T accept(ExpressionVisitor expressionVisitor, S context) { return expressionVisitor.visit(this, context); @@ -82,11 +96,8 @@ public String toString() { if (indexExpression != null) { return objExpression.toString() + "[" + indexExpression.toString() + "]"; } else { - return objExpression.toString() + "[" + - (startIndexExpression == null ? "" : startIndexExpression.toString()) + - ":" + - (stopIndexExpression == null ? "" : stopIndexExpression.toString()) + - "]"; + return objExpression.toString() + "[" + RangeExpression.toString(startIndexExpression, + stopIndexExpression, stepExpression) + "]"; } } diff --git a/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitorAdapter.java b/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitorAdapter.java index 62ab36cae3..97e5db5469 100644 --- a/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitorAdapter.java +++ b/src/main/java/net/sf/jsqlparser/expression/ExpressionVisitorAdapter.java @@ -715,6 +715,9 @@ public T visit(ArrayExpression arrayExpression, S context) { if (arrayExpression.getStopIndexExpression() != null) { subExpressions.add(arrayExpression.getStopIndexExpression()); } + if (arrayExpression.getStepExpression() != null) { + subExpressions.add(arrayExpression.getStepExpression()); + } return visitExpressions(arrayExpression, context, subExpressions); } @@ -874,7 +877,7 @@ public T visit(TrimFunction trimFunction, S context) { @Override public T visit(RangeExpression rangeExpression, S context) { return visitExpressions(rangeExpression, context, rangeExpression.getStartExpression(), - rangeExpression.getEndExpression()); + rangeExpression.getEndExpression(), rangeExpression.getStepExpression()); } @Override diff --git a/src/main/java/net/sf/jsqlparser/expression/RangeExpression.java b/src/main/java/net/sf/jsqlparser/expression/RangeExpression.java index dd05827d64..326f5b7da5 100644 --- a/src/main/java/net/sf/jsqlparser/expression/RangeExpression.java +++ b/src/main/java/net/sf/jsqlparser/expression/RangeExpression.java @@ -14,6 +14,7 @@ public class RangeExpression extends ASTNodeAccessImpl implements Expression { private Expression startExpression; private Expression endExpression; + private Expression stepExpression; public RangeExpression(Expression startExpression, Expression endExpression) { this.startExpression = startExpression; @@ -38,9 +39,42 @@ public RangeExpression setEndExpression(Expression endExpression) { return this; } + public Expression getStepExpression() { + return stepExpression; + } + + public RangeExpression setStepExpression(Expression stepExpression) { + this.stepExpression = stepExpression; + return this; + } + + public RangeExpression withStepExpression(Expression stepExpression) { + return setStepExpression(stepExpression); + } + + /** + * Renders {@code [start]:[end][:step]}, a missing end before a step as DuckDB's {@code -}. + */ + public static String toString(Expression start, Expression end, Expression step) { + StringBuilder builder = new StringBuilder(); + if (start != null) { + builder.append(start); + } + builder.append(":"); + if (end != null) { + builder.append(end); + } else if (step != null) { + builder.append("-"); + } + if (step != null) { + builder.append(":").append(step); + } + return builder.toString(); + } + @Override public String toString() { - return startExpression + ":" + endExpression; + return toString(startExpression, endExpression, stepExpression); } @Override diff --git a/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java b/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java index 728301e046..74e8e2badf 100644 --- a/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java +++ b/src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java @@ -350,8 +350,12 @@ public Void visit(TrimFunction trimFunction, S context) { @Override public Void visit(RangeExpression rangeExpression, S context) { - rangeExpression.getStartExpression().accept(this, context); - rangeExpression.getEndExpression().accept(this, context); + for (Expression bound : new Expression[] {rangeExpression.getStartExpression(), + rangeExpression.getEndExpression(), rangeExpression.getStepExpression()}) { + if (bound != null) { + bound.accept(this, context); + } + } return null; } @@ -2240,6 +2244,9 @@ public Void visit(ArrayExpression array, S context) { if (array.getStopIndexExpression() != null) { array.getStopIndexExpression().accept(this, context); } + if (array.getStepExpression() != null) { + array.getStepExpression().accept(this, context); + } return null; } diff --git a/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java b/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java index f7f4b187e9..9f2cd62448 100644 --- a/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java +++ b/src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java @@ -843,12 +843,27 @@ public void visit(TrimFunction trimFunction) { @Override public StringBuilder visit(RangeExpression rangeExpression, S context) { - rangeExpression.getStartExpression().accept(this, context); - builder.append(":"); - rangeExpression.getEndExpression().accept(this, context); + appendSlice(rangeExpression.getStartExpression(), rangeExpression.getEndExpression(), + rangeExpression.getStepExpression(), context); return builder; } + private void appendSlice(Expression start, Expression end, Expression step, S context) { + if (start != null) { + start.accept(this, context); + } + builder.append(":"); + if (end != null) { + end.accept(this, context); + } else if (step != null) { + builder.append("-"); + } + if (step != null) { + builder.append(":"); + step.accept(this, context); + } + } + @Override public StringBuilder visit(TernaryExpression ternaryExpression, S context) { ternaryExpression.getCondition().accept(this, context); @@ -1645,13 +1660,8 @@ public StringBuilder visit(ArrayExpression array, S context) { if (array.getIndexExpression() != null) { array.getIndexExpression().accept(this, context); } else { - if (array.getStartIndexExpression() != null) { - array.getStartIndexExpression().accept(this, context); - } - builder.append(":"); - if (array.getStopIndexExpression() != null) { - array.getStopIndexExpression().accept(this, context); - } + appendSlice(array.getStartIndexExpression(), array.getStopIndexExpression(), + array.getStepExpression(), context); } builder.append("]"); diff --git a/src/main/java/net/sf/jsqlparser/util/validation/validator/ExpressionValidator.java b/src/main/java/net/sf/jsqlparser/util/validation/validator/ExpressionValidator.java index 2e20fd69a1..83aa57eee2 100644 --- a/src/main/java/net/sf/jsqlparser/util/validation/validator/ExpressionValidator.java +++ b/src/main/java/net/sf/jsqlparser/util/validation/validator/ExpressionValidator.java @@ -1034,6 +1034,9 @@ public Void visit(ArrayExpression array, S context) { if (array.getStopIndexExpression() != null) { array.getStopIndexExpression().accept(this, context); } + if (array.getStepExpression() != null) { + array.getStepExpression().accept(this, context); + } return null; } @@ -1200,8 +1203,12 @@ public Void visit(TrimFunction trimFunction, S context) { @Override public Void visit(RangeExpression rangeExpression, S context) { - rangeExpression.getStartExpression().accept(this, context); - rangeExpression.getEndExpression().accept(this, context); + for (Expression bound : new Expression[] {rangeExpression.getStartExpression(), + rangeExpression.getEndExpression(), rangeExpression.getStepExpression()}) { + if (bound != null) { + bound.accept(this, context); + } + } return null; } diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index 64172751a5..f90a9d44af 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -203,6 +203,10 @@ public class CCJSqlParser extends AbstractJSqlParser { // and must not be taken as the JSON path operator private int mapKeyDepth = 0; + // depth of subscripts: a ":" there separates the bounds of a slice such as l[1:3] + // and must not be taken as the JSON path operator + private int subscriptDepth = 0; + private Expression createOutputArgument(Expression argument) throws ParseException { Expression value = argument; if (argument instanceof VariableAssignment) { @@ -1571,6 +1575,10 @@ public class CCJSqlParser extends AbstractJSqlParser { || next.kind >= MIN_NON_RESERVED_WORD && next.kind <= MAX_NON_RESERVED_WORD; } + private static boolean isSliceDelimiter(Token token) { + return ":".equals(token.image) || "]".equals(token.image) || ",".equals(token.image); + } + private boolean isAliasAhead() { if (isAsOfJoinAhead() || isSemiAntiJoinAhead() || isApproxNearestAhead()) { return false; @@ -6884,7 +6892,7 @@ Column Column() #Column : // @todo: we better should return a SEQUENCE instead of a COLUMN [ LOOKAHEAD(2) "." { data.getNames().add("nextval"); } ] - [ LOOKAHEAD(2) arrayConstructor = ArrayConstructor(false) ] + [ LOOKAHEAD(2) arrayConstructor = Subscript() ] { Column col = createColumn(data.getNames(), data.getDelimiters()); if (tk != null) { col.withCommentText(tk.image); } @@ -11341,29 +11349,83 @@ Expression AddChain(): Expression ArrayExpression(Expression obj): { Expression expr; +} { + expr = ArraySubscript(obj) + ( LOOKAHEAD(2) expr = ArraySubscript(expr) )* + { return expr; } +} + +ArrayExpression ArraySubscript(Expression obj): { Expression idxExpr = null; - Expression startExpr = null; - Expression stopExpr = null; + RangeExpression slice = null; +} { + "[" + try { + { subscriptDepth++; } + [ LOOKAHEAD(3, { !":".equals(getToken(1).image) }) idxExpr = ArrayIndexExpression() ] + [ slice = SliceTail(idxExpr) ] + } finally { + subscriptDepth--; + } + "]" + { + return slice == null + ? new ArrayExpression(obj, idxExpr, null, null) + : new ArrayExpression(obj, null, slice.getStartExpression(), + slice.getEndExpression()).withStepExpression(slice.getStepExpression()); + } +} + +/** + * The rest of a slice after its optional start: {@code :[stop][:[step]]}. As in DuckDB, a + * {@code -} stop before a step stands for the end of the list, as in {@code l[1:-:2]}. + */ +RangeExpression SliceTail(Expression start): { + Expression stop = null; + Expression step = null; +} { + ":" + [ + LOOKAHEAD("-" ":") "-" + | + LOOKAHEAD({ !isSliceDelimiter(getToken(1)) }) stop = Expression() + ] + [ + ":" [ LOOKAHEAD({ !isSliceDelimiter(getToken(1)) }) step = Expression() ] + ] + { return new RangeExpression(start, stop).withStepExpression(step); } +} + +/** A subscript of a column, such as {@code col[1]}, {@code col[1, 2]} or {@code col[1:3]}. */ +ArrayConstructor Subscript() : { + ExpressionList expList = new ExpressionList(); + Expression exp; } { "[" - [LOOKAHEAD(3) idxExpr = ArrayIndexExpression()] + try { + { subscriptDepth++; } [ - (":" { startExpr=idxExpr; idxExpr=null; }) - [stopExpr = SimpleExpression()] + exp=SubscriptElement() { expList.add(exp); } + ( "," exp=SubscriptElement() { expList.add(exp); } )* ] + } finally { + subscriptDepth--; + } "]" - { expr = new ArrayExpression(obj, idxExpr, startExpr, stopExpr); } + { return new ArrayConstructor(expList, false); } +} + +Expression SubscriptElement(): +{ + Expression expression = null; +} +{ ( - LOOKAHEAD(2) "[" - [LOOKAHEAD(3) idxExpr = ArrayIndexExpression()] - [ - (":" { startExpr=idxExpr; idxExpr=null; }) - [stopExpr = SimpleExpression()] - ] - "]" - { expr = new ArrayExpression(expr, idxExpr, startExpr, stopExpr); } - )* - { return expr; } + LOOKAHEAD({ ":".equals(getToken(1).image) }) expression=SliceTail(null) + | + expression=Expression() [ expression=SliceTail(expression) ] + ) + { return expression; } } /** @@ -11618,7 +11680,7 @@ Expression PrimaryExpression() #PrimaryExpression: // Check for JSON operands [ LOOKAHEAD(2, { !isPostgreSqlJsonAccess(getToken(1)) - && ((ternaryThenBranchDepth == 0 && mapKeyDepth == 0) + && ((ternaryThenBranchDepth == 0 && mapKeyDepth == 0 && subscriptDepth == 0) || !":".equals(getToken(1).image)) }) ( LOOKAHEAD(2) ( token="->" @@ -11983,7 +12045,7 @@ JsonExpression JsonExpression(Expression expr, List TestUtils.assertSqlCanBeParsedAndDeparsed("SELECT a[0].* FROM t", true)); } + @Test + void testColumnSlices() throws JSQLParserException { + for (String sqlStr : new String[] {"SELECT l[2:3] FROM t", "SELECT l[:2] FROM t", + "SELECT l[2:] FROM t", "SELECT l[:] FROM t", "SELECT l[-5:] FROM t", + "SELECT l[1:-1:2] FROM t", "SELECT l[1:-:2] FROM t", "SELECT l[:-:-1] FROM t", + "SELECT l[x:y] FROM t", "SELECT s[2:] > 'a' FROM t"}) { + TestUtils.assertSqlCanBeParsedAndDeparsed(sqlStr, true); + } + + PlainSelect select = (PlainSelect) TestUtils + .assertSqlCanBeParsedAndDeparsed("SELECT l[1:-:2] FROM t", true); + RangeExpression slice = (RangeExpression) select.getSelectItem(0) + .getExpression(Column.class).getArrayConstructor().getExpressions().get(0); + assertEquals("1", slice.getStartExpression().toString()); + assertNull(slice.getEndExpression()); + assertEquals("2", slice.getStepExpression().toString()); + } + + @Test + void testExpressionSlices() throws JSQLParserException { + for (String sqlStr : new String[] {"SELECT f()[2:] FROM t", "SELECT (l)[:2] FROM t", + "SELECT ([1, 2, 3])[1:-:-1]", "SELECT l[1][2:3] FROM t", + "SELECT l[:2][1] FROM t"}) { + TestUtils.assertSqlCanBeParsedAndDeparsed(sqlStr, true); + } + + ArrayExpression slice = (ArrayExpression) CCJSqlParserUtil.parseExpression("f()[x:y:2]"); + assertNull(slice.getIndexExpression()); + assertEquals("x", slice.getStartIndexExpression().toString()); + assertEquals("y", slice.getStopIndexExpression().toString()); + assertEquals("2", slice.getStepExpression().toString()); + } + + @Test + void testEmptyStepIsDropped() throws JSQLParserException { + assertEquals("l[1:2]", CCJSqlParserUtil.parseExpression("l[1:2:]").toString()); + } + + @Test + void testJsonPathOutsideSubscripts() throws JSQLParserException { + assertInstanceOf(JsonExpression.class, CCJSqlParserUtil.parseExpression("c:a[1]")); + } } diff --git a/src/test/java/net/sf/jsqlparser/util/TablesNamesFinderTraversalTest.java b/src/test/java/net/sf/jsqlparser/util/TablesNamesFinderTraversalTest.java index b96ede8d6d..6a78a0d6b5 100644 --- a/src/test/java/net/sf/jsqlparser/util/TablesNamesFinderTraversalTest.java +++ b/src/test/java/net/sf/jsqlparser/util/TablesNamesFinderTraversalTest.java @@ -18,7 +18,6 @@ import net.sf.jsqlparser.JSQLParserException; import net.sf.jsqlparser.expression.ArrayExpression; import net.sf.jsqlparser.expression.Expression; -import net.sf.jsqlparser.expression.JsonExpression; import net.sf.jsqlparser.parser.CCJSqlParserUtil; import net.sf.jsqlparser.schema.Table; import net.sf.jsqlparser.statement.Statement; @@ -193,11 +192,13 @@ void visitsParsedSliceBounds() throws JSQLParserException { } @Test - void preservesJsonArrayIndexTraversal() throws JSQLParserException { + void traversesSliceBounds() throws JSQLParserException { Expression expression = CCJSqlParserUtil.parseExpression("f()[1:2]"); assertThat(expression).isInstanceOf(ArrayExpression.class); - assertThat(((ArrayExpression) expression).getIndexExpression()) - .isInstanceOf(JsonExpression.class); + ArrayExpression slice = (ArrayExpression) expression; + assertThat(slice.getIndexExpression()).isNull(); + assertThat(slice.getStartIndexExpression()).hasToString("1"); + assertThat(slice.getStopIndexExpression()).hasToString("2"); assertThat(new TablesNamesFinder<>().getTables(expression)).isEmpty(); } From 83ca164567b2691b6c7567bb2cf097eba4385a1f Mon Sep 17 00:00:00 2001 From: Jules Ivanic Date: Tue, 13 Oct 2026 19:06:24 +1100 Subject: [PATCH 2/2] Keep JSON paths nested inside a subscript `l[(SELECT c:a FROM t)]`, `l[f(x, c:a)]` and `l[CASE WHEN c:a = 1 ...]` failed: the subscript depth disabled the JSON path operator in every expression nested in the subscript. Only a `:` at the subscript's own top level, outside brackets, braces, parentheses and CASE, now separates slice bounds. --- .../net/sf/jsqlparser/parser/JSqlParserCC.jjt | 46 ++++++++++++++----- .../expression/ArrayExpressionTest.java | 15 ++++++ 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index f90a9d44af..67f1c95017 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -203,9 +203,31 @@ public class CCJSqlParser extends AbstractJSqlParser { // and must not be taken as the JSON path operator private int mapKeyDepth = 0; - // depth of subscripts: a ":" there separates the bounds of a slice such as l[1:3] - // and must not be taken as the JSON path operator - private int subscriptDepth = 0; + // opening brackets of the enclosing subscripts: a ":" at the top level of a subscript + // separates the bounds of a slice such as l[1:3] and is not the JSON path operator + private final Deque subscriptOpenings = new ArrayDeque(); + + /** + * Whether the pending ":" is at the top level of the innermost subscript, i.e. not nested + * in brackets, braces, parentheses or a CASE expression opened inside that subscript. + */ + private boolean isSliceColonAhead() { + Token colon = getToken(1); + if (subscriptOpenings.isEmpty() || !":".equals(colon.image)) { + return false; + } + int depth = 0; + for (Token t = subscriptOpenings.peek().next; t != null && t != colon; t = t.next) { + if (t.kind == K_CASE || "(".equals(t.image) || "[".equals(t.image) + || "{".equals(t.image)) { + depth++; + } else if (t.kind == K_END || ")".equals(t.image) || "]".equals(t.image) + || "}".equals(t.image)) { + depth--; + } + } + return depth == 0; + } private Expression createOutputArgument(Expression argument) throws ParseException { Expression value = argument; @@ -11358,14 +11380,15 @@ Expression ArrayExpression(Expression obj): { ArrayExpression ArraySubscript(Expression obj): { Expression idxExpr = null; RangeExpression slice = null; + Token opening; } { - "[" + opening="[" try { - { subscriptDepth++; } + { subscriptOpenings.push(opening); } [ LOOKAHEAD(3, { !":".equals(getToken(1).image) }) idxExpr = ArrayIndexExpression() ] [ slice = SliceTail(idxExpr) ] } finally { - subscriptDepth--; + subscriptOpenings.pop(); } "]" { @@ -11400,16 +11423,17 @@ RangeExpression SliceTail(Expression start): { ArrayConstructor Subscript() : { ExpressionList expList = new ExpressionList(); Expression exp; + Token opening; } { - "[" + opening="[" try { - { subscriptDepth++; } + { subscriptOpenings.push(opening); } [ exp=SubscriptElement() { expList.add(exp); } ( "," exp=SubscriptElement() { expList.add(exp); } )* ] } finally { - subscriptDepth--; + subscriptOpenings.pop(); } "]" { return new ArrayConstructor(expList, false); } @@ -11680,7 +11704,7 @@ Expression PrimaryExpression() #PrimaryExpression: // Check for JSON operands [ LOOKAHEAD(2, { !isPostgreSqlJsonAccess(getToken(1)) - && ((ternaryThenBranchDepth == 0 && mapKeyDepth == 0 && subscriptDepth == 0) + && ((ternaryThenBranchDepth == 0 && mapKeyDepth == 0 && !isSliceColonAhead()) || !":".equals(getToken(1).image)) }) ( LOOKAHEAD(2) ( token="->" @@ -12045,7 +12069,7 @@ JsonExpression JsonExpression(Expression expr, List