Skip to content

Commit aa9ecfe

Browse files
guizmaiiguizmaii
authored
DuckDB: parse slices with omitted bounds or a step (#2794)
* Parse slices with omitted bounds or a step `l[2:]`, `l[:]` and `l[-5:]` failed, `l[2:3]` parsed as the JSON path `2 : 3` and `l[:2]` as the numeric bind `:2`. Inside a subscript, a `:` now separates the bounds of a slice rather than starting a JSON path. Slices also take DuckDB's optional step, `l[1:5:2]`, where a `-` stop stands for the end of the list, `l[1:-:2]`. `RangeExpression` and `ArrayExpression` gain a step and accept missing bounds. * Keep JSON paths nested inside a subscript `l[(SELECT c:a FROM t)]`, `l[f(x, c:a)]` and `l[CASE WHEN c:a = 1 ...]` failed: the subscript depth disabled the JSON path operator in every expression nested in the subscript. Only a `:` at the subscript's own top level, outside brackets, braces, parentheses and CASE, now separates slice bounds.
1 parent fde4f03 commit aa9ecfe

9 files changed

Lines changed: 262 additions & 45 deletions

File tree

‎src/main/java/net/sf/jsqlparser/expression/ArrayExpression.java‎

Lines changed: 16 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ public class ArrayExpression extends ASTNodeAccessImpl implements Expression {
1717
private Expression indexExpression;
1818
private Expression startIndexExpression;
1919
private Expression stopIndexExpression;
20+
private Expression stepExpression;
2021

2122

2223
public ArrayExpression() {
@@ -72,6 +73,19 @@ public void setStopIndexExpression(Expression stopIndexExpression) {
7273
this.stopIndexExpression = stopIndexExpression;
7374
}
7475

76+
public Expression getStepExpression() {
77+
return stepExpression;
78+
}
79+
80+
public void setStepExpression(Expression stepExpression) {
81+
this.stepExpression = stepExpression;
82+
}
83+
84+
public ArrayExpression withStepExpression(Expression stepExpression) {
85+
this.setStepExpression(stepExpression);
86+
return this;
87+
}
88+
7589
@Override
7690
public <T, S> T accept(ExpressionVisitor<T> expressionVisitor, S context) {
7791
return expressionVisitor.visit(this, context);
@@ -82,11 +96,8 @@ public String toString() {
8296
if (indexExpression != null) {
8397
return objExpression.toString() + "[" + indexExpression.toString() + "]";
8498
} else {
85-
return objExpression.toString() + "[" +
86-
(startIndexExpression == null ? "" : startIndexExpression.toString()) +
87-
":" +
88-
(stopIndexExpression == null ? "" : stopIndexExpression.toString()) +
89-
"]";
99+
return objExpression.toString() + "[" + RangeExpression.toString(startIndexExpression,
100+
stopIndexExpression, stepExpression) + "]";
90101
}
91102
}
92103

‎src/main/java/net/sf/jsqlparser/expression/ExpressionVisitorAdapter.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -721,6 +721,9 @@ public <S> T visit(ArrayExpression arrayExpression, S context) {
721721
if (arrayExpression.getStopIndexExpression() != null) {
722722
subExpressions.add(arrayExpression.getStopIndexExpression());
723723
}
724+
if (arrayExpression.getStepExpression() != null) {
725+
subExpressions.add(arrayExpression.getStepExpression());
726+
}
724727
return visitExpressions(arrayExpression, context, subExpressions);
725728
}
726729

@@ -880,7 +883,7 @@ public <S> T visit(TrimFunction trimFunction, S context) {
880883
@Override
881884
public <S> T visit(RangeExpression rangeExpression, S context) {
882885
return visitExpressions(rangeExpression, context, rangeExpression.getStartExpression(),
883-
rangeExpression.getEndExpression());
886+
rangeExpression.getEndExpression(), rangeExpression.getStepExpression());
884887
}
885888

886889
@Override

‎src/main/java/net/sf/jsqlparser/expression/RangeExpression.java‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
public class RangeExpression extends ASTNodeAccessImpl implements Expression {
1515
private Expression startExpression;
1616
private Expression endExpression;
17+
private Expression stepExpression;
1718

1819
public RangeExpression(Expression startExpression, Expression endExpression) {
1920
this.startExpression = startExpression;
@@ -38,9 +39,42 @@ public RangeExpression setEndExpression(Expression endExpression) {
3839
return this;
3940
}
4041

42+
public Expression getStepExpression() {
43+
return stepExpression;
44+
}
45+
46+
public RangeExpression setStepExpression(Expression stepExpression) {
47+
this.stepExpression = stepExpression;
48+
return this;
49+
}
50+
51+
public RangeExpression withStepExpression(Expression stepExpression) {
52+
return setStepExpression(stepExpression);
53+
}
54+
55+
/**
56+
* Renders {@code [start]:[end][:step]}, a missing end before a step as DuckDB's {@code -}.
57+
*/
58+
public static String toString(Expression start, Expression end, Expression step) {
59+
StringBuilder builder = new StringBuilder();
60+
if (start != null) {
61+
builder.append(start);
62+
}
63+
builder.append(":");
64+
if (end != null) {
65+
builder.append(end);
66+
} else if (step != null) {
67+
builder.append("-");
68+
}
69+
if (step != null) {
70+
builder.append(":").append(step);
71+
}
72+
return builder.toString();
73+
}
74+
4175
@Override
4276
public String toString() {
43-
return startExpression + ":" + endExpression;
77+
return toString(startExpression, endExpression, stepExpression);
4478
}
4579

4680
@Override

‎src/main/java/net/sf/jsqlparser/util/TablesNamesFinder.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -351,8 +351,12 @@ public <S> Void visit(TrimFunction trimFunction, S context) {
351351

352352
@Override
353353
public <S> Void visit(RangeExpression rangeExpression, S context) {
354-
rangeExpression.getStartExpression().accept(this, context);
355-
rangeExpression.getEndExpression().accept(this, context);
354+
for (Expression bound : new Expression[] {rangeExpression.getStartExpression(),
355+
rangeExpression.getEndExpression(), rangeExpression.getStepExpression()}) {
356+
if (bound != null) {
357+
bound.accept(this, context);
358+
}
359+
}
356360
return null;
357361
}
358362

@@ -2247,6 +2251,9 @@ public <S> Void visit(ArrayExpression array, S context) {
22472251
if (array.getStopIndexExpression() != null) {
22482252
array.getStopIndexExpression().accept(this, context);
22492253
}
2254+
if (array.getStepExpression() != null) {
2255+
array.getStepExpression().accept(this, context);
2256+
}
22502257
return null;
22512258
}
22522259

‎src/main/java/net/sf/jsqlparser/util/deparser/ExpressionDeParser.java‎

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -859,12 +859,27 @@ public void visit(TrimFunction trimFunction) {
859859

860860
@Override
861861
public <S> StringBuilder visit(RangeExpression rangeExpression, S context) {
862-
rangeExpression.getStartExpression().accept(this, context);
863-
builder.append(":");
864-
rangeExpression.getEndExpression().accept(this, context);
862+
appendSlice(rangeExpression.getStartExpression(), rangeExpression.getEndExpression(),
863+
rangeExpression.getStepExpression(), context);
865864
return builder;
866865
}
867866

867+
private <S> void appendSlice(Expression start, Expression end, Expression step, S context) {
868+
if (start != null) {
869+
start.accept(this, context);
870+
}
871+
builder.append(":");
872+
if (end != null) {
873+
end.accept(this, context);
874+
} else if (step != null) {
875+
builder.append("-");
876+
}
877+
if (step != null) {
878+
builder.append(":");
879+
step.accept(this, context);
880+
}
881+
}
882+
868883
@Override
869884
public <S> StringBuilder visit(TernaryExpression ternaryExpression, S context) {
870885
ternaryExpression.getCondition().accept(this, context);
@@ -1661,13 +1676,8 @@ public <S> StringBuilder visit(ArrayExpression array, S context) {
16611676
if (array.getIndexExpression() != null) {
16621677
array.getIndexExpression().accept(this, context);
16631678
} else {
1664-
if (array.getStartIndexExpression() != null) {
1665-
array.getStartIndexExpression().accept(this, context);
1666-
}
1667-
builder.append(":");
1668-
if (array.getStopIndexExpression() != null) {
1669-
array.getStopIndexExpression().accept(this, context);
1670-
}
1679+
appendSlice(array.getStartIndexExpression(), array.getStopIndexExpression(),
1680+
array.getStepExpression(), context);
16711681
}
16721682

16731683
builder.append("]");

‎src/main/java/net/sf/jsqlparser/util/validation/validator/ExpressionValidator.java‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1041,6 +1041,9 @@ public <S> Void visit(ArrayExpression array, S context) {
10411041
if (array.getStopIndexExpression() != null) {
10421042
array.getStopIndexExpression().accept(this, context);
10431043
}
1044+
if (array.getStepExpression() != null) {
1045+
array.getStepExpression().accept(this, context);
1046+
}
10441047
return null;
10451048
}
10461049

@@ -1207,8 +1210,12 @@ public <S> Void visit(TrimFunction trimFunction, S context) {
12071210

12081211
@Override
12091212
public <S> Void visit(RangeExpression rangeExpression, S context) {
1210-
rangeExpression.getStartExpression().accept(this, context);
1211-
rangeExpression.getEndExpression().accept(this, context);
1213+
for (Expression bound : new Expression[] {rangeExpression.getStartExpression(),
1214+
rangeExpression.getEndExpression(), rangeExpression.getStepExpression()}) {
1215+
if (bound != null) {
1216+
bound.accept(this, context);
1217+
}
1218+
}
12121219
return null;
12131220
}
12141221

‎src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt‎

Lines changed: 106 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -203,6 +203,32 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
203203
// and must not be taken as the JSON path operator
204204
private int mapKeyDepth = 0;
205205

206+
// opening brackets of the enclosing subscripts: a ":" at the top level of a subscript
207+
// separates the bounds of a slice such as l[1:3] and is not the JSON path operator
208+
private final Deque<Token> subscriptOpenings = new ArrayDeque<Token>();
209+
210+
/**
211+
* Whether the pending ":" is at the top level of the innermost subscript, i.e. not nested
212+
* in brackets, braces, parentheses or a CASE expression opened inside that subscript.
213+
*/
214+
private boolean isSliceColonAhead() {
215+
Token colon = getToken(1);
216+
if (subscriptOpenings.isEmpty() || !":".equals(colon.image)) {
217+
return false;
218+
}
219+
int depth = 0;
220+
for (Token t = subscriptOpenings.peek().next; t != null && t != colon; t = t.next) {
221+
if (t.kind == K_CASE || "(".equals(t.image) || "[".equals(t.image)
222+
|| "{".equals(t.image)) {
223+
depth++;
224+
} else if (t.kind == K_END || ")".equals(t.image) || "]".equals(t.image)
225+
|| "}".equals(t.image)) {
226+
depth--;
227+
}
228+
}
229+
return depth == 0;
230+
}
231+
206232
private Expression createOutputArgument(Expression argument) throws ParseException {
207233
Expression value = argument;
208234
if (argument instanceof VariableAssignment) {
@@ -1659,6 +1685,10 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
16591685
|| next.kind >= MIN_NON_RESERVED_WORD && next.kind <= MAX_NON_RESERVED_WORD;
16601686
}
16611687

1688+
private static boolean isSliceDelimiter(Token token) {
1689+
return ":".equals(token.image) || "]".equals(token.image) || ",".equals(token.image);
1690+
}
1691+
16621692
private boolean isAliasAhead() {
16631693
if (isAsOfJoinAhead() || isSemiAntiJoinAhead() || isApproxNearestAhead()) {
16641694
return false;
@@ -7009,7 +7039,7 @@ Column Column() #Column :
70097039
// @todo: we better should return a SEQUENCE instead of a COLUMN
70107040
[ LOOKAHEAD(2) "." <K_NEXTVAL> { data.getNames().add("nextval"); } ]
70117041

7012-
[ LOOKAHEAD(2) arrayConstructor = ArrayConstructor(false) ]
7042+
[ LOOKAHEAD(2) arrayConstructor = Subscript() ]
70137043
{
70147044
Column col = createColumn(data.getNames(), data.getDelimiters());
70157045
if (tk != null) { col.withCommentText(tk.image); }
@@ -11510,29 +11540,85 @@ Expression AddChain():
1151011540

1151111541
Expression ArrayExpression(Expression obj): {
1151211542
Expression expr;
11543+
} {
11544+
expr = ArraySubscript(obj)
11545+
( LOOKAHEAD(2) expr = ArraySubscript(expr) )*
11546+
{ return expr; }
11547+
}
11548+
11549+
ArrayExpression ArraySubscript(Expression obj): {
1151311550
Expression idxExpr = null;
11514-
Expression startExpr = null;
11515-
Expression stopExpr = null;
11551+
RangeExpression slice = null;
11552+
Token opening;
1151611553
} {
11517-
"["
11518-
[LOOKAHEAD(3) idxExpr = ArrayIndexExpression()]
11554+
opening="["
11555+
try {
11556+
{ subscriptOpenings.push(opening); }
11557+
[ LOOKAHEAD(3, { !":".equals(getToken(1).image) }) idxExpr = ArrayIndexExpression() ]
11558+
[ slice = SliceTail(idxExpr) ]
11559+
} finally {
11560+
subscriptOpenings.pop();
11561+
}
11562+
"]"
11563+
{
11564+
return slice == null
11565+
? new ArrayExpression(obj, idxExpr, null, null)
11566+
: new ArrayExpression(obj, null, slice.getStartExpression(),
11567+
slice.getEndExpression()).withStepExpression(slice.getStepExpression());
11568+
}
11569+
}
11570+
11571+
/**
11572+
* The rest of a slice after its optional start: {@code :[stop][:[step]]}. As in DuckDB, a
11573+
* {@code -} stop before a step stands for the end of the list, as in {@code l[1:-:2]}.
11574+
*/
11575+
RangeExpression SliceTail(Expression start): {
11576+
Expression stop = null;
11577+
Expression step = null;
11578+
} {
11579+
":"
11580+
[
11581+
LOOKAHEAD("-" ":") "-"
11582+
|
11583+
LOOKAHEAD({ !isSliceDelimiter(getToken(1)) }) stop = Expression()
11584+
]
11585+
[
11586+
":" [ LOOKAHEAD({ !isSliceDelimiter(getToken(1)) }) step = Expression() ]
11587+
]
11588+
{ return new RangeExpression(start, stop).withStepExpression(step); }
11589+
}
11590+
11591+
/** A subscript of a column, such as {@code col[1]}, {@code col[1, 2]} or {@code col[1:3]}. */
11592+
ArrayConstructor Subscript() : {
11593+
ExpressionList<Expression> expList = new ExpressionList<Expression>();
11594+
Expression exp;
11595+
Token opening;
11596+
} {
11597+
opening="["
11598+
try {
11599+
{ subscriptOpenings.push(opening); }
1151911600
[
11520-
(":" { startExpr=idxExpr; idxExpr=null; })
11521-
[stopExpr = SimpleExpression()]
11601+
exp=SubscriptElement() { expList.add(exp); }
11602+
( "," exp=SubscriptElement() { expList.add(exp); } )*
1152211603
]
11604+
} finally {
11605+
subscriptOpenings.pop();
11606+
}
1152311607
"]"
11524-
{ expr = new ArrayExpression(obj, idxExpr, startExpr, stopExpr); }
11608+
{ return new ArrayConstructor(expList, false); }
11609+
}
11610+
11611+
Expression SubscriptElement():
11612+
{
11613+
Expression expression = null;
11614+
}
11615+
{
1152511616
(
11526-
LOOKAHEAD(2) "["
11527-
[LOOKAHEAD(3) idxExpr = ArrayIndexExpression()]
11528-
[
11529-
(":" { startExpr=idxExpr; idxExpr=null; })
11530-
[stopExpr = SimpleExpression()]
11531-
]
11532-
"]"
11533-
{ expr = new ArrayExpression(expr, idxExpr, startExpr, stopExpr); }
11534-
)*
11535-
{ return expr; }
11617+
LOOKAHEAD({ ":".equals(getToken(1).image) }) expression=SliceTail(null)
11618+
|
11619+
expression=Expression() [ expression=SliceTail(expression) ]
11620+
)
11621+
{ return expression; }
1153611622
}
1153711623

1153811624
/**
@@ -11793,7 +11879,7 @@ Expression PrimaryExpression() #PrimaryExpression:
1179311879
// Check for JSON operands
1179411880
[
1179511881
LOOKAHEAD(2, { !isPostgreSqlJsonAccess(getToken(1))
11796-
&& ((ternaryThenBranchDepth == 0 && mapKeyDepth == 0)
11882+
&& ((ternaryThenBranchDepth == 0 && mapKeyDepth == 0 && !isSliceColonAhead())
1179711883
|| !":".equals(getToken(1).image)) }) (
1179811884
LOOKAHEAD(2) (
1179911885
token="->"
@@ -12161,7 +12247,7 @@ JsonExpression JsonExpression(Expression expr, List<Map.Entry<Expression, String
1216112247

1216212248
(
1216312249
LOOKAHEAD(2, { !isPostgreSqlJsonAccess(getToken(1))
12164-
&& ((ternaryThenBranchDepth == 0 && mapKeyDepth == 0)
12250+
&& ((ternaryThenBranchDepth == 0 && mapKeyDepth == 0 && !isSliceColonAhead())
1216512251
|| !":".equals(getToken(1).image)) }) (
1216612252
token="->"
1216712253
|

0 commit comments

Comments
 (0)