Skip to content

Commit ec693a4

Browse files
harryfoster2006claudemanticore-projects
authored
fix: allow reserved keywords as JSON path names (Snowflake, Databricks) (#2913)
* fix: allow keywords as Snowflake JSON path names Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> * fix: allow keywords after a dot in Snowflake JSON paths and link their AST nodes The previous commit lets a keyword name the first element after ":" in a JSON path. A keyword further along the path still threw a ParseException: SELECT v:a.output, v:a.b.from::INT, v:a[0].from.x[1] FROM t The expression parser reads v:a.output as a qualified column name, and a qualified name does not accept a keyword as a part. JsonPathElement now also looks ahead with isJsonPathWithKeywordAfterDot(), which walks the identifiers, dots and [...] subscripts of the path and reports whether a keyword follows a ".". If so, JsonPathKeyword parses the path, with a new JsonPathStep production taking a keyword after "." as an element name. isKeywordJsonPathName() shares its keyword test with it through isKeywordWord(). A word followed by "(" stays a function call, and t.output outside a JSON path is still rejected. JsonPathKeyword also left the last step or cast of a path without an AST node, for example the cast in v:select::INT or v:from::INT + 1. It linked the previous element before building each step, so the one built last was never linked. ASTNodeCoverageTest did not notice, as it only walks the SQL resources. Each step and cast is now linked where it is built, and the keyword-path cases are added to testStatementsTheCorpusLacks. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Sonnet 5.5 <noreply@anthropic.com> Co-authored-by: manticore-projects <andreas@manticore-projects.com>
1 parent cb2bde6 commit ec693a4

4 files changed

Lines changed: 264 additions & 10 deletions

File tree

‎src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt‎

Lines changed: 147 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1127,6 +1127,82 @@ public class CCJSqlParser extends AbstractJSqlParser<CCJSqlParser> {
11271127
return left;
11281128
}
11291129

1130+
/**
1131+
* Whether the token after the ":" of a JSON path is a word that is no identifier token, i.e. a
1132+
* keyword, which names the path element all the same. A word followed by "(" stays a function.
1133+
*/
1134+
private boolean isKeywordJsonPathName() {
1135+
Token colon = getToken(0);
1136+
return colon != null && ":".equals(colon.image) && isKeywordWord(1);
1137+
}
1138+
1139+
/**
1140+
* Whether the token at the given position is a word that is no identifier token, i.e. a keyword,
1141+
* and not the name of a function call.
1142+
*/
1143+
private boolean isKeywordWord(int position) {
1144+
Token word = getToken(position);
1145+
if (word.image == null || word.kind == S_IDENTIFIER || word.kind == S_QUOTED_IDENTIFIER
1146+
|| "(".equals(getToken(position + 1).image)) {
1147+
return false;
1148+
}
1149+
String image = word.image;
1150+
if (image.isEmpty() || (!Character.isLetter(image.charAt(0)) && image.charAt(0) != '_')) {
1151+
return false;
1152+
}
1153+
for (int i = 1; i < image.length(); i++) {
1154+
if (!Character.isLetterOrDigit(image.charAt(i)) && image.charAt(i) != '_') {
1155+
return false;
1156+
}
1157+
}
1158+
return true;
1159+
}
1160+
1161+
/**
1162+
* Whether the path after the ":" of a JSON path, which starts with an identifier, goes on with a
1163+
* keyword after a ".", e.g. v:a.output or v:a[0].b.from. The expression parser would read such a
1164+
* path as a qualified column name, which does not take a keyword as a part.
1165+
*/
1166+
private boolean isJsonPathWithKeywordAfterDot() {
1167+
Token colon = getToken(0);
1168+
if (colon == null || !":".equals(colon.image)) {
1169+
return false;
1170+
}
1171+
Token first = getToken(1);
1172+
if (first.kind != S_IDENTIFIER && first.kind != S_QUOTED_IDENTIFIER) {
1173+
return false;
1174+
}
1175+
int position = 2;
1176+
while (true) {
1177+
Token token = getToken(position);
1178+
if (".".equals(token.image)) {
1179+
if (isKeywordWord(position + 1)) {
1180+
return true;
1181+
}
1182+
Token name = getToken(position + 1);
1183+
if (name.kind != S_IDENTIFIER && name.kind != S_QUOTED_IDENTIFIER) {
1184+
return false;
1185+
}
1186+
position += 2;
1187+
} else if ("[".equals(token.image)) {
1188+
int depth = 0;
1189+
do {
1190+
token = getToken(position);
1191+
if ("[".equals(token.image)) {
1192+
depth++;
1193+
} else if ("]".equals(token.image)) {
1194+
depth--;
1195+
} else if (token.kind == EOF) {
1196+
return false;
1197+
}
1198+
position++;
1199+
} while (depth > 0);
1200+
} else {
1201+
return false;
1202+
}
1203+
}
1204+
}
1205+
11301206
private boolean isPostgreSqlJsonAccess(Token token) {
11311207
return ("->".equals(token.image) || "->>".equals(token.image)
11321208
|| "#>".equals(token.image) || "#>>".equals(token.image))
@@ -16549,11 +16625,7 @@ Expression PrimaryExpression() #PrimaryExpression:
1654916625
token="#>>"
1655016626
)
1655116627

16552-
(
16553-
LOOKAHEAD({getAsBoolean(Feature.allowComplexParsing) && !interrupted }) expression=Expression()
16554-
|
16555-
expression=SimpleExpression()
16556-
)
16628+
expression=JsonPathElement()
1655716629
{
1655816630
if (jsonIdents == null) {
1655916631
jsonIdents = new ArrayList<Map.Entry<Expression, String>>();
@@ -16938,6 +17010,75 @@ StructType StructType() #StruckType:
1693817010
}
1693917011
}
1694017012

17013+
/**
17014+
* The element after a ":" in a JSON path (Snowflake, Databricks). A path name is mandatory there,
17015+
* so any word is a name, reserved or not: v:from::INT, v:select, v:output.x[0].
17016+
*/
17017+
Expression JsonPathElement() #void:
17018+
{ Expression element; }
17019+
{
17020+
(
17021+
LOOKAHEAD({ isKeywordJsonPathName() || isJsonPathWithKeywordAfterDot() }) element=JsonPathKeyword()
17022+
|
17023+
LOOKAHEAD({getAsBoolean(Feature.allowComplexParsing) && !interrupted}) element=Expression()
17024+
|
17025+
element=SimpleExpression()
17026+
)
17027+
{ return element; }
17028+
}
17029+
17030+
/**
17031+
* A keyword used as a JSON path name. The token is taken as it is, as no token kind in the grammar
17032+
* stands for "any keyword", then the path name continues like a primary expression: further
17033+
* navigation steps, casts and the arithmetic tier SimpleExpression() also applies.
17034+
*/
17035+
Expression JsonPathKeyword() #void:
17036+
{
17037+
Token first = getToken(1);
17038+
Token keyword;
17039+
Expression element;
17040+
Expression step;
17041+
ColDataType type;
17042+
CastExpression cast;
17043+
}
17044+
{
17045+
{
17046+
keyword = getNextToken();
17047+
element = linkToLastToken(new Column(keyword.image));
17048+
}
17049+
( LOOKAHEAD(2) step=JsonPathStep(element) { element = linked(step, first); } )*
17050+
(
17051+
LOOKAHEAD(2) "::" type=ColDataType() {
17052+
cast = new CastExpression();
17053+
cast.setUseCastKeyword(false);
17054+
cast.setLeftExpression(element);
17055+
cast.setColDataType(type);
17056+
element = linked(cast, first);
17057+
}
17058+
)*
17059+
{
17060+
int arithFloor = Dialect.MYSQL.name().equals(getAsString(Feature.dialect)) ? 2 : 4;
17061+
return prattArithRest(element, first, arithFloor);
17062+
}
17063+
}
17064+
17065+
/**
17066+
* A navigation step within a JSON path, where a keyword after a "." names a path element as well:
17067+
* v:a.output, v:a[0].from.
17068+
*/
17069+
Expression JsonPathStep(Expression base) #void:
17070+
{ Expression step; }
17071+
{
17072+
(
17073+
LOOKAHEAD({ ".".equals(getToken(1).image) && isKeywordWord(2) }) "." {
17074+
step = new RowGetExpression(base, getNextToken().image);
17075+
}
17076+
|
17077+
step=NavigationStep(base)
17078+
)
17079+
{ return step; }
17080+
}
17081+
1694117082
JsonExpression JsonExpression(Expression expr, List<Map.Entry<Expression, String>> idents) : {
1694217083
JsonExpression result = new JsonExpression(expr, idents);
1694317084
Expression expression;
@@ -16981,11 +17122,7 @@ JsonExpression JsonExpression(Expression expr, List<Map.Entry<Expression, String
1698117122
token="#>>"
1698217123
)
1698317124

16984-
(
16985-
LOOKAHEAD({getAsBoolean(Feature.allowComplexParsing) && !interrupted}) expression=Expression()
16986-
|
16987-
expression=SimpleExpression()
16988-
)
17125+
expression=JsonPathElement()
1698917126
{
1699017127
result.addIdent( expression, token.image );
1699117128
}

‎src/test/java/net/sf/jsqlparser/expression/JsonExpressionTest.java‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
package net.sf.jsqlparser.expression;
1111

1212
import net.sf.jsqlparser.JSQLParserException;
13+
import net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect;
1314
import net.sf.jsqlparser.parser.CCJSqlParserUtil;
1415
import net.sf.jsqlparser.statement.select.PlainSelect;
1516
import org.junit.jupiter.api.Assertions;
@@ -55,6 +56,99 @@ void testHashArrowOperatorsRoundTrip() throws JSQLParserException {
5556
}
5657
}
5758

59+
@ParameterizedTest
60+
@ValueSource(strings = {"select", "from", "where", "join", "union", "and", "or", "not", "case",
61+
"when", "else", "as", "is", "like", "having", "distinct", "inner", "into", "with",
62+
"outer", "cross", "using", "output", "null", "order", "group"})
63+
void testSnowflakeKeywordAsPathName(String keyword) throws JSQLParserException {
64+
String sqlStr = "SELECT v:" + keyword + "::INT, v:" + keyword + ".x[0] FROM vartab";
65+
PlainSelect st = (PlainSelect) assertSqlCanBeParsedAndDeparsed(sqlStr, true);
66+
Assertions.assertInstanceOf(JsonExpression.class, st.getSelectItem(0).getExpression());
67+
Assertions.assertInstanceOf(JsonExpression.class, st.getSelectItem(1).getExpression());
68+
}
69+
70+
@Test
71+
void testSnowflakeKeywordPathNameBeforeClause() throws JSQLParserException {
72+
assertSqlCanBeParsedAndDeparsed(
73+
"SELECT v:from FROM t WHERE v:where::INT = 1 ORDER BY v:order",
74+
true);
75+
}
76+
77+
@Test
78+
void testSnowflakeKeywordPathNameInInsertSelect() throws JSQLParserException {
79+
assertSqlCanBeParsedAndDeparsed("INSERT INTO \"MY_DB\".\"MY_SCHEMA\".\"T\" (\"c\")\n"
80+
+ "SELECT \"F\".\"DATA_VALUE\":window::VARCHAR FROM \"MY_DB\".\"MY_SCHEMA\".\"RAW_TRACES\" AS \"F\"",
81+
true);
82+
}
83+
84+
@Test
85+
void testSnowflakeKeywordPathNameOnLateralFlatten() throws JSQLParserException {
86+
assertSqlCanBeParsedAndDeparsed("INSERT INTO \"MY_DB\".\"MY_SCHEMA\".\"OBS\" (\"o\")\n"
87+
+ "SELECT \"obs\".VALUE:output::VARIANT\n"
88+
+ "FROM \"MY_DB\".\"MY_SCHEMA\".\"RAW_TRACES\" AS \"F\",\n"
89+
+ " lateral flatten(input=>\"F\".\"DATA_VALUE\":observations, outer => true, mode => 'ARRAY') AS \"obs\"",
90+
true);
91+
}
92+
93+
@ParameterizedTest
94+
@ValueSource(strings = {"SELECT v:from::INT, v:output.x[0] FROM t",
95+
"SELECT v:select + 1, v:where * 2 - v:from FROM t",
96+
"SELECT v:window::INT + 1 FROM t WHERE v:order::INT > 2"})
97+
void testKeywordPathNameDatabricksAndMysqlArithmetic(String sql) throws JSQLParserException {
98+
assertSqlCanBeParsedAndDeparsed(sql, true,
99+
parser -> parser.withDialect(Dialect.DATABRICKS));
100+
assertSqlCanBeParsedAndDeparsed(sql, true, parser -> parser.withDialect(Dialect.SNOWFLAKE));
101+
assertSqlCanBeParsedAndDeparsed(sql, true, parser -> parser.withDialect(Dialect.MYSQL));
102+
}
103+
104+
@ParameterizedTest
105+
@ValueSource(strings = {
106+
"SELECT coalesce(v:from, v:select::INT) FROM t",
107+
"SELECT CASE WHEN v:when::INT = 1 THEN v:else ELSE v:end END FROM t",
108+
"SELECT * FROM (SELECT v:from AS a FROM t) x WHERE x.a IN (SELECT v:where FROM u)",
109+
"SELECT * FROM t WHERE EXISTS (SELECT 1 FROM u WHERE u.v:order::INT = t.id)",
110+
"SELECT f(v:window, (SELECT w:output FROM u)) FROM t"})
111+
void testKeywordPathNameInsideNestedConstructs(String sql) throws JSQLParserException {
112+
assertSqlCanBeParsedAndDeparsed(sql, true, parser -> parser.withDialect(Dialect.SNOWFLAKE));
113+
}
114+
115+
@ParameterizedTest
116+
@ValueSource(strings = {"select", "from", "where", "join", "union", "and", "or", "not", "case",
117+
"when", "else", "as", "is", "like", "having", "distinct", "inner", "into", "with",
118+
"outer", "cross", "using", "output", "null", "true", "false", "order", "group"})
119+
void testSnowflakeKeywordAfterDotInPath(String keyword) throws JSQLParserException {
120+
String sqlStr = "SELECT v:a." + keyword + ", v:a.b." + keyword + "::INT, v:a[0]." + keyword
121+
+ ".x[1] FROM vartab";
122+
PlainSelect st = (PlainSelect) assertSqlCanBeParsedAndDeparsed(sqlStr, true);
123+
for (int i = 0; i < 3; i++) {
124+
Assertions.assertInstanceOf(JsonExpression.class, st.getSelectItem(i).getExpression());
125+
}
126+
}
127+
128+
@ParameterizedTest
129+
@ValueSource(strings = {
130+
"SELECT v:a.output + 1, \"F\".\"X\":\"a\".from::VARIANT AS y FROM t F",
131+
"SELECT * FROM t WHERE v:payload.output::INT = 1 ORDER BY v:payload.from",
132+
"SELECT v:a.output FROM t, lateral flatten(input => t.v:a.input.where) f"})
133+
void testKeywordAfterDotInPathInContext(String sql) throws JSQLParserException {
134+
assertSqlCanBeParsedAndDeparsed(sql, true, parser -> parser.withDialect(Dialect.SNOWFLAKE));
135+
}
136+
137+
@Test
138+
void testKeywordAfterDotOutsideJsonPathIsStillRejected() {
139+
// a reserved word is not a part of a qualified name, only of a JSON path
140+
Assertions.assertThrows(JSQLParserException.class,
141+
() -> CCJSqlParserUtil.parse("SELECT t.output FROM t",
142+
parser -> parser.withDialect(Dialect.SNOWFLAKE)));
143+
}
144+
145+
@Test
146+
void testKeywordFollowedByParenthesisStaysFunction() throws JSQLParserException {
147+
// only parsing is asserted: a function behind ":" deparses with "."
148+
Assertions.assertNotNull(CCJSqlParserUtil.parse("SELECT v:left(x) FROM t",
149+
parser -> parser.withDialect(Dialect.SNOWFLAKE)));
150+
}
151+
58152
@Test
59153
void testSnowflakeGetOperator() throws JSQLParserException {
60154
// https://docs.snowflake.com/en/user-guide/querying-semistructured

‎src/test/java/net/sf/jsqlparser/parser/ASTNodeCoverageTest.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,12 @@ public void testStatementsTheCorpusLacks() throws JSQLParserException {
211211
"CLOSE ALL"}) {
212212
new Walk(sql, sql, problems).check(CCJSqlParserUtil.parseStatements(sql));
213213
}
214+
for (String sql : new String[] {"SELECT v:select FROM t", "SELECT v:select::INT FROM t",
215+
"SELECT v:select.x[0], v:from::INT + 1 FROM t",
216+
"SELECT v:a.output, v:a.b.from::INT, v:a[0].from.x[1] + 1 FROM t"}) {
217+
new Walk(sql, sql, problems).check(CCJSqlParserUtil.parseStatements(sql,
218+
parser -> parser.withDialect(AbstractJSqlParser.Dialect.SNOWFLAKE)));
219+
}
214220
String describe = "DESC TABLE t; DO 'x'; DESCRIBE t 'a%'; EXPLAIN t a; DESC t 0x61";
215221
new Walk(describe, describe, problems).check(CCJSqlParserUtil.parseStatements(describe,
216222
parser -> parser.withDialect(AbstractJSqlParser.Dialect.MYSQL)));

‎src/test/java/net/sf/jsqlparser/statement/create/CreateTableTest.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1388,4 +1388,21 @@ void testPostgreSqlDeclarativePartitionChildrenIssue2522() throws JSQLParserExce
13881388
defaultSql, true, parser -> parser.withDialect(Dialect.POSTGRESQL));
13891389
assertEquals(PartitionBound.Type.DEFAULT, defaultPartition.getPartitionBound().getType());
13901390
}
1391+
1392+
@Test
1393+
void testCreateOrReplaceTableCopyGrantsAsSelectFromFlattenedVariant()
1394+
throws JSQLParserException {
1395+
String sql =
1396+
"CREATE OR REPLACE TABLE \"MY_DB\".\"MY_SCHEMA\".\"STAGING_TRACES\" COPY GRANTS\n"
1397+
+ "AS (SELECT\n"
1398+
+ " *\n"
1399+
+ "FROM (SELECT\n"
1400+
+ " \"FLATTEN_VARIANT_INPUT\".\"DATA_VALUE\":id::VARCHAR AS \"id\",\n"
1401+
+ " \"FLATTEN_VARIANT_INPUT\".\"DATA_VALUE\":input::VARIANT AS \"input\",\n"
1402+
+ " \"FLATTEN_VARIANT_INPUT\".\"DATA_VALUE\":output::VARIANT AS \"output\"\n"
1403+
+ "FROM (SELECT\n"
1404+
+ " \"DATA_VALUE\"\n"
1405+
+ " FROM \"MY_DB\".\"MY_SCHEMA\".\"RAW_TRACES\") AS \"FLATTEN_VARIANT_INPUT\"))";
1406+
assertSqlCanBeParsedAndDeparsed(sql, true, parser -> parser.withDialect(Dialect.SNOWFLAKE));
1407+
}
13911408
}

0 commit comments

Comments
 (0)