Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package com.google.cloud.spanner.connection;

import static com.google.cloud.spanner.connection.SimpleParser.isValidIdentifierChar;
import static com.google.cloud.spanner.connection.SimpleParser.isValidIdentifierFirstChar;
import static com.google.cloud.spanner.connection.StatementHintParser.convertHintsToOptions;

import com.google.api.core.InternalApi;
Expand Down Expand Up @@ -937,19 +938,29 @@ public boolean checkReturningClause(String sql) {
/** Returns the query parameter prefix that should be used for this dialect. */
abstract String getQueryParameterPrefix();

/** Reads a dollar-quoted string literal from position index in the given sql string. */
/**
* Parses a dollar-quote tag starting at {@code index} (the character immediately following an
* opening or closing {@code '$'}) up to the matching {@code '$'}.
*
* <p>Returns {@code ""} for an empty {@code $$} delimiter, the tag name when the tag starts with
* {@link SimpleParser#isValidIdentifierFirstChar(char)} (which excludes ASCII and non-letter
* Unicode digits such as {@code '١'}) and continues with {@link
* SimpleParser#isValidIdentifierChar(char)}, or {@code null} if no valid dollar-quote tag starts
* at {@code index}.
*/
String parseDollarQuotedString(String sql, int index) {
// Look ahead to the next dollar sign (if any). Everything in between is the quote tag.
StringBuilder tag = new StringBuilder();
// Look ahead to the next dollar sign (if any). An empty tag ($$) is valid, while any non-empty
// tag must start with a valid identifier-start character so positional parameters ($1) are not
// mistaken for dollar-quote tags.
int startIndex = index;
while (index < sql.length()) {
char c = sql.charAt(index);
if (c == DOLLAR) {
return tag.toString();
return index == startIndex ? "" : sql.substring(startIndex, index);
}
if (!isValidIdentifierChar(c)) {
if (index == startIndex ? !isValidIdentifierFirstChar(c) : !isValidIdentifierChar(c)) {
break;
}
tag.append(c);
index++;
}
return null;
Expand All @@ -970,7 +981,11 @@ int skip(String sql, int currentIndex, @Nullable StringBuilder result) {
|| (supportsBacktickQuote() && currentChar == BACKTICK_QUOTE)) {
appendIfNotNull(result, currentChar);
return skipQuoted(sql, currentIndex, currentChar, result);
} else if (supportsDollarQuotedStrings() && currentChar == DOLLAR) {
} else if (supportsDollarQuotedStrings()
&& currentChar == DOLLAR
// In PostgreSQL, '$' is a valid identifier continuation character (e.g. col$tag$a, col$$)
// and cannot open a dollar-quoted literal immediately after an identifier character.
&& (currentIndex == 0 || !isValidIdentifierChar(sql.charAt(currentIndex - 1)))) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This 1-position lookback is introducing some regressions for dollar-quoted literals that follow directly after a numeric literal or directly after another dollar-quoted literal. These tests were missing in our test coverage, which would have caught this regression:

  @Test
  public void testPostgreSQLDollarQuotedRegressions() {
    assumeTrue(dialect == Dialect.POSTGRESQL);

    // Regression 1: Adjacent dollar-quoted literals (previous closing '$' makes
    // isValidIdentifierChar('$') == true).
    assertEquals(
        "SELECT $$a$$$$--comment\n$$",
        parser.removeCommentsAndTrim("SELECT $$a$$$$--comment\n$$"));
    // Currently throws SpannerException(INVALID_ARGUMENT: unclosed literal) on the single quote.
    assertEquals("SELECT $$a$$$$it's$$", parser.removeCommentsAndTrim("SELECT $$a$$$$it's$$"));

    ParametersInfo adjacentParams =
        parser.convertPositionalParametersToNamedParameters('?', "SELECT $$a$$$$?$$");
    assertEquals(0, adjacentParams.numberOfParameters);
    assertEquals("SELECT $$a$$$$?$$", adjacentParams.sqlWithNamedParameters);

    assertFalse(
        parser
            .parse(Statement.of("UPDATE t SET a = $$a$$$$ RETURNING $$"))
            .hasReturningClause());

    PostgreSQLStatementParser pgParser = (PostgreSQLStatementParser) parser;
    assertEquals(ImmutableSet.of(), pgParser.getQueryParameters("SELECT $$a$$$$1$$"));

    // Regression 2: Dollar-quoted literal immediately following a numeric literal or parameter
    // (preceding digit makes isValidIdentifierChar('1') == true).
    assertEquals(
        "SELECT 1$$--comment\n$$", parser.removeCommentsAndTrim("SELECT 1$$--comment\n$$"));
    // Currently throws SpannerException(INVALID_ARGUMENT: unclosed literal).
    assertEquals("SELECT 1$$it's$$", parser.removeCommentsAndTrim("SELECT 1$$it's$$"));
    assertEquals(ImmutableSet.of("$1"), pgParser.getQueryParameters("SELECT $1$$2$$"));
  }

String dollarTag = parseDollarQuotedString(sql, currentIndex + 1);
if (dollarTag != null) {
appendIfNotNull(result, currentChar, dollarTag, currentChar);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,139 @@ public void testPostgresSQLDialectDollarQuoted() {
assertThat(
parser.removeCommentsAndTrim("SELECT FOO, $BAR -- This is a comment\nFROM SOME_TABLE"))
.isEqualTo("SELECT FOO, $BAR \nFROM SOME_TABLE");

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There are some additional (existing) shortcomings in the current dollar-quoted literal parsing. These tests show those cases:

  @Test
  public void testPostgreSQLDollarQuotedPreExistingGaps() {
    assumeTrue(dialect == Dialect.POSTGRESQL);
    PostgreSQLStatementParser pgParser = (PostgreSQLStatementParser) parser;

    // Dollar-quoted literal immediately after a positional parameter:
    // Failed on main as tag "1"; still fails now because '1' precedes '$'.
    assertEquals(
        "SELECT $1$tag$--comment\n$tag$",
        parser.removeCommentsAndTrim("SELECT $1$tag$--comment\n$tag$"));

    // Skip still steps char-by-char through unquoted identifiers, so getQueryParameters()
    // falsely extracts $1 from identifier col$1.
    assertEquals(ImmutableSet.of("$2"), pgParser.getQueryParameters("SELECT col$1, $2 FROM t"));

    // Skip still steps char-by-char through col$returning, and checkCharPrecedingReturning()
    // accepts '$', so col$returning falsely triggers hasReturningClause().
    assertFalse(
        parser.parse(Statement.of("UPDATE t SET a = col$returning *")).hasReturningClause());

    // checkReturningClauseInternal() lowercases the whole SQL string before calling skip(),
    // violating PostgreSQL case-sensitivity for dollar-quote tags ($Tag$ != $tag$).
    assertFalse(
        parser
            .parse(Statement.of("UPDATE t SET x = $Tag$ $tag$ RETURNING * $Tag$"))
            .hasReturningClause());
  }


// Mid-identifier $tag$ is part of an unquoted identifier, not a dollar-quote opener.
ParametersInfo midIdentifierParams =
parser.convertPositionalParametersToNamedParameters(
'?', "SELECT col$tag$a, ?, col$tag$b FROM t");
assertEquals(1, midIdentifierParams.numberOfParameters);
assertEquals(
"SELECT col$tag$a, $1, col$tag$b FROM t", midIdentifierParams.sqlWithNamedParameters);
ParametersInfo singleMidIdentifierParam =
parser.convertPositionalParametersToNamedParameters('?', "SELECT col$tag$a WHERE x = ?");
assertEquals(1, singleMidIdentifierParam.numberOfParameters);
assertEquals("SELECT col$tag$a WHERE x = $1", singleMidIdentifierParam.sqlWithNamedParameters);

// Mid-identifier $$ and chained/digit-preceded $ tags strip comments and detect RETURNING.
assertThat(parser.removeCommentsAndTrim("SELECT col$$ -- comment\nFROM t"))

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: this pull request uses a mixture of Google Truth assertThat(..) style assertions, and JUnit style assertEquals(..) assertions. It should stick to one style, and the preferred style is normally JUnit.

.isEqualTo("SELECT col$$ \nFROM t");
assertThat(parser.removeCommentsAndTrim("SELECT col$$tag$a -- comment\nFROM t"))
.isEqualTo("SELECT col$$tag$a \nFROM t");
assertThat(parser.removeCommentsAndTrim("SELECT col1$tag$a -- comment\nFROM t"))
.isEqualTo("SELECT col1$tag$a \nFROM t");
assertThat(parser.removeCommentsAndTrim("SELECT $$foo$$col$tag$a -- comment\nFROM t"))
.isEqualTo("SELECT $$foo$$col$tag$a \nFROM t");
assertTrue(
parser.parse(Statement.of("UPDATE t SET a = col$$ RETURNING col$$")).hasReturningClause());
assertTrue(
parser
.parse(Statement.of("UPDATE t SET a = col$tag$a RETURNING col$tag$b"))
.hasReturningClause());
assertTrue(
parser
.parse(Statement.of("UPDATE t SET a = col1$tag$a RETURNING col1$tag$b"))
.hasReturningClause());

// Numeric-leading tags ($1$, $10$, $12$, $1a$, and non-ASCII digit $١$) are rejected.
assertThat(parser.removeCommentsAndTrim("SELECT $1$ -- comment\nFROM t"))
.isEqualTo("SELECT $1$ \nFROM t");
assertThat(parser.removeCommentsAndTrim("SELECT $10$11")).isEqualTo("SELECT $10$11");
assertThat(parser.removeCommentsAndTrim("SELECT $12$ -- comment\nFROM t"))
.isEqualTo("SELECT $12$ \nFROM t");
assertThat(parser.removeCommentsAndTrim("SELECT $1a$ -- comment\nFROM t"))
.isEqualTo("SELECT $1a$ \nFROM t");
assertThat(parser.removeCommentsAndTrim("SELECT $\u0661$ -- comment\nFROM t"))
.isEqualTo("SELECT $\u0661$ \nFROM t");
assertThat(parser.removeCommentsAndTrim("SELECT $1$ /* comment */ $1$"))
.isEqualTo("SELECT $1$ $1$");

for (String invalidNumericTag : new String[] {"$1$", "$10$", "$12$", "$1a$", "$\u0661$"}) {
ParametersInfo numericTagParams =
parser.convertPositionalParametersToNamedParameters(
'?', "SELECT " + invalidNumericTag + " + ? + " + invalidNumericTag);
assertEquals(1, numericTagParams.numberOfParameters);
assertEquals(
"SELECT " + invalidNumericTag + " + $1 + " + invalidNumericTag,
numericTagParams.sqlWithNamedParameters);
assertTrue(
parser
.parse(
Statement.of(
"UPDATE t SET a = " + invalidNumericTag + " RETURNING " + invalidNumericTag))
.hasReturningClause());
}

// Valid tags ($$, $tag$, $_1$, $tag_1$, $tag10$, $ø1$, and $a$price $$a$) preserve interior
// ?, --, /*, and RETURNING tokens while still converting/stripping/detecting outer tokens.
for (String validTag :
new String[] {"$$", "$tag$", "$_1$", "$tag_1$", "$tag10$", "$\u00f81$"}) {
String literalWithTokens =
validTag + " ? -- not a comment /* block */ RETURNING * " + validTag;
ParametersInfo validTagParams =
parser.convertPositionalParametersToNamedParameters(
'?', "SELECT " + literalWithTokens + " WHERE id = ?");
assertEquals(1, validTagParams.numberOfParameters);
assertEquals(
"SELECT " + literalWithTokens + " WHERE id = $1", validTagParams.sqlWithNamedParameters);
assertThat(parser.removeCommentsAndTrim("SELECT " + literalWithTokens + " -- outer comment"))
.isEqualTo("SELECT " + literalWithTokens);
assertFalse(
parser
.parse(Statement.of("UPDATE t SET x = " + literalWithTokens + " WHERE id = 1"))
.hasReturningClause());
}

// Closing delimiter preceded by '$' inside literal content ($a$price $$a$) still closes.
ParametersInfo dollarBeforeCloseParams =
parser.convertPositionalParametersToNamedParameters(
'?', "SELECT $a$price ? $$a$ WHERE id = ?");
assertEquals(1, dollarBeforeCloseParams.numberOfParameters);
assertEquals(
"SELECT $a$price ? $$a$ WHERE id = $1", dollarBeforeCloseParams.sqlWithNamedParameters);
assertThat(parser.removeCommentsAndTrim("SELECT $a$-- price $$a$ -- outer comment"))
.isEqualTo("SELECT $a$-- price $$a$");
assertFalse(
parser
.parse(Statement.of("UPDATE t SET x = $a$ RETURNING price $$a$ WHERE id = 1"))
.hasReturningClause());
assertTrue(
parser
.parse(Statement.of("UPDATE t SET x = $a$price $$a$ WHERE id = 1 RETURNING *"))
.hasReturningClause());

assertUnclosedLiteral(parser, "SELECT $$unclosed");
assertUnclosedLiteral(parser, "SELECT $_1$unclosed");
}

@Test
public void testPostgreSQLSkipDollarQuoted() {
assumeTrue(dialect == Dialect.POSTGRESQL);

// Valid dollar-quoted literals at index 0 and after non-identifier delimiters.
assertEquals("$$foo$$", skipPostgreSQL("$$foo$$ bar", 0));
assertEquals("$tag$foo$tag$", skipPostgreSQL("$tag$foo$tag$ bar", 0));
assertEquals("$_1$foo$_1$", skipPostgreSQL("$_1$foo$_1$ bar", 0));
assertEquals("$\u00f81$foo$\u00f81$", skipPostgreSQL("$\u00f81$foo$\u00f81$ bar", 0));
assertEquals("$a$price $$a$", skipPostgreSQL("$a$price $$a$ bar", 0));
assertEquals("$$foo$$", skipPostgreSQL("SELECT $$foo$$", 7));

// Mid-identifier $, numeric-leading tags, and trailing $ advance exactly 1 character.
assertEquals("$", skipPostgreSQL("col$tag$a$tag$", 3));
assertEquals("$", skipPostgreSQL("col$$foo$$", 3));
assertEquals("$", skipPostgreSQL("col1$tag$a$tag$", 4));
assertEquals("$", skipPostgreSQL("$1$foo$1$", 0));
assertEquals("$", skipPostgreSQL("SELECT $12$foo$12$", 7));
assertEquals("$", skipPostgreSQL("SELECT $\u0661$foo$\u0661$", 7));
assertEquals("$", skipPostgreSQL("col$", 3));
assertEquals("$", skipPostgreSQL("SELECT $", 7));
}

private String skipPostgreSQL(String sql, int currentIndex) {
StringBuilder result = new StringBuilder();
int nextIndex = parser.skip(sql, currentIndex, result);
String skipped = sql.substring(currentIndex, nextIndex);
assertEquals(skipped, result.toString());
return skipped;
}

@Test
Expand Down Expand Up @@ -1538,6 +1671,15 @@ public void testPostgreSQLGetQueryParameters() {
"/* @lock_scanned_ranges = exclusive */ select -- random comment\n"
+ " '$2' from foo /* comment /* with nested comment */ outside of nested comment */"
+ " where bar=$1 and baz=$foo"));
assertEquals(ImmutableSet.of("$1", "$2"), parser.getQueryParameters("SELECT $1$2"));
assertEquals(ImmutableSet.of("$10", "$11"), parser.getQueryParameters("SELECT $10$11"));
assertEquals(ImmutableSet.of("$1", "$2"), parser.getQueryParameters("SELECT $1$, $2"));
assertEquals(
ImmutableSet.of("$1"), parser.getQueryParameters("SELECT col$tag$a, $1, col$tag$b"));
assertEquals(
ImmutableSet.of("$1", "$2"),
parser.getQueryParameters("SELECT col$tag$a, $1, col$tag$b, $2"));
assertEquals(ImmutableSet.of("$1"), parser.getQueryParameters("SELECT $_1$$2$_1$, $1"));
}

@Test
Expand Down
Loading