fix: lex MySQL backslash escapes consistently in string literals (#2790)

With backslash escapes enabled, as in the MySQL and MariaDB presets,
the lexer recognized only a few escapes and no doubled quote in the
same literal. 'it\'s ''x''' silently split into a string and an alias,
'a\0b\'c', '\Z\'' and 'a''b\'c' failed, and the double-quoted "it\"s"
ended at the escaped quote.

Let both quote styles read any backslash escape and doubled quotes,
then trim the token to the closing quote of the rules in effect. A
backslash only escapes with backslash escapes enabled or in PostgreSQL
E'' strings, so standard strings, LIKE ... ESCAPE '\' and quoted
identifiers such as "col\" are unchanged.

Signed-off-by: minleejae <mmj9808@gmail.com>

Author: minleejae

src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt

@@ -2462,26 +2462,33 @@ TOKEN_MGR_DECLS : {
         }
     }
 
-    // Finds first occurrence of "\\'"
-    public static int indexOfSequence(String s, String target) {
-        int len = s.length();
-        for (int i = 0; i < len - 1; i++) {
-            if (s.charAt(i) == '\\' && s.charAt(i + 1) == '\'') {
-                return i;
+    // Returns the length up to the closing quote of a quoted literal starting at open, or -1 when
+    // the image is unterminated. A doubled quote is literal; a backslash escapes the next character
+    // only if requested.
+    public static int closingQuote(String s, int open, char quote, boolean backslashEscapes) {
+        int i = open + 1;
+        while (i < s.length()) {
+            char c = s.charAt(i);
+            if (backslashEscapes && c == '\\' && i + 1 < s.length()) {
+                i += 2;
+            } else if (c == quote && i + 1 < s.length() && s.charAt(i + 1) == quote) {
+                i += 2;
+            } else if (c == quote) {
+                return i + 1;
+            } else {
+                i++;
             }
         }
         return -1;
     }
 
-    // Finds last occurrence of "\\''"
-    public static int lastIndexOfSequence(String s, String target) {
-        int len = s.length();
-        for (int i = len - 3; i >= 0; i--) {
-            if (s.charAt(i) == '\\' && s.charAt(i + 1) == '\'' && s.charAt(i + 2) == '\'') {
-                return i;
-            }
+    // Like closingQuote for a single-quoted literal after its optional prefix. Oracle q'..'
+    // literals have their own delimiters and are left unchanged.
+    public static int closingSingleQuote(String s, boolean backslashEscapes) {
+        if (s.length() > 1 && (s.charAt(0) == 'q' || s.charAt(0) == 'Q') && s.charAt(1) == '\'') {
+            return -1;
         }
-        return -1;
+        return closingQuote(s, s.indexOf('\''), '\'', backslashEscapes);
     }
 
     private RowPatternTokenContext rowPatternContext;
@@ -3442,12 +3449,11 @@ TOKEN:
 // CJK Unified Ideographs block according to https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
 | <#CJK: ["\uAC00"-"\uD7A3", "\u4E00"-"\u9FFF"]>
 
-| < #SPECIAL_ESC: "\\'" > /* Allowing this will break LIKE ... ESCAPE ...  */
-| < #ESC: "\\" ["n","t","b","r","f","\\","\""] >
 | < S_CHAR_LITERAL: (
                         (["U","E","N","R","B"]|"RB"|"_utf8")?
                         (
-                            ("'" ( <ESC> | <SPECIAL_ESC> | ~["'", "\\"] )* "'") | ("'" ("''" | ~["'"])* "'")
+                            // Backslash escapes and doubled quotes; trimmed below to the rules in effect
+                            ("'" ( "\\" ~[] | "''" | ~["'", "\\"] )* "'") | ("'" ("''" | ~["'"])* "'")
                             // Alternative Oracle Escape Modes
                             | ("q'{" (~[])* "}'")
                             | ("q'(" (~[])* ")'")
@@ -3457,36 +3463,36 @@ TOKEN:
                         )
                      ) >
     {
-        // <S_CHAR_LITERAL> contains the <SPECIAL_ESC> token and always the longest match is returned
-        // So when Backslash is explicitly not allowed as an Escape Character and a <S_CHAR_LITERAL> is found
-        // which contains the <SPECIAL_ESC>, then we will need to
-        // 1) break the <S_CHAR_LITERAL> at <SPECIAL_ESC> close it with a "'"
-        // 2) continue tokenizing after that <SPECIAL_ESC> with a new <S_CHAR_LITERAL> or any other Token
-        boolean allowEscape = configuration.getAsBoolean(Feature.allowBackslashEscapeCharacter)
+        // The longest match may run past the closing quote of the active rules. A backslash
+        // escapes the next character only with backslash escapes or in a PostgreSQL E'' string;
+        // otherwise the literal ends at the first quote that is not doubled, so LIKE ... ESCAPE '\\'
+        // still works.
+        boolean backslashEscapes = configuration.getAsBoolean(Feature.allowBackslashEscapeCharacter)
                 || (AbstractJSqlParser.Dialect.POSTGRESQL.name().equals(configuration.getValue(Feature.dialect))
                     && matchedToken.image.regionMatches(true, 0, "E'", 0, 2));
-                String img = matchedToken.image;
-                int pos;
-                if (!allowEscape) {
-                    pos = indexOfSequence(img, "\\'");
-                    if (pos > 0) {
-                        matchedToken.image = image.substring( 0, image.indexOf("\\'") + 1 ) + "'";
-                        // `charLiteralIndex` defined in TokenManagerDeclaration above
-                        matchedToken.kind = charLiteralIndex;
-                        input_stream.backup(image.length() - matchedToken.image.length());
-                    }
-                } else {
-                    pos = lastIndexOfSequence(img, "\\''");
-                    if (pos > 0) {
-                        matchedToken.image = image.substring( 0, image.lastIndexOf("\\'") + 3);
-                        // `charLiteralIndex` defined in TokenManagerDeclaration above
-                        matchedToken.kind = charLiteralIndex;
-                        input_stream.backup(image.length() - matchedToken.image.length() );
-                    }
-                }
-    }
-| < S_QUOTED_IDENTIFIER: "\"" ( "\"\"" | ~["\n","\r","\""])* "\"" | ("`" ("``" | ~["\n","\r","`"])* "`") | ( "[" (~["\n","\r","]"])* "]" ) >
-    {
+        int end = closingSingleQuote(matchedToken.image, backslashEscapes);
+        if (end > 0 && end < matchedToken.image.length()) {
+            matchedToken.image = image.substring(0, end);
+            input_stream.backup(image.length() - end);
+        }
+    }
+| < S_QUOTED_IDENTIFIER: "\"" ( "\"\"" | ~["\n","\r","\""])* "\""
+        // MySQL double-quoted string with backslash escapes; trimmed below when not enabled
+        | "\"" ( "\"\"" | "\\" ~["\n","\r"] | ~["\n","\r","\"","\\"])* "\""
+        | ("`" ("``" | ~["\n","\r","`"])* "`") | ( "[" (~["\n","\r","]"])* "]" ) >
+    {
+        if ( matchedToken.image.charAt(0) == '"' ) {
+            // Both alternatives above compete for the longest match. Keep the closing quote of
+            // the active quoting rules: a backslash escapes the next character only in
+            // double-quoted strings with backslash escapes, as in MySQL.
+            int end = closingQuote(matchedToken.image, 0, '"',
+                    configuration.getAsBoolean(Feature.allowDoubleQuotedStrings)
+                    && configuration.getAsBoolean(Feature.allowBackslashEscapeCharacter));
+            if (end > 0 && end < matchedToken.image.length()) {
+                matchedToken.image = image.substring(0, end);
+                input_stream.backup(image.length() - end);
+            }
+        }
         if ( !configuration.getAsBoolean(Feature.allowSquareBracketQuotation)
             && matchedToken.image.charAt(0) == '[' ) {
 

src/site/sphinx/usage.rst

@@ -1568,7 +1568,7 @@ The individual features
     * - ``withBackslashEscapeCharacter``
       - ``\\..`` escaping inside string literals, in addition to the standard ``'..`` doubling
     * - ``withDoubleQuotedStrings``
-      - ``".."`` reads as a string literal instead of a quoted identifier (BigQuery, Spark/Databricks, MySQL default ``sql_mode``)
+      - ``".."`` reads as a string literal instead of a quoted identifier (BigQuery, Spark/Databricks, MySQL default ``sql_mode``); combined with ``withBackslashEscapeCharacter``, ``\\"`` does not end the literal, as in MySQL ``"it\\"s"``
     * - ``withHashLineComments``
       - ``#`` starts a line comment
     * - ``withAdjacentStringLiterals``

src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java

@@ -748,6 +748,75 @@ public void testDoubleQuotedStringsPreset() throws Exception {
         }
     }
 
+    @Test
+    public void testDoubleQuotedStringsWithBackslashEscapes() throws Exception {
+        // MySQL reads "it\"s" as it"s; the escaped quote must not end the literal
+        for (AbstractJSqlParser.Dialect dialect : new AbstractJSqlParser.Dialect[] {
+                AbstractJSqlParser.Dialect.MYSQL, AbstractJSqlParser.Dialect.MARIADB,
+                AbstractJSqlParser.Dialect.BIGQUERY, AbstractJSqlParser.Dialect.DATABRICKS}) {
+            PlainSelect select = (PlainSelect) ((Select) CCJSqlParserUtil.parse(
+                    "SELECT \"it\\\"s\", \"a\\\\\"", p -> p.withDialect(dialect)))
+                    .getSelectBody();
+            assertEquals("it\\\"s",
+                    ((StringValue) select.getSelectItems().get(0).getExpression()).getValue());
+            assertEquals("a\\\\",
+                    ((StringValue) select.getSelectItems().get(1).getExpression()).getValue());
+            assertEquals("SELECT \"it\\\"s\", \"a\\\\\"", select.toString());
+        }
+        // a doubled quote still continues a MySQL literal
+        assertEquals("SELECT \"b\"\"c\\\"d\"", CCJSqlParserUtil.parse("SELECT \"b\"\"c\\\"d\"",
+                p -> p.withDialect(AbstractJSqlParser.Dialect.MYSQL)).toString());
+        String alter = "ALTER TABLE t ADD COLUMN s VARCHAR (10) DEFAULT \"it\\\"s\"";
+        assertEquals(alter, CCJSqlParserUtil
+                .parse(alter, p -> p.withDialect(AbstractJSqlParser.Dialect.MYSQL)).toString());
+        // the same escape needs both features
+        assertEquals("SELECT \"it\\\"s\"", CCJSqlParserUtil.parse("SELECT \"it\\\"s\"",
+                p -> p.withDoubleQuotedStrings(true).withBackslashEscapeCharacter(true))
+                .toString());
+        assertThrows(JSQLParserException.class, () -> CCJSqlParserUtil
+                .parse("SELECT \"it\\\"s\"", p -> p.withDoubleQuotedStrings(true)));
+        // MySQL closes "a\"" before b, so the rest is unterminated as on the server
+        assertThrows(JSQLParserException.class, () -> CCJSqlParserUtil.parse(
+                "SELECT \"a\\\"\"b\"", p -> p.withDialect(AbstractJSqlParser.Dialect.MYSQL)));
+        // quoted identifiers keep a trailing backslash
+        for (AbstractJSqlParser.Dialect dialect : new AbstractJSqlParser.Dialect[] {null,
+                AbstractJSqlParser.Dialect.POSTGRESQL, AbstractJSqlParser.Dialect.SNOWFLAKE}) {
+            PlainSelect select = (PlainSelect) ((Select) CCJSqlParserUtil.parse(
+                    "SELECT \"col\\\" FROM \"t\"", p -> {
+                        if (dialect != null) {
+                            p.withDialect(dialect);
+                        }
+                    })).getSelectBody();
+            assertEquals("\"col\\\"",
+                    ((Column) select.getSelectItems().get(0).getExpression()).getColumnName());
+            assertEquals("SELECT \"col\\\" FROM \"t\"", select.toString());
+        }
+    }
+
+    @Test
+    public void testSingleQuotedStringsMixBackslashEscapesAndDoubledQuotes() throws Exception {
+        // each is one MySQL string; the first used to split into a string and an alias
+        for (String literal : new String[] {"'it\\'s ''x'''", "'a\\0b\\'c'", "'\\Z\\''",
+                "'a''b\\'c'", "'x\\%\\'y'", "N'it\\'s'"}) {
+            PlainSelect select = (PlainSelect) ((Select) CCJSqlParserUtil.parse(
+                    "SELECT " + literal + " FROM t",
+                    p -> p.withDialect(AbstractJSqlParser.Dialect.MYSQL))).getSelectBody();
+            assertEquals(1, select.getSelectItems().size(), literal);
+            assertNull(select.getSelectItems().get(0).getAlias(), literal);
+            assertInstanceOf(StringValue.class, select.getSelectItems().get(0).getExpression());
+            assertEquals("SELECT " + literal + " FROM t", select.toString());
+        }
+        // PostgreSQL E'' strings escape quotes; standard strings keep a literal backslash
+        assertEquals("SELECT E'it\\'s ''x'''", CCJSqlParserUtil.parse("SELECT E'it\\'s ''x'''",
+                p -> p.withDialect(AbstractJSqlParser.Dialect.POSTGRESQL)).toString());
+        assertEquals("SELECT 'a\\', 'b'", CCJSqlParserUtil.parse("SELECT 'a\\', 'b'",
+                p -> p.withDialect(AbstractJSqlParser.Dialect.POSTGRESQL)).toString());
+        // without backslash escapes the literal still ends at the escaped-looking quote
+        assertEquals("SELECT 'a\\' AS x", CCJSqlParserUtil.parse("SELECT 'a\\' AS x").toString());
+        assertEquals("SELECT * FROM t WHERE a LIKE 'x\\_%' ESCAPE '\\'", CCJSqlParserUtil
+                .parse("SELECT * FROM t WHERE a LIKE 'x\\_%' ESCAPE '\\'").toString());
+    }
+
     @Test
     public void testAdjacentStringLiteralsNewline() throws Exception {
         // default (off): the MySQL / SQL Server alias reading, unchanged