diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index eb904885a..5028de901 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -2275,26 +2275,33 @@ TOKEN_MGR_DECLS : { } } - // Finds first occurrence of "\\'" - public static int indexOfSequence(String s, String target) { - int len = s.length(); - for (int i = 0; i < len - 1; i++) { - if (s.charAt(i) == '\\' && s.charAt(i + 1) == '\'') { - return i; + // Returns the length up to the closing quote of a quoted literal starting at open, or -1 when + // the image is unterminated. A doubled quote is literal; a backslash escapes the next character + // only if requested. + public static int closingQuote(String s, int open, char quote, boolean backslashEscapes) { + int i = open + 1; + while (i < s.length()) { + char c = s.charAt(i); + if (backslashEscapes && c == '\\' && i + 1 < s.length()) { + i += 2; + } else if (c == quote && i + 1 < s.length() && s.charAt(i + 1) == quote) { + i += 2; + } else if (c == quote) { + return i + 1; + } else { + i++; } } return -1; } - // Finds last occurrence of "\\''" - public static int lastIndexOfSequence(String s, String target) { - int len = s.length(); - for (int i = len - 3; i >= 0; i--) { - if (s.charAt(i) == '\\' && s.charAt(i + 1) == '\'' && s.charAt(i + 2) == '\'') { - return i; - } + // Like closingQuote for a single-quoted literal after its optional prefix. Oracle q'..' + // literals have their own delimiters and are left unchanged. + public static int closingSingleQuote(String s, boolean backslashEscapes) { + if (s.length() > 1 && (s.charAt(0) == 'q' || s.charAt(0) == 'Q') && s.charAt(1) == '\'') { + return -1; } - return -1; + return closingQuote(s, s.indexOf('\''), '\'', backslashEscapes); } private RowPatternTokenContext rowPatternContext; @@ -3222,12 +3229,11 @@ TOKEN: // CJK Unified Ideographs block according to https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block) | <#CJK: ["\uAC00"-"\uD7A3", "\u4E00"-"\u9FFF"]> -| < #SPECIAL_ESC: "\\'" > /* Allowing this will break LIKE ... ESCAPE ... */ -| < #ESC: "\\" ["n","t","b","r","f","\\","\""] > | < S_CHAR_LITERAL: ( (["U","E","N","R","B"]|"RB"|"_utf8")? ( - ("'" ( | | ~["'", "\\"] )* "'") | ("'" ("''" | ~["'"])* "'") + // Backslash escapes and doubled quotes; trimmed below to the rules in effect + ("'" ( "\\" ~[] | "''" | ~["'", "\\"] )* "'") | ("'" ("''" | ~["'"])* "'") // Alternative Oracle Escape Modes | ("q'{" (~[])* "}'") | ("q'(" (~[])* ")'") @@ -3237,36 +3243,36 @@ TOKEN: ) ) > { - // contains the token and always the longest match is returned - // So when Backslash is explicitly not allowed as an Escape Character and a is found - // which contains the , then we will need to - // 1) break the at close it with a "'" - // 2) continue tokenizing after that with a new or any other Token - boolean allowEscape = configuration.getAsBoolean(Feature.allowBackslashEscapeCharacter) + // The longest match may run past the closing quote of the active rules. A backslash + // escapes the next character only with backslash escapes or in a PostgreSQL E'' string; + // otherwise the literal ends at the first quote that is not doubled, so LIKE ... ESCAPE '\\' + // still works. + boolean backslashEscapes = configuration.getAsBoolean(Feature.allowBackslashEscapeCharacter) || (AbstractJSqlParser.Dialect.POSTGRESQL.name().equals(configuration.getValue(Feature.dialect)) && matchedToken.image.regionMatches(true, 0, "E'", 0, 2)); - String img = matchedToken.image; - int pos; - if (!allowEscape) { - pos = indexOfSequence(img, "\\'"); - if (pos > 0) { - matchedToken.image = image.substring( 0, image.indexOf("\\'") + 1 ) + "'"; - // `charLiteralIndex` defined in TokenManagerDeclaration above - matchedToken.kind = charLiteralIndex; - input_stream.backup(image.length() - matchedToken.image.length()); - } - } else { - pos = lastIndexOfSequence(img, "\\''"); - if (pos > 0) { - matchedToken.image = image.substring( 0, image.lastIndexOf("\\'") + 3); - // `charLiteralIndex` defined in TokenManagerDeclaration above - matchedToken.kind = charLiteralIndex; - input_stream.backup(image.length() - matchedToken.image.length() ); - } - } - } -| < S_QUOTED_IDENTIFIER: "\"" ( "\"\"" | ~["\n","\r","\""])* "\"" | ("`" ("``" | ~["\n","\r","`"])* "`") | ( "[" (~["\n","\r","]"])* "]" ) > - { + int end = closingSingleQuote(matchedToken.image, backslashEscapes); + if (end > 0 && end < matchedToken.image.length()) { + matchedToken.image = image.substring(0, end); + input_stream.backup(image.length() - end); + } + } +| < S_QUOTED_IDENTIFIER: "\"" ( "\"\"" | ~["\n","\r","\""])* "\"" + // MySQL double-quoted string with backslash escapes; trimmed below when not enabled + | "\"" ( "\"\"" | "\\" ~["\n","\r"] | ~["\n","\r","\"","\\"])* "\"" + | ("`" ("``" | ~["\n","\r","`"])* "`") | ( "[" (~["\n","\r","]"])* "]" ) > + { + if ( matchedToken.image.charAt(0) == '"' ) { + // Both alternatives above compete for the longest match. Keep the closing quote of + // the active quoting rules: a backslash escapes the next character only in + // double-quoted strings with backslash escapes, as in MySQL. + int end = closingQuote(matchedToken.image, 0, '"', + configuration.getAsBoolean(Feature.allowDoubleQuotedStrings) + && configuration.getAsBoolean(Feature.allowBackslashEscapeCharacter)); + if (end > 0 && end < matchedToken.image.length()) { + matchedToken.image = image.substring(0, end); + input_stream.backup(image.length() - end); + } + } if ( !configuration.getAsBoolean(Feature.allowSquareBracketQuotation) && matchedToken.image.charAt(0) == '[' ) { diff --git a/src/site/sphinx/usage.rst b/src/site/sphinx/usage.rst index 1d204c6e9..a9d98001b 100644 --- a/src/site/sphinx/usage.rst +++ b/src/site/sphinx/usage.rst @@ -1568,7 +1568,7 @@ The individual features * - ``withBackslashEscapeCharacter`` - ``\\..`` escaping inside string literals, in addition to the standard ``'..`` doubling * - ``withDoubleQuotedStrings`` - - ``".."`` reads as a string literal instead of a quoted identifier (BigQuery, Spark/Databricks, MySQL default ``sql_mode``) + - ``".."`` reads as a string literal instead of a quoted identifier (BigQuery, Spark/Databricks, MySQL default ``sql_mode``); combined with ``withBackslashEscapeCharacter``, ``\\"`` does not end the literal, as in MySQL ``"it\\"s"`` * - ``withHashLineComments`` - ``#`` starts a line comment * - ``withAdjacentStringLiterals`` diff --git a/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java b/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java index a1686ff2d..93edeebd4 100644 --- a/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java +++ b/src/test/java/net/sf/jsqlparser/parser/CCJSqlParserUtilTest.java @@ -748,6 +748,75 @@ public void testDoubleQuotedStringsPreset() throws Exception { } } + @Test + public void testDoubleQuotedStringsWithBackslashEscapes() throws Exception { + // MySQL reads "it\"s" as it"s; the escaped quote must not end the literal + for (AbstractJSqlParser.Dialect dialect : new AbstractJSqlParser.Dialect[] { + AbstractJSqlParser.Dialect.MYSQL, AbstractJSqlParser.Dialect.MARIADB, + AbstractJSqlParser.Dialect.BIGQUERY, AbstractJSqlParser.Dialect.DATABRICKS}) { + PlainSelect select = (PlainSelect) ((Select) CCJSqlParserUtil.parse( + "SELECT \"it\\\"s\", \"a\\\\\"", p -> p.withDialect(dialect))) + .getSelectBody(); + assertEquals("it\\\"s", + ((StringValue) select.getSelectItems().get(0).getExpression()).getValue()); + assertEquals("a\\\\", + ((StringValue) select.getSelectItems().get(1).getExpression()).getValue()); + assertEquals("SELECT \"it\\\"s\", \"a\\\\\"", select.toString()); + } + // a doubled quote still continues a MySQL literal + assertEquals("SELECT \"b\"\"c\\\"d\"", CCJSqlParserUtil.parse("SELECT \"b\"\"c\\\"d\"", + p -> p.withDialect(AbstractJSqlParser.Dialect.MYSQL)).toString()); + String alter = "ALTER TABLE t ADD COLUMN s VARCHAR (10) DEFAULT \"it\\\"s\""; + assertEquals(alter, CCJSqlParserUtil + .parse(alter, p -> p.withDialect(AbstractJSqlParser.Dialect.MYSQL)).toString()); + // the same escape needs both features + assertEquals("SELECT \"it\\\"s\"", CCJSqlParserUtil.parse("SELECT \"it\\\"s\"", + p -> p.withDoubleQuotedStrings(true).withBackslashEscapeCharacter(true)) + .toString()); + assertThrows(JSQLParserException.class, () -> CCJSqlParserUtil + .parse("SELECT \"it\\\"s\"", p -> p.withDoubleQuotedStrings(true))); + // MySQL closes "a\"" before b, so the rest is unterminated as on the server + assertThrows(JSQLParserException.class, () -> CCJSqlParserUtil.parse( + "SELECT \"a\\\"\"b\"", p -> p.withDialect(AbstractJSqlParser.Dialect.MYSQL))); + // quoted identifiers keep a trailing backslash + for (AbstractJSqlParser.Dialect dialect : new AbstractJSqlParser.Dialect[] {null, + AbstractJSqlParser.Dialect.POSTGRESQL, AbstractJSqlParser.Dialect.SNOWFLAKE}) { + PlainSelect select = (PlainSelect) ((Select) CCJSqlParserUtil.parse( + "SELECT \"col\\\" FROM \"t\"", p -> { + if (dialect != null) { + p.withDialect(dialect); + } + })).getSelectBody(); + assertEquals("\"col\\\"", + ((Column) select.getSelectItems().get(0).getExpression()).getColumnName()); + assertEquals("SELECT \"col\\\" FROM \"t\"", select.toString()); + } + } + + @Test + public void testSingleQuotedStringsMixBackslashEscapesAndDoubledQuotes() throws Exception { + // each is one MySQL string; the first used to split into a string and an alias + for (String literal : new String[] {"'it\\'s ''x'''", "'a\\0b\\'c'", "'\\Z\\''", + "'a''b\\'c'", "'x\\%\\'y'", "N'it\\'s'"}) { + PlainSelect select = (PlainSelect) ((Select) CCJSqlParserUtil.parse( + "SELECT " + literal + " FROM t", + p -> p.withDialect(AbstractJSqlParser.Dialect.MYSQL))).getSelectBody(); + assertEquals(1, select.getSelectItems().size(), literal); + assertNull(select.getSelectItems().get(0).getAlias(), literal); + assertInstanceOf(StringValue.class, select.getSelectItems().get(0).getExpression()); + assertEquals("SELECT " + literal + " FROM t", select.toString()); + } + // PostgreSQL E'' strings escape quotes; standard strings keep a literal backslash + assertEquals("SELECT E'it\\'s ''x'''", CCJSqlParserUtil.parse("SELECT E'it\\'s ''x'''", + p -> p.withDialect(AbstractJSqlParser.Dialect.POSTGRESQL)).toString()); + assertEquals("SELECT 'a\\', 'b'", CCJSqlParserUtil.parse("SELECT 'a\\', 'b'", + p -> p.withDialect(AbstractJSqlParser.Dialect.POSTGRESQL)).toString()); + // without backslash escapes the literal still ends at the escaped-looking quote + assertEquals("SELECT 'a\\' AS x", CCJSqlParserUtil.parse("SELECT 'a\\' AS x").toString()); + assertEquals("SELECT * FROM t WHERE a LIKE 'x\\_%' ESCAPE '\\'", CCJSqlParserUtil + .parse("SELECT * FROM t WHERE a LIKE 'x\\_%' ESCAPE '\\'").toString()); + } + @Test public void testAdjacentStringLiteralsNewline() throws Exception { // default (off): the MySQL / SQL Server alias reading, unchanged