From c55818fbfb3c3a9f652532d1fceb46ea72ac91a9 Mon Sep 17 00:00:00 2001 From: Jules Ivanic Date: Tue, 13 Oct 2026 19:22:52 +1100 Subject: [PATCH 1/2] DuckDB: accept digit separators in numeric literals `1_000_000`, `92_000.0` and `1e1_0` are numbers in DuckDB and PostgreSQL 16, but MySQL reads `2024_01` as an identifier. The new `Feature.allowDigitSeparators`, enabled by the PostgreSQL dialect, drops the separators from the token image, so every consumer of number tokens keeps working. Without it, lexing is unchanged. --- .../jsqlparser/parser/AbstractJSqlParser.java | 11 +++- .../sf/jsqlparser/parser/feature/Feature.java | 6 ++ .../net/sf/jsqlparser/parser/JSqlParserCC.jjt | 36 ++++++++++- .../expression/DigitSeparatorsTest.java | 64 +++++++++++++++++++ 4 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 src/test/java/net/sf/jsqlparser/expression/DigitSeparatorsTest.java diff --git a/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java b/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java index 9a1b4100b..86ba7aa07 100644 --- a/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java +++ b/src/main/java/net/sf/jsqlparser/parser/AbstractJSqlParser.java @@ -39,7 +39,8 @@ public enum Dialect { Feature.allowDoubleQuotedStrings), SQLSERVER(AdjacentStringLiterals.OFF, Feature.allowSquareBracketQuotation), POSTGRESQL( AdjacentStringLiterals.NEWLINE, - Feature.allowDollarQuotedStringTags), H2, EXASOL, BIGQUERY( + Feature.allowDollarQuotedStringTags, + Feature.allowDigitSeparators), H2, EXASOL, BIGQUERY( AdjacentStringLiterals.WHITESPACE, Feature.allowDoubleQuotedStrings, Feature.allowHashLineComments, @@ -194,6 +195,14 @@ public P withHashLineComments(boolean allowHashLineComments) { return withFeature(Feature.allowHashLineComments, allowHashLineComments); } + public P withDigitSeparators() { + return withFeature(Feature.allowDigitSeparators, true); + } + + public P withDigitSeparators(boolean allowDigitSeparators) { + return withFeature(Feature.allowDigitSeparators, allowDigitSeparators); + } + public P withUnparenthesizedSubSelects() { return withFeature(Feature.allowUnparenthesizedSubSelects, true); } diff --git a/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java b/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java index 9e0bd5ce1..fea65dfea 100644 --- a/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java +++ b/src/main/java/net/sf/jsqlparser/parser/feature/Feature.java @@ -889,6 +889,12 @@ public enum Feature { */ allowHashLineComments(false), + /** + * allows digit separators in numeric literals, e.g. {@code 1_000_000} (DuckDB, PostgreSQL 16); + * disabled by default, where MySQL reads {@code 2024_01} as an identifier + */ + allowDigitSeparators(false), + /** * allows sub selects without parentheses, e.g. `select * from dual where 1 = select 1` */ diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index 64172751a..f586eb99b 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -2431,6 +2431,38 @@ TOKEN_MGR_DECLS : { } /** Splits a longest-match identifier without losing token positions. */ + /** + * DuckDB's and PostgreSQL's digit separators, as in {@code 1_000_000}. Under + * Feature.allowDigitSeparators they are dropped from the image. Otherwise the token is + * lexed as without separators: the longer of the number before the first {@code _} and the + * identifier starting there, so that MySQL's {@code 2024_01} stays an identifier. + */ + private void lexDigitSeparators(Token token) { + int separator = token.image.indexOf('_'); + if (separator < 0) { + return; + } + if (input_stream.featureConfiguration != null + && Boolean.TRUE.equals(configuration.getValue(Feature.allowDigitSeparators))) { + token.image = token.image.replace("_", ""); + return; + } + int identifierLength = 0; + while (identifierLength < token.image.length() + && (Character.isLetterOrDigit(token.image.charAt(identifierLength)) + || token.image.charAt(identifierLength) == '_')) { + identifierLength++; + } + if (identifierLength > separator) { + truncateIdentifierToken(token, identifierLength); + token.kind = S_IDENTIFIER; + } else { + truncateIdentifierToken(token, separator); + token.kind = token.image.indexOf('.') >= 0 || token.image.indexOf('e') >= 0 + || token.image.indexOf('E') >= 0 ? S_DOUBLE : S_LONG; + } + } + private void truncateIdentifierToken(Token token, int length) { input_stream.backup(token.image.length() - length); token.image = token.image.substring(0, length); @@ -3126,8 +3158,8 @@ TOKEN : /* Numeric Constants */ "." (["e","E"] (["+", "-"])? )? | ["e","E"] (["+", "-"])? - )> - | < S_LONG: ( )+ > + )> { lexDigitSeparators(matchedToken); } + | < S_LONG: ( )+ ( "_" ( )+ )* > { lexDigitSeparators(matchedToken); } | < #DIGIT: ["0" - "9"] > | < S_HEX: ("X" ( (" ")* )* | "0x" ( )+ ) > | < #QUOTED_HEX: "'" ( )* "'" > diff --git a/src/test/java/net/sf/jsqlparser/expression/DigitSeparatorsTest.java b/src/test/java/net/sf/jsqlparser/expression/DigitSeparatorsTest.java new file mode 100644 index 000000000..c9b46ac60 --- /dev/null +++ b/src/test/java/net/sf/jsqlparser/expression/DigitSeparatorsTest.java @@ -0,0 +1,64 @@ +/*- + * #%L + * JSQLParser library + * %% + * Copyright (C) 2004 - 2026 JSQLParser + * %% + * Dual licensed under GNU LGPL 2.1 or Apache License 2.0 + * #L% + */ +package net.sf.jsqlparser.expression; + +import net.sf.jsqlparser.JSQLParserException; +import net.sf.jsqlparser.parser.AbstractJSqlParser.Dialect; +import net.sf.jsqlparser.parser.CCJSqlParserUtil; +import net.sf.jsqlparser.schema.Column; +import net.sf.jsqlparser.statement.select.PlainSelect; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +class DigitSeparatorsTest { + + private static Expression firstItem(String sql, boolean digitSeparators) + throws JSQLParserException { + PlainSelect select = (PlainSelect) CCJSqlParserUtil.parse(sql, + parser -> parser.withDigitSeparators(digitSeparators)); + return select.getSelectItem(0).getExpression(); + } + + @ParameterizedTest + @CsvSource({"1_000, 1000", "1_000_000, 1000000", "92_000.0, 92000.0", "1_000.5_5, 1000.55", + ".5_5, .55", "1e1_0, 1e10", "1_0e10, 10e10", "1.5e+1_0, 1.5e+10"}) + void testSeparatorsAreDropped(String literal, String expected) throws JSQLParserException { + assertEquals(expected, firstItem("SELECT " + literal, true).toString()); + } + + @Test + void testSeparatedNumbersAreNumbers() throws JSQLParserException { + assertEquals(1000L, ((LongValue) firstItem("SELECT 1_000", true)).getValue()); + assertEquals(92000.0, ((DoubleValue) firstItem("SELECT 92_000.0", true)).getValue()); + assertEquals("SELECT * FROM t LIMIT 1000", CCJSqlParserUtil + .parse("SELECT * FROM t LIMIT 1_000", parser -> parser.withDigitSeparators()) + .toString()); + } + + @Test + void testPostgreSqlAllowsSeparators() throws JSQLParserException { + PlainSelect select = (PlainSelect) CCJSqlParserUtil.parse("SELECT 1_000", + parser -> parser.withDialect(Dialect.POSTGRESQL)); + assertInstanceOf(LongValue.class, select.getSelectItem(0).getExpression()); + } + + // MySQL identifiers may start with digits + @Test + void testWithoutSeparatorsDigitLedNamesStayIdentifiers() throws JSQLParserException { + assertInstanceOf(Column.class, firstItem("SELECT 1_000", false)); + assertInstanceOf(Column.class, firstItem("SELECT 2024_01.c FROM 2024_01", false)); + assertEquals("SELECT .5 _5", CCJSqlParserUtil.parse("SELECT .5_5").toString()); + assertEquals("SELECT 1.5e+1 _0", CCJSqlParserUtil.parse("SELECT 1.5e+1_0").toString()); + } +} From 2c7ae8c6188d83a1a0917df35bde7d770cd84938 Mon Sep 17 00:00:00 2001 From: Jules Ivanic Date: Tue, 13 Oct 2026 19:33:22 +1100 Subject: [PATCH 2/2] Keep the javadoc of `truncateIdentifierToken` above it --- src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt index f586eb99b..e18d7589b 100644 --- a/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt +++ b/src/main/jjtree/net/sf/jsqlparser/parser/JSqlParserCC.jjt @@ -2430,7 +2430,6 @@ TOKEN_MGR_DECLS : { token.endColumn = input_stream.getEndColumn(); } - /** Splits a longest-match identifier without losing token positions. */ /** * DuckDB's and PostgreSQL's digit separators, as in {@code 1_000_000}. Under * Feature.allowDigitSeparators they are dropped from the image. Otherwise the token is @@ -2463,6 +2462,7 @@ TOKEN_MGR_DECLS : { } } + /** Splits a longest-match identifier without losing token positions. */ private void truncateIdentifierToken(Token token, int length) { input_stream.backup(token.image.length() - length); token.image = token.image.substring(0, length);