From bf1f4450b875e3ae0766904f8c554da978077527 Mon Sep 17 00:00:00 2001 From: LucaCappelletti94 Date: Fri, 2 Oct 2026 18:14:15 +0200 Subject: [PATCH 1/6] SQLite: Tokenize parameters the way SQLite does --- src/dialect/mod.rs | 6 ++++ src/dialect/sqlite.rs | 11 +++++++ src/tokenizer.rs | 64 +++++++++++++++++++++++++++++++++++++++ tests/sqlparser_common.rs | 7 +++-- tests/sqlparser_sqlite.rs | 52 +++++++++++++++++++++++++++++++ 5 files changed, 138 insertions(+), 2 deletions(-) diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index c867b76f61..c7ab52d63d 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -1284,6 +1284,12 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if `$`, `@`, `:` and `#` parameters follow SQLite's tokenizer rules. + /// See + fn supports_sqlite_variable_syntax(&self) -> bool { + false + } + /// Returns true if this dialect supports `$` as a prefix for money literals /// e.g. `SELECT $123.45` (SQL Server) fn supports_dollar_as_money_prefix(&self) -> bool { diff --git a/src/dialect/sqlite.rs b/src/dialect/sqlite.rs index 44dcc9d267..31fc0f2bd9 100644 --- a/src/dialect/sqlite.rs +++ b/src/dialect/sqlite.rs @@ -46,6 +46,13 @@ impl Dialect for SQLiteDialect { ch == '`' || ch == '"' || ch == '[' } + fn get_next_precedence(&self, parser: &Parser) -> Option> { + match parser.peek_token_ref().token { + Token::Colon => Some(Ok(self.prec_unknown())), + _ => None, + } + } + fn identifier_quote_style(&self, _identifier: &str) -> Option { Some('`') } @@ -171,6 +178,10 @@ impl Dialect for SQLiteDialect { true } + fn supports_sqlite_variable_syntax(&self) -> bool { + true + } + fn supports_double_eq_assignment(&self) -> bool { true } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 9276f91ded..5bf756aaef 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1283,6 +1283,11 @@ impl<'a> Tokenizer<'a> { prev_token: Option<&Token>, prev_non_ws_token: Option<&Token>, ) -> Result, TokenizerError> { + if self.dialect.supports_sqlite_variable_syntax() { + if let Some(token) = self.tokenize_sqlite_variable(chars)? { + return Ok(Some(token)); + } + } match chars.peek() { Some(&ch) => match ch { ch if prev_token == Some(&Token::Period) @@ -2158,6 +2163,65 @@ impl<'a> Tokenizer<'a> { } } + /// Scans a `$`, `@`, `:` or `#` variable with SQLite's tokenizer rules, returning `None` + /// without consuming input when no name follows the prefix. + fn tokenize_sqlite_variable(&self, chars: &mut State) -> Result, TokenizerError> { + let mut lookahead = chars.peekable.clone(); + let Some(prefix @ ('$' | '@' | ':' | '#')) = lookahead.next() else { + return Ok(None); + }; + // `#` followed by a digit is a register reference SQLite only allows in nested parses + if prefix == '#' && lookahead.peek().is_some_and(char::is_ascii_digit) { + return Ok(None); + } + let is_name_char = + |c: char| c.is_ascii_alphanumeric() || c == '_' || c == '$' || !c.is_ascii(); + let mut len = 1; + let mut has_name = false; + loop { + match lookahead.next() { + Some(c) if is_name_char(c) => { + has_name = true; + len += 1; + } + Some(':') if lookahead.next_if_eq(&':').is_some() => len += 2, + Some('(') if has_name => { + len += 1; + loop { + match lookahead.next() { + Some(')') => { + len += 1; + break; + } + Some(c) + if !matches!( + c, + ' ' | '\t' | '\n' | '\u{000b}' | '\u{000c}' | '\r' + ) => + { + len += 1 + } + _ => { + return self.tokenizer_error( + chars.location(), + "Unterminated SQLite variable subscript", + ) + } + } + } + break; + } + _ => break, + } + } + if !has_name { + return Ok(None); + } + Ok(Some(Token::Placeholder( + (0..len).filter_map(|_| chars.next()).collect(), + ))) + } + /// Tokenize dollar preceded value (i.e: a string/placeholder) fn tokenize_dollar_preceded_value(&self, chars: &mut State) -> Result { let mut s = String::new(); diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index f71869e820..ecefe43c65 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -10617,7 +10617,8 @@ fn parse_grant() { verified_stmt("GRANT SELECT ON ALL MATERIALIZED VIEWS IN SCHEMA db1.sc1 TO ROLE role1"); verified_stmt("GRANT SELECT ON ALL EXTERNAL TABLES IN SCHEMA db1.sc1 TO ROLE role1"); verified_stmt("GRANT USAGE ON ALL FUNCTIONS IN SCHEMA db1.sc1 TO ROLE role1"); - verified_stmt("GRANT USAGE ON SCHEMA sc1 TO a:b"); + all_dialects_except(|d| d.supports_sqlite_variable_syntax()) + .verified_stmt("GRANT USAGE ON SCHEMA sc1 TO a:b"); verified_stmt("GRANT USAGE ON SCHEMA sc1 TO GROUP group1"); verified_stmt("GRANT OWNERSHIP ON ALL TABLES IN SCHEMA DEV_STAS_ROGOZHIN TO ROLE ANALYST"); verified_stmt("GRANT OWNERSHIP ON ALL TABLES IN SCHEMA DEV_STAS_ROGOZHIN TO ROLE ANALYST COPY CURRENT GRANTS"); @@ -20932,7 +20933,9 @@ fn parse_stage_table_factor() { supported.verified_stmt("SELECT * FROM @stage"); supported.verified_stmt("SELECT * FROM @stage, my_table"); - let unsupported = all_dialects_where(|d| !d.supports_stages() && !d.is_identifier_start('@')); + let unsupported = all_dialects_where(|d| { + !d.supports_stages() && !d.is_identifier_start('@') && !d.supports_sqlite_variable_syntax() + }); assert_eq!( unsupported .parse_sql_statements("SELECT * FROM @stage") diff --git a/tests/sqlparser_sqlite.rs b/tests/sqlparser_sqlite.rs index ad80e95a93..317ab41e13 100644 --- a/tests/sqlparser_sqlite.rs +++ b/tests/sqlparser_sqlite.rs @@ -1785,4 +1785,56 @@ fn parse_bitwise_shift_precedence_boundaries() { ] { assert_eq!(sqlite().verified_expr(sql), expected, "{sql}"); } +fn parse_sqlite_variable_tokens() { + for sql in [ + "SELECT #a", + "SELECT @a", + "SELECT :a", + "SELECT $a", + "SELECT @2t", + "SELECT :9c", + "SELECT @a$", + "SELECT :a$", + "SELECT :a()", + "SELECT :a(x)", + "SELECT @a()", + "SELECT :::6", + "SELECT :a::b", + "SELECT $a::b", + "SELECT @::b", + "SELECT #::b", + "SELECT ($$::)", + "SELECT @\u{00e9}", + "SELECT :\u{00e9}", + "SELECT #\u{00e9}", + "SELECT :::\u{00e9}", + ] { + sqlite().verified_stmt(sql); + } + sqlite().one_statement_parses_to("SELECT @2t i", "SELECT @2t AS i"); + + let generic = TestedDialects::new(vec![Box::new(GenericDialect {})]); + for sql in ["SELECT :::6", "SELECT ($$::)"] { + assert!(generic.parse_sql_statements(sql).is_err(), "{sql}"); + } + // SQLite ends a subscript at whitespace and rejects the token + assert!(sqlite().parse_sql_statements("SELECT $a(x y)").is_err()); + // A vertical tab also ends a subscript in SQLite's tokenizer + assert!(sqlite() + .parse_sql_statements("SELECT :a(x\u{000b}y)") + .is_err()); +} + +#[test] +fn parse_colon_is_not_an_operator() { + for sql in [ + "SELECT b : c", + "SELECT a / b : c", + "SELECT b : c.d", + "SELECT b : \"c\"", + "SELECT b :c", + ] { + assert!(sqlite().parse_sql_statements(sql).is_err(), "{sql}"); + } + sqlite().verified_stmt("SELECT a FROM t WHERE b = :c"); } From ee8316ed0effb19f07cd83c88cec9bb08e96a2e0 Mon Sep 17 00:00:00 2001 From: LucaCappelletti94 Date: Sat, 17 Oct 2026 00:16:38 +0200 Subject: [PATCH 2/6] Assert exact errors for the new deny lints --- tests/sqlparser_sqlite.rs | 60 ++++++++++++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/tests/sqlparser_sqlite.rs b/tests/sqlparser_sqlite.rs index 317ab41e13..676e16e65a 100644 --- a/tests/sqlparser_sqlite.rs +++ b/tests/sqlparser_sqlite.rs @@ -1814,27 +1814,61 @@ fn parse_sqlite_variable_tokens() { sqlite().one_statement_parses_to("SELECT @2t i", "SELECT @2t AS i"); let generic = TestedDialects::new(vec![Box::new(GenericDialect {})]); - for sql in ["SELECT :::6", "SELECT ($$::)"] { - assert!(generic.parse_sql_statements(sql).is_err(), "{sql}"); + for (sql, err) in [ + ( + "SELECT :::6", + ParserError::ParserError("Expected: an expression, found: ::".to_string()), + ), + ( + "SELECT ($$::)", + ParserError::TokenizerError( + "Unterminated dollar-quoted string at Line: 1, Column: 14".to_string(), + ), + ), + ] { + assert_eq!(generic.parse_sql_statements(sql), Err(err), "{sql}"); } // SQLite ends a subscript at whitespace and rejects the token - assert!(sqlite().parse_sql_statements("SELECT $a(x y)").is_err()); + assert_eq!( + sqlite().parse_sql_statements("SELECT $a(x y)"), + Err(ParserError::TokenizerError( + "Unterminated SQLite variable subscript at Line: 1, Column: 8".to_string() + )) + ); // A vertical tab also ends a subscript in SQLite's tokenizer - assert!(sqlite() - .parse_sql_statements("SELECT :a(x\u{000b}y)") - .is_err()); + assert_eq!( + sqlite().parse_sql_statements("SELECT :a(x\u{000b}y)"), + Err(ParserError::TokenizerError( + "Unterminated SQLite variable subscript at Line: 1, Column: 8".to_string() + )) + ); } #[test] fn parse_colon_is_not_an_operator() { - for sql in [ - "SELECT b : c", - "SELECT a / b : c", - "SELECT b : c.d", - "SELECT b : \"c\"", - "SELECT b :c", + for (sql, err) in [ + ( + "SELECT b : c", + ParserError::ParserError("Expected: end of statement, found: :".to_string()), + ), + ( + "SELECT a / b : c", + ParserError::ParserError("Expected: end of statement, found: :".to_string()), + ), + ( + "SELECT b : c.d", + ParserError::ParserError("Expected: end of statement, found: :".to_string()), + ), + ( + "SELECT b : \"c\"", + ParserError::ParserError("Expected: end of statement, found: :".to_string()), + ), + ( + "SELECT b :c", + ParserError::ParserError("Expected: end of statement, found: :c".to_string()), + ), ] { - assert!(sqlite().parse_sql_statements(sql).is_err(), "{sql}"); + assert_eq!(sqlite().parse_sql_statements(sql), Err(err), "{sql}"); } sqlite().verified_stmt("SELECT a FROM t WHERE b = :c"); } From df21d02f73832c4d6238e6fe71db80cb08b6a6d8 Mon Sep 17 00:00:00 2001 From: LucaCappelletti94 Date: Tue, 20 Oct 2026 12:51:25 +0200 Subject: [PATCH 3/6] Cover hash-digit variable fallback --- tests/sqlparser_sqlite.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/sqlparser_sqlite.rs b/tests/sqlparser_sqlite.rs index 676e16e65a..f4a91ac15f 100644 --- a/tests/sqlparser_sqlite.rs +++ b/tests/sqlparser_sqlite.rs @@ -31,7 +31,7 @@ use sqlparser::ast::Value::Placeholder; use sqlparser::ast::*; use sqlparser::dialect::{AnsiDialect, GenericDialect, SQLiteDialect}; use sqlparser::parser::{ParserError, ParserOptions}; -use sqlparser::tokenizer::Token; +use sqlparser::tokenizer::{Token, Tokenizer}; #[test] fn pragma_no_value() { @@ -1872,3 +1872,12 @@ fn parse_colon_is_not_an_operator() { } sqlite().verified_stmt("SELECT a FROM t WHERE b = :c"); } + +#[test] +fn tokenize_hash_digit_is_not_a_variable() { + let tokens = Tokenizer::new(&SQLiteDialect {}, "#123") + .tokenize() + .unwrap(); + let expected = vec![Token::Sharp, Token::Number("123".to_string(), false)]; + assert_eq!(tokens, expected); +} From 368b9af83c66cd221d06eb9963d70c1b5a96024e Mon Sep 17 00:00:00 2001 From: LucaCappelletti94 Date: Wed, 21 Oct 2026 09:52:53 +0200 Subject: [PATCH 4/6] Combine SQLite variable and bitwise precedence overrides --- src/dialect/sqlite.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/dialect/sqlite.rs b/src/dialect/sqlite.rs index 31fc0f2bd9..cae1f5f7b2 100644 --- a/src/dialect/sqlite.rs +++ b/src/dialect/sqlite.rs @@ -46,13 +46,6 @@ impl Dialect for SQLiteDialect { ch == '`' || ch == '"' || ch == '[' } - fn get_next_precedence(&self, parser: &Parser) -> Option> { - match parser.peek_token_ref().token { - Token::Colon => Some(Ok(self.prec_unknown())), - _ => None, - } - } - fn identifier_quote_style(&self, _identifier: &str) -> Option { Some('`') } @@ -124,6 +117,7 @@ impl Dialect for SQLiteDialect { Some(Ok(self.prec_value(Precedence::Ampersand))) } Token::Lt | Token::LtEq | Token::Gt | Token::GtEq => Some(Ok(COMPARISON_PREC)), + Token::Colon => Some(Ok(self.prec_unknown())), _ => None, } } From 1bc69b4483a6b1bb77817d92405c8aa643e42eeb Mon Sep 17 00:00:00 2001 From: LucaCappelletti94 Date: Sat, 24 Oct 2026 00:26:24 +0200 Subject: [PATCH 5/6] Place regression tests beside related syntax cases --- tests/sqlparser_sqlite.rs | 193 +++++++++++++++++++------------------- 1 file changed, 98 insertions(+), 95 deletions(-) diff --git a/tests/sqlparser_sqlite.rs b/tests/sqlparser_sqlite.rs index f4a91ac15f..7da301dd39 100644 --- a/tests/sqlparser_sqlite.rs +++ b/tests/sqlparser_sqlite.rs @@ -596,6 +596,104 @@ fn test_dollar_identifier_as_placeholder() { } } +#[test] +fn parse_sqlite_variable_tokens() { + for sql in [ + "SELECT #a", + "SELECT @a", + "SELECT :a", + "SELECT $a", + "SELECT @2t", + "SELECT :9c", + "SELECT @a$", + "SELECT :a$", + "SELECT :a()", + "SELECT :a(x)", + "SELECT @a()", + "SELECT :::6", + "SELECT :a::b", + "SELECT $a::b", + "SELECT @::b", + "SELECT #::b", + "SELECT ($$::)", + "SELECT @\u{00e9}", + "SELECT :\u{00e9}", + "SELECT #\u{00e9}", + "SELECT :::\u{00e9}", + ] { + sqlite().verified_stmt(sql); + } + sqlite().one_statement_parses_to("SELECT @2t i", "SELECT @2t AS i"); + + let generic = TestedDialects::new(vec![Box::new(GenericDialect {})]); + for (sql, err) in [ + ( + "SELECT :::6", + ParserError::ParserError("Expected: an expression, found: ::".to_string()), + ), + ( + "SELECT ($$::)", + ParserError::TokenizerError( + "Unterminated dollar-quoted string at Line: 1, Column: 14".to_string(), + ), + ), + ] { + assert_eq!(generic.parse_sql_statements(sql), Err(err), "{sql}"); + } + // SQLite ends a subscript at whitespace and rejects the token + assert_eq!( + sqlite().parse_sql_statements("SELECT $a(x y)"), + Err(ParserError::TokenizerError( + "Unterminated SQLite variable subscript at Line: 1, Column: 8".to_string() + )) + ); + // A vertical tab also ends a subscript in SQLite's tokenizer + assert_eq!( + sqlite().parse_sql_statements("SELECT :a(x\u{000b}y)"), + Err(ParserError::TokenizerError( + "Unterminated SQLite variable subscript at Line: 1, Column: 8".to_string() + )) + ); +} + +#[test] +fn parse_colon_is_not_an_operator() { + for (sql, err) in [ + ( + "SELECT b : c", + ParserError::ParserError("Expected: end of statement, found: :".to_string()), + ), + ( + "SELECT a / b : c", + ParserError::ParserError("Expected: end of statement, found: :".to_string()), + ), + ( + "SELECT b : c.d", + ParserError::ParserError("Expected: end of statement, found: :".to_string()), + ), + ( + "SELECT b : \"c\"", + ParserError::ParserError("Expected: end of statement, found: :".to_string()), + ), + ( + "SELECT b :c", + ParserError::ParserError("Expected: end of statement, found: :c".to_string()), + ), + ] { + assert_eq!(sqlite().parse_sql_statements(sql), Err(err), "{sql}"); + } + sqlite().verified_stmt("SELECT a FROM t WHERE b = :c"); +} + +#[test] +fn tokenize_hash_digit_is_not_a_variable() { + let tokens = Tokenizer::new(&SQLiteDialect {}, "#123") + .tokenize() + .unwrap(); + let expected = vec![Token::Sharp, Token::Number("123".to_string(), false)]; + assert_eq!(tokens, expected); +} + #[test] fn test_match_operator() { assert_eq!( @@ -1785,99 +1883,4 @@ fn parse_bitwise_shift_precedence_boundaries() { ] { assert_eq!(sqlite().verified_expr(sql), expected, "{sql}"); } -fn parse_sqlite_variable_tokens() { - for sql in [ - "SELECT #a", - "SELECT @a", - "SELECT :a", - "SELECT $a", - "SELECT @2t", - "SELECT :9c", - "SELECT @a$", - "SELECT :a$", - "SELECT :a()", - "SELECT :a(x)", - "SELECT @a()", - "SELECT :::6", - "SELECT :a::b", - "SELECT $a::b", - "SELECT @::b", - "SELECT #::b", - "SELECT ($$::)", - "SELECT @\u{00e9}", - "SELECT :\u{00e9}", - "SELECT #\u{00e9}", - "SELECT :::\u{00e9}", - ] { - sqlite().verified_stmt(sql); - } - sqlite().one_statement_parses_to("SELECT @2t i", "SELECT @2t AS i"); - - let generic = TestedDialects::new(vec![Box::new(GenericDialect {})]); - for (sql, err) in [ - ( - "SELECT :::6", - ParserError::ParserError("Expected: an expression, found: ::".to_string()), - ), - ( - "SELECT ($$::)", - ParserError::TokenizerError( - "Unterminated dollar-quoted string at Line: 1, Column: 14".to_string(), - ), - ), - ] { - assert_eq!(generic.parse_sql_statements(sql), Err(err), "{sql}"); - } - // SQLite ends a subscript at whitespace and rejects the token - assert_eq!( - sqlite().parse_sql_statements("SELECT $a(x y)"), - Err(ParserError::TokenizerError( - "Unterminated SQLite variable subscript at Line: 1, Column: 8".to_string() - )) - ); - // A vertical tab also ends a subscript in SQLite's tokenizer - assert_eq!( - sqlite().parse_sql_statements("SELECT :a(x\u{000b}y)"), - Err(ParserError::TokenizerError( - "Unterminated SQLite variable subscript at Line: 1, Column: 8".to_string() - )) - ); -} - -#[test] -fn parse_colon_is_not_an_operator() { - for (sql, err) in [ - ( - "SELECT b : c", - ParserError::ParserError("Expected: end of statement, found: :".to_string()), - ), - ( - "SELECT a / b : c", - ParserError::ParserError("Expected: end of statement, found: :".to_string()), - ), - ( - "SELECT b : c.d", - ParserError::ParserError("Expected: end of statement, found: :".to_string()), - ), - ( - "SELECT b : \"c\"", - ParserError::ParserError("Expected: end of statement, found: :".to_string()), - ), - ( - "SELECT b :c", - ParserError::ParserError("Expected: end of statement, found: :c".to_string()), - ), - ] { - assert_eq!(sqlite().parse_sql_statements(sql), Err(err), "{sql}"); - } - sqlite().verified_stmt("SELECT a FROM t WHERE b = :c"); -} - -#[test] -fn tokenize_hash_digit_is_not_a_variable() { - let tokens = Tokenizer::new(&SQLiteDialect {}, "#123") - .tokenize() - .unwrap(); - let expected = vec![Token::Sharp, Token::Number("123".to_string(), false)]; - assert_eq!(tokens, expected); } From 6a3380c8d5c8a6fd35222a9b383198d9f4194424 Mon Sep 17 00:00:00 2001 From: LucaCappelletti94 Date: Sat, 24 Oct 2026 13:43:00 +0200 Subject: [PATCH 6/6] Add fuzz seeds for sqlite variable tokens --- fuzz/fuzz_seeds/sqlite_2559_variable_tokens.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 fuzz/fuzz_seeds/sqlite_2559_variable_tokens.sql diff --git a/fuzz/fuzz_seeds/sqlite_2559_variable_tokens.sql b/fuzz/fuzz_seeds/sqlite_2559_variable_tokens.sql new file mode 100644 index 0000000000..c716c3b982 --- /dev/null +++ b/fuzz/fuzz_seeds/sqlite_2559_variable_tokens.sql @@ -0,0 +1,2 @@ +SELECT #a, @2t, :9c, @a$, :a(x), @a(), :::6, :a::b, $a::b, @::b, #::b, ($$::), @é, :é, #é, :::é; +SELECT a FROM t WHERE b = :c;