diff --git a/fuzz/fuzz_seeds/sqlite_2559_variable_tokens.sql b/fuzz/fuzz_seeds/sqlite_2559_variable_tokens.sql new file mode 100644 index 0000000000..c716c3b982 --- /dev/null +++ b/fuzz/fuzz_seeds/sqlite_2559_variable_tokens.sql @@ -0,0 +1,2 @@ +SELECT #a, @2t, :9c, @a$, :a(x), @a(), :::6, :a::b, $a::b, @::b, #::b, ($$::), @é, :é, #é, :::é; +SELECT a FROM t WHERE b = :c; diff --git a/src/dialect/mod.rs b/src/dialect/mod.rs index c867b76f61..c7ab52d63d 100644 --- a/src/dialect/mod.rs +++ b/src/dialect/mod.rs @@ -1284,6 +1284,12 @@ pub trait Dialect: Debug + Any { false } + /// Returns true if `$`, `@`, `:` and `#` parameters follow SQLite's tokenizer rules. + /// See + fn supports_sqlite_variable_syntax(&self) -> bool { + false + } + /// Returns true if this dialect supports `$` as a prefix for money literals /// e.g. `SELECT $123.45` (SQL Server) fn supports_dollar_as_money_prefix(&self) -> bool { diff --git a/src/dialect/sqlite.rs b/src/dialect/sqlite.rs index 44dcc9d267..cae1f5f7b2 100644 --- a/src/dialect/sqlite.rs +++ b/src/dialect/sqlite.rs @@ -117,6 +117,7 @@ impl Dialect for SQLiteDialect { Some(Ok(self.prec_value(Precedence::Ampersand))) } Token::Lt | Token::LtEq | Token::Gt | Token::GtEq => Some(Ok(COMPARISON_PREC)), + Token::Colon => Some(Ok(self.prec_unknown())), _ => None, } } @@ -171,6 +172,10 @@ impl Dialect for SQLiteDialect { true } + fn supports_sqlite_variable_syntax(&self) -> bool { + true + } + fn supports_double_eq_assignment(&self) -> bool { true } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 9276f91ded..5bf756aaef 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1283,6 +1283,11 @@ impl<'a> Tokenizer<'a> { prev_token: Option<&Token>, prev_non_ws_token: Option<&Token>, ) -> Result, TokenizerError> { + if self.dialect.supports_sqlite_variable_syntax() { + if let Some(token) = self.tokenize_sqlite_variable(chars)? { + return Ok(Some(token)); + } + } match chars.peek() { Some(&ch) => match ch { ch if prev_token == Some(&Token::Period) @@ -2158,6 +2163,65 @@ impl<'a> Tokenizer<'a> { } } + /// Scans a `$`, `@`, `:` or `#` variable with SQLite's tokenizer rules, returning `None` + /// without consuming input when no name follows the prefix. + fn tokenize_sqlite_variable(&self, chars: &mut State) -> Result, TokenizerError> { + let mut lookahead = chars.peekable.clone(); + let Some(prefix @ ('$' | '@' | ':' | '#')) = lookahead.next() else { + return Ok(None); + }; + // `#` followed by a digit is a register reference SQLite only allows in nested parses + if prefix == '#' && lookahead.peek().is_some_and(char::is_ascii_digit) { + return Ok(None); + } + let is_name_char = + |c: char| c.is_ascii_alphanumeric() || c == '_' || c == '$' || !c.is_ascii(); + let mut len = 1; + let mut has_name = false; + loop { + match lookahead.next() { + Some(c) if is_name_char(c) => { + has_name = true; + len += 1; + } + Some(':') if lookahead.next_if_eq(&':').is_some() => len += 2, + Some('(') if has_name => { + len += 1; + loop { + match lookahead.next() { + Some(')') => { + len += 1; + break; + } + Some(c) + if !matches!( + c, + ' ' | '\t' | '\n' | '\u{000b}' | '\u{000c}' | '\r' + ) => + { + len += 1 + } + _ => { + return self.tokenizer_error( + chars.location(), + "Unterminated SQLite variable subscript", + ) + } + } + } + break; + } + _ => break, + } + } + if !has_name { + return Ok(None); + } + Ok(Some(Token::Placeholder( + (0..len).filter_map(|_| chars.next()).collect(), + ))) + } + /// Tokenize dollar preceded value (i.e: a string/placeholder) fn tokenize_dollar_preceded_value(&self, chars: &mut State) -> Result { let mut s = String::new(); diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index f71869e820..ecefe43c65 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -10617,7 +10617,8 @@ fn parse_grant() { verified_stmt("GRANT SELECT ON ALL MATERIALIZED VIEWS IN SCHEMA db1.sc1 TO ROLE role1"); verified_stmt("GRANT SELECT ON ALL EXTERNAL TABLES IN SCHEMA db1.sc1 TO ROLE role1"); verified_stmt("GRANT USAGE ON ALL FUNCTIONS IN SCHEMA db1.sc1 TO ROLE role1"); - verified_stmt("GRANT USAGE ON SCHEMA sc1 TO a:b"); + all_dialects_except(|d| d.supports_sqlite_variable_syntax()) + .verified_stmt("GRANT USAGE ON SCHEMA sc1 TO a:b"); verified_stmt("GRANT USAGE ON SCHEMA sc1 TO GROUP group1"); verified_stmt("GRANT OWNERSHIP ON ALL TABLES IN SCHEMA DEV_STAS_ROGOZHIN TO ROLE ANALYST"); verified_stmt("GRANT OWNERSHIP ON ALL TABLES IN SCHEMA DEV_STAS_ROGOZHIN TO ROLE ANALYST COPY CURRENT GRANTS"); @@ -20932,7 +20933,9 @@ fn parse_stage_table_factor() { supported.verified_stmt("SELECT * FROM @stage"); supported.verified_stmt("SELECT * FROM @stage, my_table"); - let unsupported = all_dialects_where(|d| !d.supports_stages() && !d.is_identifier_start('@')); + let unsupported = all_dialects_where(|d| { + !d.supports_stages() && !d.is_identifier_start('@') && !d.supports_sqlite_variable_syntax() + }); assert_eq!( unsupported .parse_sql_statements("SELECT * FROM @stage") diff --git a/tests/sqlparser_sqlite.rs b/tests/sqlparser_sqlite.rs index ad80e95a93..7da301dd39 100644 --- a/tests/sqlparser_sqlite.rs +++ b/tests/sqlparser_sqlite.rs @@ -31,7 +31,7 @@ use sqlparser::ast::Value::Placeholder; use sqlparser::ast::*; use sqlparser::dialect::{AnsiDialect, GenericDialect, SQLiteDialect}; use sqlparser::parser::{ParserError, ParserOptions}; -use sqlparser::tokenizer::Token; +use sqlparser::tokenizer::{Token, Tokenizer}; #[test] fn pragma_no_value() { @@ -596,6 +596,104 @@ fn test_dollar_identifier_as_placeholder() { } } +#[test] +fn parse_sqlite_variable_tokens() { + for sql in [ + "SELECT #a", + "SELECT @a", + "SELECT :a", + "SELECT $a", + "SELECT @2t", + "SELECT :9c", + "SELECT @a$", + "SELECT :a$", + "SELECT :a()", + "SELECT :a(x)", + "SELECT @a()", + "SELECT :::6", + "SELECT :a::b", + "SELECT $a::b", + "SELECT @::b", + "SELECT #::b", + "SELECT ($$::)", + "SELECT @\u{00e9}", + "SELECT :\u{00e9}", + "SELECT #\u{00e9}", + "SELECT :::\u{00e9}", + ] { + sqlite().verified_stmt(sql); + } + sqlite().one_statement_parses_to("SELECT @2t i", "SELECT @2t AS i"); + + let generic = TestedDialects::new(vec![Box::new(GenericDialect {})]); + for (sql, err) in [ + ( + "SELECT :::6", + ParserError::ParserError("Expected: an expression, found: ::".to_string()), + ), + ( + "SELECT ($$::)", + ParserError::TokenizerError( + "Unterminated dollar-quoted string at Line: 1, Column: 14".to_string(), + ), + ), + ] { + assert_eq!(generic.parse_sql_statements(sql), Err(err), "{sql}"); + } + // SQLite ends a subscript at whitespace and rejects the token + assert_eq!( + sqlite().parse_sql_statements("SELECT $a(x y)"), + Err(ParserError::TokenizerError( + "Unterminated SQLite variable subscript at Line: 1, Column: 8".to_string() + )) + ); + // A vertical tab also ends a subscript in SQLite's tokenizer + assert_eq!( + sqlite().parse_sql_statements("SELECT :a(x\u{000b}y)"), + Err(ParserError::TokenizerError( + "Unterminated SQLite variable subscript at Line: 1, Column: 8".to_string() + )) + ); +} + +#[test] +fn parse_colon_is_not_an_operator() { + for (sql, err) in [ + ( + "SELECT b : c", + ParserError::ParserError("Expected: end of statement, found: :".to_string()), + ), + ( + "SELECT a / b : c", + ParserError::ParserError("Expected: end of statement, found: :".to_string()), + ), + ( + "SELECT b : c.d", + ParserError::ParserError("Expected: end of statement, found: :".to_string()), + ), + ( + "SELECT b : \"c\"", + ParserError::ParserError("Expected: end of statement, found: :".to_string()), + ), + ( + "SELECT b :c", + ParserError::ParserError("Expected: end of statement, found: :c".to_string()), + ), + ] { + assert_eq!(sqlite().parse_sql_statements(sql), Err(err), "{sql}"); + } + sqlite().verified_stmt("SELECT a FROM t WHERE b = :c"); +} + +#[test] +fn tokenize_hash_digit_is_not_a_variable() { + let tokens = Tokenizer::new(&SQLiteDialect {}, "#123") + .tokenize() + .unwrap(); + let expected = vec![Token::Sharp, Token::Number("123".to_string(), false)]; + assert_eq!(tokens, expected); +} + #[test] fn test_match_operator() { assert_eq!(