From 1e9b0995d03e99d61f9b75ecd2812dc3bc49d3a8 Mon Sep 17 00:00:00 2001 From: Mosha Pasumansky Date: Wed, 12 Aug 2026 11:05:56 -0700 Subject: [PATCH 1/3] BigQuery: support `CREATE VECTOR INDEX` BigQuery can create a vector index for approximate nearest-neighbor search over an embedding column: CREATE [OR REPLACE] VECTOR INDEX [IF NOT EXISTS] ON () OPTIONS(index_type = 'IVF', distance_type = 'COSINE', ...) `VECTOR` is not a keyword, so `parse_create` previously failed with `Expected: an object type after CREATE, found: VECTOR`. ### Changes - Add `vector`, `or_replace` and `options` fields to `CreateIndex`. `VECTOR` is a modifier on `CREATE INDEX` (like `EXTERNAL` on `CREATE TABLE`), so it reuses the existing node rather than a new statement variant. `Display` renders `CREATE [OR REPLACE ]VECTOR INDEX ...` plus a trailing `OPTIONS(...)`. - Parse the form in `parse_create` via a small `parse_create_vector_index` helper; the `OPTIONS(...)` clause reuses `parse_options` / `SqlOption`, so it parses and renders the same way as `CREATE TABLE` / `CREATE VIEW` OPTIONS. - Plain `CREATE INDEX` is unchanged (all three fields default to false/empty). - New test `parse_bigquery_create_vector_index` in `tests/sqlparser_bigquery.rs` verifies the round-trip and covers `OR REPLACE`, `IF NOT EXISTS`, multi-part names and the OPTIONS-less form. Docs: https://cloud.google.com/bigquery/docs/reference/standard-sql/data-definition-language#create_vector_index_statement Co-authored-by: Claude Opus 4.8 --- src/ast/ddl.rs | 13 ++++++- src/ast/spans.rs | 4 ++ src/parser/mod.rs | 40 +++++++++++++++++++ tests/sqlparser_bigquery.rs | 77 +++++++++++++++++++++++++++++++++++++ tests/sqlparser_common.rs | 6 +++ tests/sqlparser_postgres.rs | 33 ++++++++++++++++ 6 files changed, 172 insertions(+), 1 deletion(-) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index a0e69ad8ac..0709f63ebe 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -2827,6 +2827,10 @@ pub struct CreateIndex { pub using: Option, /// columns included in the index pub columns: Vec, + /// whether this is a BigQuery `CREATE VECTOR INDEX` + pub vector: bool, + /// whether the statement is `CREATE OR REPLACE` + pub or_replace: bool, /// whether the index is unique pub unique: bool, /// whether the index is created concurrently @@ -2843,6 +2847,8 @@ pub struct CreateIndex { pub nulls_distinct: Option, /// WITH clause: pub with: Vec, + /// BigQuery `OPTIONS(...)` clause, e.g. on `CREATE VECTOR INDEX` + pub options: Vec, /// WHERE clause: pub predicate: Option, /// Index options: @@ -2860,8 +2866,10 @@ impl fmt::Display for CreateIndex { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!( f, - "CREATE {unique}INDEX {concurrently}{async_}{if_not_exists}", + "CREATE {or_replace}{unique}{vector}INDEX {concurrently}{async_}{if_not_exists}", + or_replace = if self.or_replace { "OR REPLACE " } else { "" }, unique = if self.unique { "UNIQUE " } else { "" }, + vector = if self.vector { "VECTOR " } else { "" }, concurrently = if self.concurrently { "CONCURRENTLY " } else { @@ -2882,6 +2890,9 @@ impl fmt::Display for CreateIndex { write!(f, " USING {value} ")?; } write!(f, "({})", display_comma_separated(&self.columns))?; + if !self.options.is_empty() { + write!(f, " OPTIONS({})", display_comma_separated(&self.options))?; + } if !self.include.is_empty() { write!(f, " INCLUDE ({})", display_comma_separated(&self.include))?; } diff --git a/src/ast/spans.rs b/src/ast/spans.rs index a34fe66d98..ab5f105c8f 100644 --- a/src/ast/spans.rs +++ b/src/ast/spans.rs @@ -699,6 +699,8 @@ impl Spanned for CreateIndex { table_name, using: _, columns, + vector: _, // bool + or_replace: _, // bool unique: _, // bool concurrently: _, // bool r#async: _, // bool @@ -706,6 +708,7 @@ impl Spanned for CreateIndex { include, nulls_distinct: _, // bool with, + options, predicate, index_options: _, alter_options, @@ -718,6 +721,7 @@ impl Spanned for CreateIndex { .chain(columns.iter().map(|i| i.column.span())) .chain(include.iter().map(|i| i.span)) .chain(with.iter().map(|i| i.span())) + .chain(options.iter().map(|i| i.span())) .chain(predicate.iter().map(|i| i.span())) .chain(alter_options.iter().map(|i| i.span())), ) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index b2b3f42bbf..530fd2534f 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -5288,6 +5288,13 @@ impl<'a> Parser<'a> { self.parse_create_schema(or_replace) } else if self.parse_keyword(Keyword::WAREHOUSE) { self.parse_create_warehouse(or_replace).map(Into::into) + } else if matches!( + &self.peek_token_ref().token, + Token::Word(w) if w.keyword == Keyword::NoKeyword && w.value.eq_ignore_ascii_case("VECTOR") + ) { + // BigQuery `CREATE [OR REPLACE] VECTOR INDEX ...`; VECTOR is not a keyword. + self.next_token(); + self.parse_create_vector_index(or_replace).map(Into::into) } else if or_replace { self.expected_ref( "[EXTERNAL] TABLE or [MATERIALIZED] VIEW or FUNCTION or SCHEMA or WAREHOUSE after CREATE OR REPLACE", @@ -8245,6 +8252,36 @@ impl<'a> Parser<'a> { Ok(Statement::Discard { object_type }) } + /// Parse a BigQuery `CREATE [OR REPLACE] VECTOR INDEX` statement (`VECTOR` already consumed). + fn parse_create_vector_index(&mut self, or_replace: bool) -> Result { + self.expect_keyword_is(Keyword::INDEX)?; + let if_not_exists = self.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]); + let name = self.parse_object_name(false)?; + self.expect_keyword_is(Keyword::ON)?; + let table_name = self.parse_object_name(false)?; + let columns = self.parse_parenthesized_index_column_list()?; + let options = self.parse_options(Keyword::OPTIONS)?; + Ok(CreateIndex { + name: Some(name), + table_name, + using: None, + columns, + vector: true, + or_replace, + unique: false, + concurrently: false, + r#async: false, + if_not_exists, + include: vec![], + nulls_distinct: None, + with: vec![], + options, + predicate: None, + index_options: vec![], + alter_options: vec![], + }) + } + /// Parse a `CREATE INDEX` statement. pub fn parse_create_index(&mut self, unique: bool) -> Result { let concurrently = self.parse_keyword(Keyword::CONCURRENTLY); @@ -8326,6 +8363,8 @@ impl<'a> Parser<'a> { table_name, using, columns, + vector: false, + or_replace: false, unique, concurrently, r#async, @@ -8333,6 +8372,7 @@ impl<'a> Parser<'a> { include, nulls_distinct, with, + options: vec![], predicate, index_options, alter_options, diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index f6d4483c24..e1c10d943e 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -2950,3 +2950,80 @@ fn test_create_snapshot_table() { "CREATE SNAPSHOT TABLE IF NOT EXISTS dataset_id.table1 CLONE dataset_id.table2 FOR SYSTEM_TIME AS OF TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR) OPTIONS(expiration_timestamp = TIMESTAMP '2025-01-01 00:00:00 UTC')", ); } +#[test] +fn parse_bigquery_create_vector_index() { + // `CREATE VECTOR INDEX ... OPTIONS(...)` is a CreateIndex flagged `vector`. + let sql = + "CREATE VECTOR INDEX emb ON t(embedding) OPTIONS(distance_type = 'COSINE', dimension = 4)"; + match bigquery().verified_stmt(sql) { + Statement::CreateIndex(CreateIndex { + name, + table_name, + using, + columns, + vector, + or_replace, + unique, + if_not_exists, + with, + options, + .. + }) => { + assert!(vector); + assert!(!or_replace); + assert!(!unique); + assert!(!if_not_exists); + assert_eq!(name.unwrap().to_string(), "emb"); + assert_eq!(table_name.to_string(), "t"); + assert_eq!(using, None); + assert!(with.is_empty()); + assert_eq!( + columns.iter().map(|c| c.to_string()).collect::>(), + vec!["embedding"] + ); + assert_eq!( + options, + vec![ + SqlOption::KeyValue { + key: Ident::new("distance_type"), + value: Expr::Value( + Value::SingleQuotedString("COSINE".to_string()).with_empty_span() + ), + }, + SqlOption::KeyValue { + key: Ident::new("dimension"), + value: Expr::value(number("4")), + }, + ] + ); + } + other => panic!("expected CreateIndex, got {other:?}"), + } + + // OR REPLACE, IF NOT EXISTS, multi-part names, and the OPTIONS-less form all round-trip. + match bigquery().verified_stmt( + "CREATE OR REPLACE VECTOR INDEX emb ON t(embedding) OPTIONS(distance_type = 'COSINE')", + ) { + Statement::CreateIndex(CreateIndex { + vector, or_replace, .. + }) => { + assert!(vector); + assert!(or_replace); + } + other => panic!("expected CreateIndex, got {other:?}"), + } + match bigquery().verified_stmt( + "CREATE VECTOR INDEX IF NOT EXISTS mydataset.emb ON mydataset.t(embedding) OPTIONS(distance_type = 'EUCLIDEAN')", + ) { + Statement::CreateIndex(CreateIndex { + vector, + if_not_exists, + .. + }) => { + assert!(vector); + assert!(if_not_exists); + } + other => panic!("expected CreateIndex, got {other:?}"), + } + bigquery().verified_stmt("CREATE VECTOR INDEX emb ON t(embedding)"); +} diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 0800bc41fd..99e1304839 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -9728,6 +9728,9 @@ fn test_create_index_with_using_function() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq!("idx_name", name.to_string()); assert_eq!("test", table_name.to_string()); @@ -9785,6 +9788,9 @@ fn test_create_index_with_with_clause() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { pretty_assertions::assert_eq!("title_idx", name.to_string()); pretty_assertions::assert_eq!("films", table_name.to_string()); diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index a7128eafd8..9c12dc8af7 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -2993,6 +2993,9 @@ fn parse_create_index() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3030,6 +3033,9 @@ fn parse_create_anonymous_index() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq!(None, name); assert_eq_vec(&["my_table"], &table_name); @@ -3150,6 +3156,9 @@ fn parse_create_indices_with_operator_classes() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["the_index_name"], &name); assert_eq_vec(&["users"], &table_name); @@ -3179,6 +3188,9 @@ fn parse_create_indices_with_operator_classes() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["the_index_name"], &name); assert_eq_vec(&["users"], &table_name); @@ -3263,6 +3275,9 @@ fn parse_create_bloom() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["bloomidx"], &name); assert_eq_vec(&["tbloom"], &table_name); @@ -3320,6 +3335,9 @@ fn parse_create_brin() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["brin_sensor_data_recorded_at"], &name); assert_eq_vec(&["sensor_data"], &table_name); @@ -3388,6 +3406,9 @@ fn parse_create_index_concurrently() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3425,6 +3446,9 @@ fn parse_create_index_with_predicate() { predicate: Some(_), index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3462,6 +3486,9 @@ fn parse_create_index_with_include() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3499,6 +3526,9 @@ fn parse_create_index_with_nulls_distinct() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3534,6 +3564,9 @@ fn parse_create_index_with_nulls_distinct() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); From 265c46988693a1f25ba508537811acd4db717c73 Mon Sep 17 00:00:00 2001 From: Mosha Pasumansky Date: Tue, 18 Aug 2026 12:52:03 -0700 Subject: [PATCH 2/3] Generalize `CREATE VECTOR INDEX` beyond BigQuery Address review feedback that `CREATE VECTOR INDEX` is not BigQuery-specific (Oracle, SQL Server, MariaDB and TiDB also have it, hence Generic too). - Route the statement through `parse_create_index` instead of a separate BigQuery helper, so it inherits the standard index trailers (`USING`, `INCLUDE`, `WITH`, expression targets, index options) that cover the Oracle / SQL Server / TiDB variants, plus the BigQuery `OPTIONS(...)` clause. - Align `Display` order (OPTIONS after WITH) with parse order so `INCLUDE` + `OPTIONS` combinations round-trip. - Drop the BigQuery-specific doc-comment framing. - Move the test to `tests/sqlparser_common.rs` as `parse_create_vector_index`, running across all dialects and covering `OR REPLACE`, `IF NOT EXISTS`, schema-qualified names, `OPTIONS(...)`, the `INCLUDE` trailer, and an expression target. Co-authored-by: Claude Opus 4.8 --- src/ast/ddl.rs | 10 ++--- src/parser/mod.rs | 56 +++++++++--------------- tests/sqlparser_bigquery.rs | 77 --------------------------------- tests/sqlparser_common.rs | 86 +++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 117 deletions(-) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index 0709f63ebe..e201d9a4d7 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -2827,7 +2827,7 @@ pub struct CreateIndex { pub using: Option, /// columns included in the index pub columns: Vec, - /// whether this is a BigQuery `CREATE VECTOR INDEX` + /// whether this is a `CREATE VECTOR INDEX` pub vector: bool, /// whether the statement is `CREATE OR REPLACE` pub or_replace: bool, @@ -2847,7 +2847,7 @@ pub struct CreateIndex { pub nulls_distinct: Option, /// WITH clause: pub with: Vec, - /// BigQuery `OPTIONS(...)` clause, e.g. on `CREATE VECTOR INDEX` + /// `OPTIONS(...)` clause, e.g. on `CREATE VECTOR INDEX` pub options: Vec, /// WHERE clause: pub predicate: Option, @@ -2890,9 +2890,6 @@ impl fmt::Display for CreateIndex { write!(f, " USING {value} ")?; } write!(f, "({})", display_comma_separated(&self.columns))?; - if !self.options.is_empty() { - write!(f, " OPTIONS({})", display_comma_separated(&self.options))?; - } if !self.include.is_empty() { write!(f, " INCLUDE ({})", display_comma_separated(&self.include))?; } @@ -2906,6 +2903,9 @@ impl fmt::Display for CreateIndex { if !self.with.is_empty() { write!(f, " WITH ({})", display_comma_separated(&self.with))?; } + if !self.options.is_empty() { + write!(f, " OPTIONS({})", display_comma_separated(&self.options))?; + } if let Some(predicate) = &self.predicate { write!(f, " WHERE {predicate}")?; } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 530fd2534f..a713486ff9 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -5292,9 +5292,11 @@ impl<'a> Parser<'a> { &self.peek_token_ref().token, Token::Word(w) if w.keyword == Keyword::NoKeyword && w.value.eq_ignore_ascii_case("VECTOR") ) { - // BigQuery `CREATE [OR REPLACE] VECTOR INDEX ...`; VECTOR is not a keyword. + // `CREATE [OR REPLACE] VECTOR INDEX ...`; VECTOR is not a keyword. self.next_token(); - self.parse_create_vector_index(or_replace).map(Into::into) + self.expect_keyword_is(Keyword::INDEX)?; + self.parse_create_index_inner(false, true, or_replace) + .map(Into::into) } else if or_replace { self.expected_ref( "[EXTERNAL] TABLE or [MATERIALIZED] VIEW or FUNCTION or SCHEMA or WAREHOUSE after CREATE OR REPLACE", @@ -8252,38 +8254,19 @@ impl<'a> Parser<'a> { Ok(Statement::Discard { object_type }) } - /// Parse a BigQuery `CREATE [OR REPLACE] VECTOR INDEX` statement (`VECTOR` already consumed). - fn parse_create_vector_index(&mut self, or_replace: bool) -> Result { - self.expect_keyword_is(Keyword::INDEX)?; - let if_not_exists = self.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]); - let name = self.parse_object_name(false)?; - self.expect_keyword_is(Keyword::ON)?; - let table_name = self.parse_object_name(false)?; - let columns = self.parse_parenthesized_index_column_list()?; - let options = self.parse_options(Keyword::OPTIONS)?; - Ok(CreateIndex { - name: Some(name), - table_name, - using: None, - columns, - vector: true, - or_replace, - unique: false, - concurrently: false, - r#async: false, - if_not_exists, - include: vec![], - nulls_distinct: None, - with: vec![], - options, - predicate: None, - index_options: vec![], - alter_options: vec![], - }) - } - /// Parse a `CREATE INDEX` statement. pub fn parse_create_index(&mut self, unique: bool) -> Result { + self.parse_create_index_inner(unique, false, false) + } + + /// Parse the body of a `CREATE [UNIQUE | VECTOR] INDEX` statement, with the + /// leading `[UNIQUE | VECTOR] INDEX` keyword(s) already consumed. + fn parse_create_index_inner( + &mut self, + unique: bool, + vector: bool, + or_replace: bool, + ) -> Result { let concurrently = self.parse_keyword(Keyword::CONCURRENTLY); let r#async = self.parse_keyword(Keyword::ASYNC); let if_not_exists = self.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]); @@ -8336,6 +8319,9 @@ impl<'a> Parser<'a> { Vec::new() }; + // `OPTIONS(...)` clause (e.g. `CREATE VECTOR INDEX`); no-op when absent. + let options = self.parse_options(Keyword::OPTIONS)?; + let predicate = if self.parse_keyword(Keyword::WHERE) { Some(self.parse_expr()?) } else { @@ -8363,8 +8349,8 @@ impl<'a> Parser<'a> { table_name, using, columns, - vector: false, - or_replace: false, + vector, + or_replace, unique, concurrently, r#async, @@ -8372,7 +8358,7 @@ impl<'a> Parser<'a> { include, nulls_distinct, with, - options: vec![], + options, predicate, index_options, alter_options, diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index e1c10d943e..f6d4483c24 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -2950,80 +2950,3 @@ fn test_create_snapshot_table() { "CREATE SNAPSHOT TABLE IF NOT EXISTS dataset_id.table1 CLONE dataset_id.table2 FOR SYSTEM_TIME AS OF TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR) OPTIONS(expiration_timestamp = TIMESTAMP '2025-01-01 00:00:00 UTC')", ); } -#[test] -fn parse_bigquery_create_vector_index() { - // `CREATE VECTOR INDEX ... OPTIONS(...)` is a CreateIndex flagged `vector`. - let sql = - "CREATE VECTOR INDEX emb ON t(embedding) OPTIONS(distance_type = 'COSINE', dimension = 4)"; - match bigquery().verified_stmt(sql) { - Statement::CreateIndex(CreateIndex { - name, - table_name, - using, - columns, - vector, - or_replace, - unique, - if_not_exists, - with, - options, - .. - }) => { - assert!(vector); - assert!(!or_replace); - assert!(!unique); - assert!(!if_not_exists); - assert_eq!(name.unwrap().to_string(), "emb"); - assert_eq!(table_name.to_string(), "t"); - assert_eq!(using, None); - assert!(with.is_empty()); - assert_eq!( - columns.iter().map(|c| c.to_string()).collect::>(), - vec!["embedding"] - ); - assert_eq!( - options, - vec![ - SqlOption::KeyValue { - key: Ident::new("distance_type"), - value: Expr::Value( - Value::SingleQuotedString("COSINE".to_string()).with_empty_span() - ), - }, - SqlOption::KeyValue { - key: Ident::new("dimension"), - value: Expr::value(number("4")), - }, - ] - ); - } - other => panic!("expected CreateIndex, got {other:?}"), - } - - // OR REPLACE, IF NOT EXISTS, multi-part names, and the OPTIONS-less form all round-trip. - match bigquery().verified_stmt( - "CREATE OR REPLACE VECTOR INDEX emb ON t(embedding) OPTIONS(distance_type = 'COSINE')", - ) { - Statement::CreateIndex(CreateIndex { - vector, or_replace, .. - }) => { - assert!(vector); - assert!(or_replace); - } - other => panic!("expected CreateIndex, got {other:?}"), - } - match bigquery().verified_stmt( - "CREATE VECTOR INDEX IF NOT EXISTS mydataset.emb ON mydataset.t(embedding) OPTIONS(distance_type = 'EUCLIDEAN')", - ) { - Statement::CreateIndex(CreateIndex { - vector, - if_not_exists, - .. - }) => { - assert!(vector); - assert!(if_not_exists); - } - other => panic!("expected CreateIndex, got {other:?}"), - } - bigquery().verified_stmt("CREATE VECTOR INDEX emb ON t(embedding)"); -} diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 99e1304839..4fc90ef44c 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -9814,6 +9814,92 @@ fn parse_create_index_async() { verified_stmt("CREATE UNIQUE INDEX ASYNC my_index ON my_table(col1)"); } +#[test] +fn parse_create_vector_index() { + // `CREATE VECTOR INDEX` parses for every dialect as a `CreateIndex` flagged + // `vector`; the `OPTIONS(...)` trailer lands in `options`. + let sql = + "CREATE VECTOR INDEX emb ON t(embedding) OPTIONS(distance_type = 'COSINE', dimension = 4)"; + match verified_stmt(sql) { + Statement::CreateIndex(CreateIndex { + name, + table_name, + using, + columns, + vector, + or_replace, + unique, + if_not_exists, + with, + options, + .. + }) => { + assert!(vector); + assert!(!or_replace); + assert!(!unique); + assert!(!if_not_exists); + assert_eq!(name.unwrap().to_string(), "emb"); + assert_eq!(table_name.to_string(), "t"); + assert_eq!(using, None); + assert!(with.is_empty()); + assert_eq!( + columns.iter().map(|c| c.to_string()).collect::>(), + vec!["embedding"] + ); + assert_eq!( + options, + vec![ + SqlOption::KeyValue { + key: Ident::new("distance_type"), + value: Expr::Value( + Value::SingleQuotedString("COSINE".to_string()).with_empty_span() + ), + }, + SqlOption::KeyValue { + key: Ident::new("dimension"), + value: Expr::value(number("4")), + }, + ] + ); + } + other => panic!("expected CreateIndex, got {other:?}"), + } + + // `OR REPLACE`. + match verified_stmt( + "CREATE OR REPLACE VECTOR INDEX emb ON t(embedding) OPTIONS(distance_type = 'COSINE')", + ) { + Statement::CreateIndex(CreateIndex { + vector, or_replace, .. + }) => { + assert!(vector); + assert!(or_replace); + } + other => panic!("expected CreateIndex, got {other:?}"), + } + + // `IF NOT EXISTS` and schema-qualified names. + match verified_stmt( + "CREATE VECTOR INDEX IF NOT EXISTS s.emb ON s.t(embedding) OPTIONS(distance_type = 'EUCLIDEAN')", + ) { + Statement::CreateIndex(CreateIndex { + vector, + if_not_exists, + .. + }) => { + assert!(vector); + assert!(if_not_exists); + } + other => panic!("expected CreateIndex, got {other:?}"), + } + + // The bare core, an `INCLUDE` covering-column trailer, and an expression + // target all round-trip. + verified_stmt("CREATE VECTOR INDEX emb ON t(embedding)"); + verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) INCLUDE (a, b)"); + verified_stmt("CREATE VECTOR INDEX emb ON t(VEC_COSINE_DISTANCE(embedding))"); +} + #[test] fn parse_drop_index() { let sql = "DROP INDEX idx_a"; From dd1de78460101e19a102cc7712da9b574b7d5678 Mon Sep 17 00:00:00 2001 From: Mosha Pasumansky Date: Tue, 18 Aug 2026 17:26:49 -0700 Subject: [PATCH 3/3] Cover more `CREATE VECTOR INDEX` dialect variants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the shared `CREATE VECTOR INDEX` parsing to the trailers used by the other dialects, and add per-dialect tests: - `STORING(...)` covering-column clause (BigQuery); new `storing` field on `CreateIndex`, rendered after `INCLUDE`. - Accept a `WITH (...)` options clause on a vector index in every dialect (SQL Server `WITH (METRIC = ..., TYPE = ..., MAXDOP = ...)`), not only the dialects that enable it for a plain `CREATE INDEX`. Tests: - `tests/sqlparser_common.rs` — the generic core plus the shared trailers (`INCLUDE`, `STORING`, `WITH`, `USING`, expression targets) across all dialects. - `tests/sqlparser_bigquery.rs` — `OPTIONS(...)` with index_type / distance_type / JSON tuning keys, and `STORING(...)`. - `tests/sqlparser_mssql.rs` — bracket-quoted names with `WITH (...)`. - `tests/sqlparser_mysql.rs` — TiDB's distance-function target with `USING`. - `tests/sqlparser_oracle.rs` — the core, an expression target and an `INCLUDE` list (Oracle's `ORGANIZATION` / `DISTANCE` / `WITH TARGET ACCURACY` / `PARAMETERS` clauses are not yet parsed). Co-authored-by: Claude Opus 4.8 --- src/ast/ddl.rs | 5 +++++ src/ast/spans.rs | 2 ++ src/keywords.rs | 1 + src/parser/mod.rs | 15 ++++++++++++++- tests/sqlparser_bigquery.rs | 12 ++++++++++++ tests/sqlparser_common.rs | 12 +++++++++--- tests/sqlparser_mssql.rs | 9 +++++++++ tests/sqlparser_mysql.rs | 9 +++++++++ tests/sqlparser_oracle.rs | 10 ++++++++++ tests/sqlparser_postgres.rs | 11 +++++++++++ 10 files changed, 82 insertions(+), 4 deletions(-) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index e201d9a4d7..e280f700b6 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -2843,6 +2843,8 @@ pub struct CreateIndex { pub if_not_exists: bool, /// INCLUDE clause: pub include: Vec, + /// `STORING(...)` clause (covering columns on a `CREATE VECTOR INDEX`) + pub storing: Vec, /// NULLS DISTINCT / NOT DISTINCT clause: pub nulls_distinct: Option, /// WITH clause: @@ -2893,6 +2895,9 @@ impl fmt::Display for CreateIndex { if !self.include.is_empty() { write!(f, " INCLUDE ({})", display_comma_separated(&self.include))?; } + if !self.storing.is_empty() { + write!(f, " STORING({})", display_comma_separated(&self.storing))?; + } if let Some(value) = self.nulls_distinct { if value { write!(f, " NULLS DISTINCT")?; diff --git a/src/ast/spans.rs b/src/ast/spans.rs index ab5f105c8f..da46786765 100644 --- a/src/ast/spans.rs +++ b/src/ast/spans.rs @@ -706,6 +706,7 @@ impl Spanned for CreateIndex { r#async: _, // bool if_not_exists: _, // bool include, + storing, nulls_distinct: _, // bool with, options, @@ -720,6 +721,7 @@ impl Spanned for CreateIndex { .chain(core::iter::once(table_name.span())) .chain(columns.iter().map(|i| i.column.span())) .chain(include.iter().map(|i| i.span)) + .chain(storing.iter().map(|i| i.span)) .chain(with.iter().map(|i| i.span())) .chain(options.iter().map(|i| i.span())) .chain(predicate.iter().map(|i| i.span())) diff --git a/src/keywords.rs b/src/keywords.rs index 0c50703c3f..605328d2b4 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -1008,6 +1008,7 @@ define_keywords!( STORAGE_INTEGRATION, STORAGE_SERIALIZATION_POLICY, STORED, + STORING, STRAIGHT_JOIN, STREAM, STRICT, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index a713486ff9..d78866c24e 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -8300,6 +8300,16 @@ impl<'a> Parser<'a> { vec![] }; + // `STORING(...)` covering columns (e.g. `CREATE VECTOR INDEX`). + let storing = if self.parse_keyword(Keyword::STORING) { + self.expect_token(&Token::LParen)?; + let columns = self.parse_comma_separated(|p| p.parse_identifier())?; + self.expect_token(&Token::RParen)?; + columns + } else { + vec![] + }; + let nulls_distinct = if self.parse_keyword(Keyword::NULLS) { let not = self.parse_keyword(Keyword::NOT); self.expect_keyword_is(Keyword::DISTINCT)?; @@ -8308,7 +8318,9 @@ impl<'a> Parser<'a> { None }; - let with = if self.dialect.supports_create_index_with_clause() + // A vector index accepts a `WITH (...)` options clause in every dialect + // (e.g. SQL Server `WITH (METRIC = ..., TYPE = ...)`). + let with = if (self.dialect.supports_create_index_with_clause() || vector) && self.parse_keyword(Keyword::WITH) { self.expect_token(&Token::LParen)?; @@ -8356,6 +8368,7 @@ impl<'a> Parser<'a> { r#async, if_not_exists, include, + storing, nulls_distinct, with, options, diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index f6d4483c24..4fbd9e1d3f 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -2950,3 +2950,15 @@ fn test_create_snapshot_table() { "CREATE SNAPSHOT TABLE IF NOT EXISTS dataset_id.table1 CLONE dataset_id.table2 FOR SYSTEM_TIME AS OF TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR) OPTIONS(expiration_timestamp = TIMESTAMP '2025-01-01 00:00:00 UTC')", ); } + +#[test] +fn parse_bigquery_create_vector_index() { + // BigQuery's form: an `OPTIONS(...)` clause with its index_type / distance_type + // / JSON tuning keys, and a `STORING(...)` covering-column list. + bigquery().verified_stmt( + "CREATE VECTOR INDEX my_index ON my_dataset.my_table(embedding) OPTIONS(index_type = 'IVF', distance_type = 'COSINE', ivf_options = '{\"num_lists\": 2500}')", + ); + bigquery().verified_stmt( + "CREATE OR REPLACE VECTOR INDEX my_index ON my_dataset.my_table(embedding) STORING(type, creation_time) OPTIONS(index_type = 'TREE_AH', distance_type = 'EUCLIDEAN')", + ); +} diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 4fc90ef44c..48bc99f888 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -9731,6 +9731,7 @@ fn test_create_index_with_using_function() { vector: _, or_replace: _, options: _, + storing: _, }) => { assert_eq!("idx_name", name.to_string()); assert_eq!("test", table_name.to_string()); @@ -9791,6 +9792,7 @@ fn test_create_index_with_with_clause() { vector: _, or_replace: _, options: _, + storing: _, }) => { pretty_assertions::assert_eq!("title_idx", name.to_string()); pretty_assertions::assert_eq!("films", table_name.to_string()); @@ -9893,11 +9895,15 @@ fn parse_create_vector_index() { other => panic!("expected CreateIndex, got {other:?}"), } - // The bare core, an `INCLUDE` covering-column trailer, and an expression - // target all round-trip. + // The bare core plus the shared trailers all round-trip across dialects: an + // expression target, `INCLUDE` / `STORING` covering columns, a `WITH` + // options clause, and a trailing `USING `. verified_stmt("CREATE VECTOR INDEX emb ON t(embedding)"); - verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) INCLUDE (a, b)"); verified_stmt("CREATE VECTOR INDEX emb ON t(VEC_COSINE_DISTANCE(embedding))"); + verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) INCLUDE (a, b)"); + verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) STORING(a, b)"); + verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) WITH (metric = 'cosine')"); + verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) USING HNSW"); } #[test] diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 3faf56f0d9..e56fd89409 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -2925,3 +2925,12 @@ fn parse_mssql_money_constants() { expr_from_projection(only(&select.projection)), ); } + +#[test] +fn parse_mssql_create_vector_index() { + // SQL Server's form: bracket-quoted names and a `WITH (...)` options clause + // (`METRIC` / `TYPE` / `MAXDOP`). + ms().verified_stmt( + "CREATE VECTOR INDEX vec_idx ON [dbo].[articles]([title_vector]) WITH (METRIC = 'cosine', TYPE = 'DiskANN', MAXDOP = 8)", + ); +} diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index 797a12551b..d2f8d40509 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -4946,3 +4946,12 @@ fn parse_adjacent_string_literal_concatenation() { fn parse_group_by_with_rollup() { mysql().verified_stmt("SELECT * FROM tbl GROUP BY col1, col2 WITH ROLLUP"); } + +#[test] +fn parse_mysql_create_vector_index() { + // TiDB's form: the vector column is wrapped in a distance function and the + // algorithm is named with a trailing `USING`. + mysql().verified_stmt( + "CREATE VECTOR INDEX idx_cos ON tidb_vectors((VEC_COSINE_DISTANCE(embedding))) USING HNSW", + ); +} diff --git a/tests/sqlparser_oracle.rs b/tests/sqlparser_oracle.rs index 888778e235..c8438f6675 100644 --- a/tests/sqlparser_oracle.rs +++ b/tests/sqlparser_oracle.rs @@ -539,3 +539,13 @@ fn test_insert_without_alias() { if matches!(&*source, Query { body, .. } if matches!(&**body, SetExpr::Values(_))) )); } + +#[test] +fn parse_oracle_create_vector_index() { + // Oracle's specialized clauses (ORGANIZATION / DISTANCE / WITH TARGET + // ACCURACY / PARAMETERS) are not yet parsed; the forms it shares with the + // common grammar — an expression target and an `INCLUDE` list — round-trip. + oracle().verified_stmt("CREATE VECTOR INDEX g_idx ON galaxies(embedding)"); + oracle().verified_stmt("CREATE VECTOR INDEX g_idx ON galaxies(VEC_DISTANCE(embedding))"); + oracle().verified_stmt("CREATE VECTOR INDEX g_idx ON galaxies(embedding) INCLUDE (id)"); +} diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 9c12dc8af7..e033b4ce67 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -2996,6 +2996,7 @@ fn parse_create_index() { vector: _, or_replace: _, options: _, + storing: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3036,6 +3037,7 @@ fn parse_create_anonymous_index() { vector: _, or_replace: _, options: _, + storing: _, }) => { assert_eq!(None, name); assert_eq_vec(&["my_table"], &table_name); @@ -3159,6 +3161,7 @@ fn parse_create_indices_with_operator_classes() { vector: _, or_replace: _, options: _, + storing: _, }) => { assert_eq_vec(&["the_index_name"], &name); assert_eq_vec(&["users"], &table_name); @@ -3191,6 +3194,7 @@ fn parse_create_indices_with_operator_classes() { vector: _, or_replace: _, options: _, + storing: _, }) => { assert_eq_vec(&["the_index_name"], &name); assert_eq_vec(&["users"], &table_name); @@ -3278,6 +3282,7 @@ fn parse_create_bloom() { vector: _, or_replace: _, options: _, + storing: _, }) => { assert_eq_vec(&["bloomidx"], &name); assert_eq_vec(&["tbloom"], &table_name); @@ -3338,6 +3343,7 @@ fn parse_create_brin() { vector: _, or_replace: _, options: _, + storing: _, }) => { assert_eq_vec(&["brin_sensor_data_recorded_at"], &name); assert_eq_vec(&["sensor_data"], &table_name); @@ -3409,6 +3415,7 @@ fn parse_create_index_concurrently() { vector: _, or_replace: _, options: _, + storing: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3449,6 +3456,7 @@ fn parse_create_index_with_predicate() { vector: _, or_replace: _, options: _, + storing: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3489,6 +3497,7 @@ fn parse_create_index_with_include() { vector: _, or_replace: _, options: _, + storing: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3529,6 +3538,7 @@ fn parse_create_index_with_nulls_distinct() { vector: _, or_replace: _, options: _, + storing: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3567,6 +3577,7 @@ fn parse_create_index_with_nulls_distinct() { vector: _, or_replace: _, options: _, + storing: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name);