diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index a0e69ad8a..e280f700b 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -2827,6 +2827,10 @@ pub struct CreateIndex { pub using: Option, /// columns included in the index pub columns: Vec, + /// whether this is a `CREATE VECTOR INDEX` + pub vector: bool, + /// whether the statement is `CREATE OR REPLACE` + pub or_replace: bool, /// whether the index is unique pub unique: bool, /// whether the index is created concurrently @@ -2839,10 +2843,14 @@ pub struct CreateIndex { pub if_not_exists: bool, /// INCLUDE clause: pub include: Vec, + /// `STORING(...)` clause (covering columns on a `CREATE VECTOR INDEX`) + pub storing: Vec, /// NULLS DISTINCT / NOT DISTINCT clause: pub nulls_distinct: Option, /// WITH clause: pub with: Vec, + /// `OPTIONS(...)` clause, e.g. on `CREATE VECTOR INDEX` + pub options: Vec, /// WHERE clause: pub predicate: Option, /// Index options: @@ -2860,8 +2868,10 @@ impl fmt::Display for CreateIndex { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!( f, - "CREATE {unique}INDEX {concurrently}{async_}{if_not_exists}", + "CREATE {or_replace}{unique}{vector}INDEX {concurrently}{async_}{if_not_exists}", + or_replace = if self.or_replace { "OR REPLACE " } else { "" }, unique = if self.unique { "UNIQUE " } else { "" }, + vector = if self.vector { "VECTOR " } else { "" }, concurrently = if self.concurrently { "CONCURRENTLY " } else { @@ -2885,6 +2895,9 @@ impl fmt::Display for CreateIndex { if !self.include.is_empty() { write!(f, " INCLUDE ({})", display_comma_separated(&self.include))?; } + if !self.storing.is_empty() { + write!(f, " STORING({})", display_comma_separated(&self.storing))?; + } if let Some(value) = self.nulls_distinct { if value { write!(f, " NULLS DISTINCT")?; @@ -2895,6 +2908,9 @@ impl fmt::Display for CreateIndex { if !self.with.is_empty() { write!(f, " WITH ({})", display_comma_separated(&self.with))?; } + if !self.options.is_empty() { + write!(f, " OPTIONS({})", display_comma_separated(&self.options))?; + } if let Some(predicate) = &self.predicate { write!(f, " WHERE {predicate}")?; } diff --git a/src/ast/spans.rs b/src/ast/spans.rs index a34fe66d9..da4678676 100644 --- a/src/ast/spans.rs +++ b/src/ast/spans.rs @@ -699,13 +699,17 @@ impl Spanned for CreateIndex { table_name, using: _, columns, + vector: _, // bool + or_replace: _, // bool unique: _, // bool concurrently: _, // bool r#async: _, // bool if_not_exists: _, // bool include, + storing, nulls_distinct: _, // bool with, + options, predicate, index_options: _, alter_options, @@ -717,7 +721,9 @@ impl Spanned for CreateIndex { .chain(core::iter::once(table_name.span())) .chain(columns.iter().map(|i| i.column.span())) .chain(include.iter().map(|i| i.span)) + .chain(storing.iter().map(|i| i.span)) .chain(with.iter().map(|i| i.span())) + .chain(options.iter().map(|i| i.span())) .chain(predicate.iter().map(|i| i.span())) .chain(alter_options.iter().map(|i| i.span())), ) diff --git a/src/keywords.rs b/src/keywords.rs index 0c50703c3..605328d2b 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -1008,6 +1008,7 @@ define_keywords!( STORAGE_INTEGRATION, STORAGE_SERIALIZATION_POLICY, STORED, + STORING, STRAIGHT_JOIN, STREAM, STRICT, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index b2b3f42bb..d78866c24 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -5288,6 +5288,15 @@ impl<'a> Parser<'a> { self.parse_create_schema(or_replace) } else if self.parse_keyword(Keyword::WAREHOUSE) { self.parse_create_warehouse(or_replace).map(Into::into) + } else if matches!( + &self.peek_token_ref().token, + Token::Word(w) if w.keyword == Keyword::NoKeyword && w.value.eq_ignore_ascii_case("VECTOR") + ) { + // `CREATE [OR REPLACE] VECTOR INDEX ...`; VECTOR is not a keyword. + self.next_token(); + self.expect_keyword_is(Keyword::INDEX)?; + self.parse_create_index_inner(false, true, or_replace) + .map(Into::into) } else if or_replace { self.expected_ref( "[EXTERNAL] TABLE or [MATERIALIZED] VIEW or FUNCTION or SCHEMA or WAREHOUSE after CREATE OR REPLACE", @@ -8247,6 +8256,17 @@ impl<'a> Parser<'a> { /// Parse a `CREATE INDEX` statement. pub fn parse_create_index(&mut self, unique: bool) -> Result { + self.parse_create_index_inner(unique, false, false) + } + + /// Parse the body of a `CREATE [UNIQUE | VECTOR] INDEX` statement, with the + /// leading `[UNIQUE | VECTOR] INDEX` keyword(s) already consumed. + fn parse_create_index_inner( + &mut self, + unique: bool, + vector: bool, + or_replace: bool, + ) -> Result { let concurrently = self.parse_keyword(Keyword::CONCURRENTLY); let r#async = self.parse_keyword(Keyword::ASYNC); let if_not_exists = self.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]); @@ -8280,6 +8300,16 @@ impl<'a> Parser<'a> { vec![] }; + // `STORING(...)` covering columns (e.g. `CREATE VECTOR INDEX`). + let storing = if self.parse_keyword(Keyword::STORING) { + self.expect_token(&Token::LParen)?; + let columns = self.parse_comma_separated(|p| p.parse_identifier())?; + self.expect_token(&Token::RParen)?; + columns + } else { + vec![] + }; + let nulls_distinct = if self.parse_keyword(Keyword::NULLS) { let not = self.parse_keyword(Keyword::NOT); self.expect_keyword_is(Keyword::DISTINCT)?; @@ -8288,7 +8318,9 @@ impl<'a> Parser<'a> { None }; - let with = if self.dialect.supports_create_index_with_clause() + // A vector index accepts a `WITH (...)` options clause in every dialect + // (e.g. SQL Server `WITH (METRIC = ..., TYPE = ...)`). + let with = if (self.dialect.supports_create_index_with_clause() || vector) && self.parse_keyword(Keyword::WITH) { self.expect_token(&Token::LParen)?; @@ -8299,6 +8331,9 @@ impl<'a> Parser<'a> { Vec::new() }; + // `OPTIONS(...)` clause (e.g. `CREATE VECTOR INDEX`); no-op when absent. + let options = self.parse_options(Keyword::OPTIONS)?; + let predicate = if self.parse_keyword(Keyword::WHERE) { Some(self.parse_expr()?) } else { @@ -8326,13 +8361,17 @@ impl<'a> Parser<'a> { table_name, using, columns, + vector, + or_replace, unique, concurrently, r#async, if_not_exists, include, + storing, nulls_distinct, with, + options, predicate, index_options, alter_options, diff --git a/tests/sqlparser_bigquery.rs b/tests/sqlparser_bigquery.rs index f6d4483c2..4fbd9e1d3 100644 --- a/tests/sqlparser_bigquery.rs +++ b/tests/sqlparser_bigquery.rs @@ -2950,3 +2950,15 @@ fn test_create_snapshot_table() { "CREATE SNAPSHOT TABLE IF NOT EXISTS dataset_id.table1 CLONE dataset_id.table2 FOR SYSTEM_TIME AS OF TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR) OPTIONS(expiration_timestamp = TIMESTAMP '2025-01-01 00:00:00 UTC')", ); } + +#[test] +fn parse_bigquery_create_vector_index() { + // BigQuery's form: an `OPTIONS(...)` clause with its index_type / distance_type + // / JSON tuning keys, and a `STORING(...)` covering-column list. + bigquery().verified_stmt( + "CREATE VECTOR INDEX my_index ON my_dataset.my_table(embedding) OPTIONS(index_type = 'IVF', distance_type = 'COSINE', ivf_options = '{\"num_lists\": 2500}')", + ); + bigquery().verified_stmt( + "CREATE OR REPLACE VECTOR INDEX my_index ON my_dataset.my_table(embedding) STORING(type, creation_time) OPTIONS(index_type = 'TREE_AH', distance_type = 'EUCLIDEAN')", + ); +} diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 0800bc41f..48bc99f88 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -9728,6 +9728,10 @@ fn test_create_index_with_using_function() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, + storing: _, }) => { assert_eq!("idx_name", name.to_string()); assert_eq!("test", table_name.to_string()); @@ -9785,6 +9789,10 @@ fn test_create_index_with_with_clause() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, + storing: _, }) => { pretty_assertions::assert_eq!("title_idx", name.to_string()); pretty_assertions::assert_eq!("films", table_name.to_string()); @@ -9808,6 +9816,96 @@ fn parse_create_index_async() { verified_stmt("CREATE UNIQUE INDEX ASYNC my_index ON my_table(col1)"); } +#[test] +fn parse_create_vector_index() { + // `CREATE VECTOR INDEX` parses for every dialect as a `CreateIndex` flagged + // `vector`; the `OPTIONS(...)` trailer lands in `options`. + let sql = + "CREATE VECTOR INDEX emb ON t(embedding) OPTIONS(distance_type = 'COSINE', dimension = 4)"; + match verified_stmt(sql) { + Statement::CreateIndex(CreateIndex { + name, + table_name, + using, + columns, + vector, + or_replace, + unique, + if_not_exists, + with, + options, + .. + }) => { + assert!(vector); + assert!(!or_replace); + assert!(!unique); + assert!(!if_not_exists); + assert_eq!(name.unwrap().to_string(), "emb"); + assert_eq!(table_name.to_string(), "t"); + assert_eq!(using, None); + assert!(with.is_empty()); + assert_eq!( + columns.iter().map(|c| c.to_string()).collect::>(), + vec!["embedding"] + ); + assert_eq!( + options, + vec![ + SqlOption::KeyValue { + key: Ident::new("distance_type"), + value: Expr::Value( + Value::SingleQuotedString("COSINE".to_string()).with_empty_span() + ), + }, + SqlOption::KeyValue { + key: Ident::new("dimension"), + value: Expr::value(number("4")), + }, + ] + ); + } + other => panic!("expected CreateIndex, got {other:?}"), + } + + // `OR REPLACE`. + match verified_stmt( + "CREATE OR REPLACE VECTOR INDEX emb ON t(embedding) OPTIONS(distance_type = 'COSINE')", + ) { + Statement::CreateIndex(CreateIndex { + vector, or_replace, .. + }) => { + assert!(vector); + assert!(or_replace); + } + other => panic!("expected CreateIndex, got {other:?}"), + } + + // `IF NOT EXISTS` and schema-qualified names. + match verified_stmt( + "CREATE VECTOR INDEX IF NOT EXISTS s.emb ON s.t(embedding) OPTIONS(distance_type = 'EUCLIDEAN')", + ) { + Statement::CreateIndex(CreateIndex { + vector, + if_not_exists, + .. + }) => { + assert!(vector); + assert!(if_not_exists); + } + other => panic!("expected CreateIndex, got {other:?}"), + } + + // The bare core plus the shared trailers all round-trip across dialects: an + // expression target, `INCLUDE` / `STORING` covering columns, a `WITH` + // options clause, and a trailing `USING `. + verified_stmt("CREATE VECTOR INDEX emb ON t(embedding)"); + verified_stmt("CREATE VECTOR INDEX emb ON t(VEC_COSINE_DISTANCE(embedding))"); + verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) INCLUDE (a, b)"); + verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) STORING(a, b)"); + verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) WITH (metric = 'cosine')"); + verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) USING HNSW"); +} + #[test] fn parse_drop_index() { let sql = "DROP INDEX idx_a"; diff --git a/tests/sqlparser_mssql.rs b/tests/sqlparser_mssql.rs index 3faf56f0d..e56fd8940 100644 --- a/tests/sqlparser_mssql.rs +++ b/tests/sqlparser_mssql.rs @@ -2925,3 +2925,12 @@ fn parse_mssql_money_constants() { expr_from_projection(only(&select.projection)), ); } + +#[test] +fn parse_mssql_create_vector_index() { + // SQL Server's form: bracket-quoted names and a `WITH (...)` options clause + // (`METRIC` / `TYPE` / `MAXDOP`). + ms().verified_stmt( + "CREATE VECTOR INDEX vec_idx ON [dbo].[articles]([title_vector]) WITH (METRIC = 'cosine', TYPE = 'DiskANN', MAXDOP = 8)", + ); +} diff --git a/tests/sqlparser_mysql.rs b/tests/sqlparser_mysql.rs index 797a12551..d2f8d4050 100644 --- a/tests/sqlparser_mysql.rs +++ b/tests/sqlparser_mysql.rs @@ -4946,3 +4946,12 @@ fn parse_adjacent_string_literal_concatenation() { fn parse_group_by_with_rollup() { mysql().verified_stmt("SELECT * FROM tbl GROUP BY col1, col2 WITH ROLLUP"); } + +#[test] +fn parse_mysql_create_vector_index() { + // TiDB's form: the vector column is wrapped in a distance function and the + // algorithm is named with a trailing `USING`. + mysql().verified_stmt( + "CREATE VECTOR INDEX idx_cos ON tidb_vectors((VEC_COSINE_DISTANCE(embedding))) USING HNSW", + ); +} diff --git a/tests/sqlparser_oracle.rs b/tests/sqlparser_oracle.rs index 888778e23..c8438f667 100644 --- a/tests/sqlparser_oracle.rs +++ b/tests/sqlparser_oracle.rs @@ -539,3 +539,13 @@ fn test_insert_without_alias() { if matches!(&*source, Query { body, .. } if matches!(&**body, SetExpr::Values(_))) )); } + +#[test] +fn parse_oracle_create_vector_index() { + // Oracle's specialized clauses (ORGANIZATION / DISTANCE / WITH TARGET + // ACCURACY / PARAMETERS) are not yet parsed; the forms it shares with the + // common grammar — an expression target and an `INCLUDE` list — round-trip. + oracle().verified_stmt("CREATE VECTOR INDEX g_idx ON galaxies(embedding)"); + oracle().verified_stmt("CREATE VECTOR INDEX g_idx ON galaxies(VEC_DISTANCE(embedding))"); + oracle().verified_stmt("CREATE VECTOR INDEX g_idx ON galaxies(embedding) INCLUDE (id)"); +} diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index a7128eafd..e033b4ce6 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -2993,6 +2993,10 @@ fn parse_create_index() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, + storing: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3030,6 +3034,10 @@ fn parse_create_anonymous_index() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, + storing: _, }) => { assert_eq!(None, name); assert_eq_vec(&["my_table"], &table_name); @@ -3150,6 +3158,10 @@ fn parse_create_indices_with_operator_classes() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, + storing: _, }) => { assert_eq_vec(&["the_index_name"], &name); assert_eq_vec(&["users"], &table_name); @@ -3179,6 +3191,10 @@ fn parse_create_indices_with_operator_classes() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, + storing: _, }) => { assert_eq_vec(&["the_index_name"], &name); assert_eq_vec(&["users"], &table_name); @@ -3263,6 +3279,10 @@ fn parse_create_bloom() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, + storing: _, }) => { assert_eq_vec(&["bloomidx"], &name); assert_eq_vec(&["tbloom"], &table_name); @@ -3320,6 +3340,10 @@ fn parse_create_brin() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, + storing: _, }) => { assert_eq_vec(&["brin_sensor_data_recorded_at"], &name); assert_eq_vec(&["sensor_data"], &table_name); @@ -3388,6 +3412,10 @@ fn parse_create_index_concurrently() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, + storing: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3425,6 +3453,10 @@ fn parse_create_index_with_predicate() { predicate: Some(_), index_options, alter_options, + vector: _, + or_replace: _, + options: _, + storing: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3462,6 +3494,10 @@ fn parse_create_index_with_include() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, + storing: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3499,6 +3535,10 @@ fn parse_create_index_with_nulls_distinct() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, + storing: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name); @@ -3534,6 +3574,10 @@ fn parse_create_index_with_nulls_distinct() { predicate: None, index_options, alter_options, + vector: _, + or_replace: _, + options: _, + storing: _, }) => { assert_eq_vec(&["my_index"], &name); assert_eq_vec(&["my_table"], &table_name);