Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion src/ast/ddl.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2827,6 +2827,10 @@ pub struct CreateIndex {
pub using: Option<IndexType>,
/// columns included in the index
pub columns: Vec<IndexColumn>,
/// whether this is a `CREATE VECTOR INDEX`
pub vector: bool,
/// whether the statement is `CREATE OR REPLACE`
pub or_replace: bool,
/// whether the index is unique
pub unique: bool,
/// whether the index is created concurrently
Expand All @@ -2839,10 +2843,14 @@ pub struct CreateIndex {
pub if_not_exists: bool,
/// INCLUDE clause: <https://www.postgresql.org/docs/current/sql-createindex.html>
pub include: Vec<Ident>,
/// `STORING(...)` clause (covering columns on a `CREATE VECTOR INDEX`)
pub storing: Vec<Ident>,
/// NULLS DISTINCT / NOT DISTINCT clause: <https://www.postgresql.org/docs/current/sql-createindex.html>
pub nulls_distinct: Option<bool>,
/// WITH clause: <https://www.postgresql.org/docs/current/sql-createindex.html>
pub with: Vec<Expr>,
/// `OPTIONS(...)` clause, e.g. on `CREATE VECTOR INDEX`
pub options: Vec<SqlOption>,
/// WHERE clause: <https://www.postgresql.org/docs/current/sql-createindex.html>
pub predicate: Option<Expr>,
/// Index options: <https://www.postgresql.org/docs/current/sql-createindex.html>
Expand All @@ -2860,8 +2868,10 @@ impl fmt::Display for CreateIndex {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(
f,
"CREATE {unique}INDEX {concurrently}{async_}{if_not_exists}",
"CREATE {or_replace}{unique}{vector}INDEX {concurrently}{async_}{if_not_exists}",
or_replace = if self.or_replace { "OR REPLACE " } else { "" },
unique = if self.unique { "UNIQUE " } else { "" },
vector = if self.vector { "VECTOR " } else { "" },
concurrently = if self.concurrently {
"CONCURRENTLY "
} else {
Expand All @@ -2885,6 +2895,9 @@ impl fmt::Display for CreateIndex {
if !self.include.is_empty() {
write!(f, " INCLUDE ({})", display_comma_separated(&self.include))?;
}
if !self.storing.is_empty() {
write!(f, " STORING({})", display_comma_separated(&self.storing))?;
}
if let Some(value) = self.nulls_distinct {
if value {
write!(f, " NULLS DISTINCT")?;
Expand All @@ -2895,6 +2908,9 @@ impl fmt::Display for CreateIndex {
if !self.with.is_empty() {
write!(f, " WITH ({})", display_comma_separated(&self.with))?;
}
if !self.options.is_empty() {
write!(f, " OPTIONS({})", display_comma_separated(&self.options))?;
}
if let Some(predicate) = &self.predicate {
write!(f, " WHERE {predicate}")?;
}
Expand Down
6 changes: 6 additions & 0 deletions src/ast/spans.rs
Original file line number Diff line number Diff line change
Expand Up @@ -699,13 +699,17 @@ impl Spanned for CreateIndex {
table_name,
using: _,
columns,
vector: _, // bool
or_replace: _, // bool
unique: _, // bool
concurrently: _, // bool
r#async: _, // bool
if_not_exists: _, // bool
include,
storing,
nulls_distinct: _, // bool
with,
options,
predicate,
index_options: _,
alter_options,
Expand All @@ -717,7 +721,9 @@ impl Spanned for CreateIndex {
.chain(core::iter::once(table_name.span()))
.chain(columns.iter().map(|i| i.column.span()))
.chain(include.iter().map(|i| i.span))
.chain(storing.iter().map(|i| i.span))
.chain(with.iter().map(|i| i.span()))
.chain(options.iter().map(|i| i.span()))
.chain(predicate.iter().map(|i| i.span()))
.chain(alter_options.iter().map(|i| i.span())),
)
Expand Down
1 change: 1 addition & 0 deletions src/keywords.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1008,6 +1008,7 @@ define_keywords!(
STORAGE_INTEGRATION,
STORAGE_SERIALIZATION_POLICY,
STORED,
STORING,
STRAIGHT_JOIN,
STREAM,
STRICT,
Expand Down
41 changes: 40 additions & 1 deletion src/parser/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5288,6 +5288,15 @@ impl<'a> Parser<'a> {
self.parse_create_schema(or_replace)
} else if self.parse_keyword(Keyword::WAREHOUSE) {
self.parse_create_warehouse(or_replace).map(Into::into)
} else if matches!(
&self.peek_token_ref().token,
Token::Word(w) if w.keyword == Keyword::NoKeyword && w.value.eq_ignore_ascii_case("VECTOR")
) {
// `CREATE [OR REPLACE] VECTOR INDEX ...`; VECTOR is not a keyword.
self.next_token();
self.expect_keyword_is(Keyword::INDEX)?;
self.parse_create_index_inner(false, true, or_replace)
.map(Into::into)
} else if or_replace {
self.expected_ref(
"[EXTERNAL] TABLE or [MATERIALIZED] VIEW or FUNCTION or SCHEMA or WAREHOUSE after CREATE OR REPLACE",
Expand Down Expand Up @@ -8247,6 +8256,17 @@ impl<'a> Parser<'a> {

/// Parse a `CREATE INDEX` statement.
pub fn parse_create_index(&mut self, unique: bool) -> Result<CreateIndex, ParserError> {
self.parse_create_index_inner(unique, false, false)
}

/// Parse the body of a `CREATE [UNIQUE | VECTOR] INDEX` statement, with the
/// leading `[UNIQUE | VECTOR] INDEX` keyword(s) already consumed.
fn parse_create_index_inner(
&mut self,
unique: bool,
vector: bool,
or_replace: bool,
) -> Result<CreateIndex, ParserError> {
let concurrently = self.parse_keyword(Keyword::CONCURRENTLY);
let r#async = self.parse_keyword(Keyword::ASYNC);
let if_not_exists = self.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]);
Expand Down Expand Up @@ -8280,6 +8300,16 @@ impl<'a> Parser<'a> {
vec![]
};

// `STORING(...)` covering columns (e.g. `CREATE VECTOR INDEX`).
let storing = if self.parse_keyword(Keyword::STORING) {
self.expect_token(&Token::LParen)?;
let columns = self.parse_comma_separated(|p| p.parse_identifier())?;
self.expect_token(&Token::RParen)?;
columns
} else {
vec![]
};

let nulls_distinct = if self.parse_keyword(Keyword::NULLS) {
let not = self.parse_keyword(Keyword::NOT);
self.expect_keyword_is(Keyword::DISTINCT)?;
Expand All @@ -8288,7 +8318,9 @@ impl<'a> Parser<'a> {
None
};

let with = if self.dialect.supports_create_index_with_clause()
// A vector index accepts a `WITH (...)` options clause in every dialect
// (e.g. SQL Server `WITH (METRIC = ..., TYPE = ...)`).
let with = if (self.dialect.supports_create_index_with_clause() || vector)
&& self.parse_keyword(Keyword::WITH)
{
self.expect_token(&Token::LParen)?;
Expand All @@ -8299,6 +8331,9 @@ impl<'a> Parser<'a> {
Vec::new()
};

// `OPTIONS(...)` clause (e.g. `CREATE VECTOR INDEX`); no-op when absent.
let options = self.parse_options(Keyword::OPTIONS)?;

let predicate = if self.parse_keyword(Keyword::WHERE) {
Some(self.parse_expr()?)
} else {
Expand Down Expand Up @@ -8326,13 +8361,17 @@ impl<'a> Parser<'a> {
table_name,
using,
columns,
vector,
or_replace,
unique,
concurrently,
r#async,
if_not_exists,
include,
storing,
nulls_distinct,
with,
options,
predicate,
index_options,
alter_options,
Expand Down
12 changes: 12 additions & 0 deletions tests/sqlparser_bigquery.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2950,3 +2950,15 @@ fn test_create_snapshot_table() {
"CREATE SNAPSHOT TABLE IF NOT EXISTS dataset_id.table1 CLONE dataset_id.table2 FOR SYSTEM_TIME AS OF TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 1 HOUR) OPTIONS(expiration_timestamp = TIMESTAMP '2025-01-01 00:00:00 UTC')",
);
}

#[test]
fn parse_bigquery_create_vector_index() {
// BigQuery's form: an `OPTIONS(...)` clause with its index_type / distance_type
// / JSON tuning keys, and a `STORING(...)` covering-column list.
bigquery().verified_stmt(
"CREATE VECTOR INDEX my_index ON my_dataset.my_table(embedding) OPTIONS(index_type = 'IVF', distance_type = 'COSINE', ivf_options = '{\"num_lists\": 2500}')",
);
bigquery().verified_stmt(
"CREATE OR REPLACE VECTOR INDEX my_index ON my_dataset.my_table(embedding) STORING(type, creation_time) OPTIONS(index_type = 'TREE_AH', distance_type = 'EUCLIDEAN')",
);
}
98 changes: 98 additions & 0 deletions tests/sqlparser_common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9728,6 +9728,10 @@ fn test_create_index_with_using_function() {
predicate: None,
index_options,
alter_options,
vector: _,
or_replace: _,
options: _,
storing: _,
}) => {
assert_eq!("idx_name", name.to_string());
assert_eq!("test", table_name.to_string());
Expand Down Expand Up @@ -9785,6 +9789,10 @@ fn test_create_index_with_with_clause() {
predicate: None,
index_options,
alter_options,
vector: _,
or_replace: _,
options: _,
storing: _,
}) => {
pretty_assertions::assert_eq!("title_idx", name.to_string());
pretty_assertions::assert_eq!("films", table_name.to_string());
Expand All @@ -9808,6 +9816,96 @@ fn parse_create_index_async() {
verified_stmt("CREATE UNIQUE INDEX ASYNC my_index ON my_table(col1)");
}

#[test]
fn parse_create_vector_index() {
// `CREATE VECTOR INDEX` parses for every dialect as a `CreateIndex` flagged
// `vector`; the `OPTIONS(...)` trailer lands in `options`.
let sql =
"CREATE VECTOR INDEX emb ON t(embedding) OPTIONS(distance_type = 'COSINE', dimension = 4)";
match verified_stmt(sql) {
Statement::CreateIndex(CreateIndex {
name,
table_name,
using,
columns,
vector,
or_replace,
unique,
if_not_exists,
with,
options,
..
}) => {
assert!(vector);
assert!(!or_replace);
assert!(!unique);
assert!(!if_not_exists);
assert_eq!(name.unwrap().to_string(), "emb");
assert_eq!(table_name.to_string(), "t");
assert_eq!(using, None);
assert!(with.is_empty());
assert_eq!(
columns.iter().map(|c| c.to_string()).collect::<Vec<_>>(),
vec!["embedding"]
);
assert_eq!(
options,
vec![
SqlOption::KeyValue {
key: Ident::new("distance_type"),
value: Expr::Value(
Value::SingleQuotedString("COSINE".to_string()).with_empty_span()
),
},
SqlOption::KeyValue {
key: Ident::new("dimension"),
value: Expr::value(number("4")),
},
]
);
}
other => panic!("expected CreateIndex, got {other:?}"),
}

// `OR REPLACE`.
match verified_stmt(
"CREATE OR REPLACE VECTOR INDEX emb ON t(embedding) OPTIONS(distance_type = 'COSINE')",
) {
Statement::CreateIndex(CreateIndex {
vector, or_replace, ..
}) => {
assert!(vector);
assert!(or_replace);
}
other => panic!("expected CreateIndex, got {other:?}"),
}

// `IF NOT EXISTS` and schema-qualified names.
match verified_stmt(
"CREATE VECTOR INDEX IF NOT EXISTS s.emb ON s.t(embedding) OPTIONS(distance_type = 'EUCLIDEAN')",
) {
Statement::CreateIndex(CreateIndex {
vector,
if_not_exists,
..
}) => {
assert!(vector);
assert!(if_not_exists);
}
other => panic!("expected CreateIndex, got {other:?}"),
}

// The bare core plus the shared trailers all round-trip across dialects: an
// expression target, `INCLUDE` / `STORING` covering columns, a `WITH`
// options clause, and a trailing `USING <method>`.
verified_stmt("CREATE VECTOR INDEX emb ON t(embedding)");
verified_stmt("CREATE VECTOR INDEX emb ON t(VEC_COSINE_DISTANCE(embedding))");
verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) INCLUDE (a, b)");
verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) STORING(a, b)");
verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) WITH (metric = 'cosine')");
verified_stmt("CREATE VECTOR INDEX emb ON t(embedding) USING HNSW");
}

#[test]
fn parse_drop_index() {
let sql = "DROP INDEX idx_a";
Expand Down
9 changes: 9 additions & 0 deletions tests/sqlparser_mssql.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2925,3 +2925,12 @@ fn parse_mssql_money_constants() {
expr_from_projection(only(&select.projection)),
);
}

#[test]
fn parse_mssql_create_vector_index() {
// SQL Server's form: bracket-quoted names and a `WITH (...)` options clause
// (`METRIC` / `TYPE` / `MAXDOP`).
ms().verified_stmt(
"CREATE VECTOR INDEX vec_idx ON [dbo].[articles]([title_vector]) WITH (METRIC = 'cosine', TYPE = 'DiskANN', MAXDOP = 8)",
);
}
9 changes: 9 additions & 0 deletions tests/sqlparser_mysql.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4946,3 +4946,12 @@ fn parse_adjacent_string_literal_concatenation() {
fn parse_group_by_with_rollup() {
mysql().verified_stmt("SELECT * FROM tbl GROUP BY col1, col2 WITH ROLLUP");
}

#[test]
fn parse_mysql_create_vector_index() {
// TiDB's form: the vector column is wrapped in a distance function and the
// algorithm is named with a trailing `USING`.
mysql().verified_stmt(
"CREATE VECTOR INDEX idx_cos ON tidb_vectors((VEC_COSINE_DISTANCE(embedding))) USING HNSW",
);
}
10 changes: 10 additions & 0 deletions tests/sqlparser_oracle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -539,3 +539,13 @@ fn test_insert_without_alias() {
if matches!(&*source, Query { body, .. } if matches!(&**body, SetExpr::Values(_)))
));
}

#[test]
fn parse_oracle_create_vector_index() {
// Oracle's specialized clauses (ORGANIZATION / DISTANCE / WITH TARGET
// ACCURACY / PARAMETERS) are not yet parsed; the forms it shares with the
// common grammar — an expression target and an `INCLUDE` list — round-trip.
oracle().verified_stmt("CREATE VECTOR INDEX g_idx ON galaxies(embedding)");
oracle().verified_stmt("CREATE VECTOR INDEX g_idx ON galaxies(VEC_DISTANCE(embedding))");
oracle().verified_stmt("CREATE VECTOR INDEX g_idx ON galaxies(embedding) INCLUDE (id)");
}
Loading
Loading