diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5d2513904f1..3da2dda73a5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -529,6 +529,13 @@ jobs: save-if: false prefix-key: v1 + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: 22 + + - uses: ./.github/actions/setup-pnpm + - name: Run bindgen tests run: cargo ci wasm-bindings diff --git a/crates/bench/src/spacetime_raw.rs b/crates/bench/src/spacetime_raw.rs index 119d8417ec2..bc98446047a 100644 --- a/crates/bench/src/spacetime_raw.rs +++ b/crates/bench/src/spacetime_raw.rs @@ -69,7 +69,7 @@ impl BenchDatabase for SpacetimeRaw { IndexSchema { index_id: IndexId::SENTINEL, table_id, - index_name: column.name.clone().unwrap(), + index_name: column.name.clone().unwrap().into(), index_algorithm: IndexAlgorithm::BTree(BTreeAlgorithm { columns: ColId(i as _).into(), }), diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h index 05ec121213d..ea2e4b5ec85 100644 --- a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawModuleDefV10Section.g.h @@ -12,22 +12,23 @@ #include #include "../autogen_base.h" #include "spacetimedb/bsatn/bsatn.h" -#include "RawViewPrimaryKeyDefV10.g.h" #include "RawTableDefV10.g.h" -#include "RawViewDefV10.g.h" -#include "RawScheduleDefV10.g.h" -#include "CaseConversionPolicy.g.h" -#include "RawRowLevelSecurityDefV9.g.h" -#include "RawHttpRouteDefV10.g.h" -#include "RawReducerDefV10.g.h" #include "Typespace.g.h" #include "RawLifeCycleReducerDefV10.g.h" -#include "ExplicitNames.g.h" -#include "RawHttpHandlerDefV10.g.h" +#include "RawReducerDefV10.g.h" #include "RawTypeDefV10.g.h" +#include "RawRowLevelSecurityDefV9.g.h" +#include "RawHttpRouteDefV10.g.h" +#include "RawSubmoduleV10.g.h" +#include "RawViewDefV10.g.h" +#include "ExplicitNames.g.h" #include "RawProcedureDefV10.g.h" +#include "CaseConversionPolicy.g.h" +#include "RawScheduleDefV10.g.h" +#include "RawViewPrimaryKeyDefV10.g.h" +#include "RawHttpHandlerDefV10.g.h" namespace SpacetimeDB::Internal { -SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector) +SPACETIMEDB_INTERNAL_TAGGED_ENUM(RawModuleDefV10Section, SpacetimeDB::Internal::Typespace, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, std::vector, SpacetimeDB::Internal::CaseConversionPolicy, SpacetimeDB::Internal::ExplicitNames, std::vector, std::vector, std::vector, std::vector) } // namespace SpacetimeDB::Internal diff --git a/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawSubmoduleV10.g.h b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawSubmoduleV10.g.h new file mode 100644 index 00000000000..c089a4c7820 --- /dev/null +++ b/crates/bindings-cpp/include/spacetimedb/internal/autogen/RawSubmoduleV10.g.h @@ -0,0 +1,32 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb codegen. + +#pragma once + +#include +#include +#include +#include +#include +#include "../autogen_base.h" +#include "spacetimedb/bsatn/bsatn.h" + +namespace SpacetimeDB::Internal { +struct RawModuleDefV10; +} // namespace SpacetimeDB::Internal + +namespace SpacetimeDB::Internal { + +SPACETIMEDB_INTERNAL_PRODUCT_TYPE(RawSubmoduleV10) { + std::string namespace_; + std::shared_ptr module; + + void bsatn_serialize(::SpacetimeDB::bsatn::Writer& writer) const { + ::SpacetimeDB::bsatn::serialize(writer, namespace_); + ::SpacetimeDB::bsatn::serialize(writer, *module); + } + SPACETIMEDB_PRODUCT_TYPE_EQUALITY(namespace_, module) +}; +} // namespace SpacetimeDB::Internal diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs index 124cf639b64..61212c98e89 100644 --- a/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawModuleDefV10Section.g.cs @@ -22,6 +22,7 @@ public partial record RawModuleDefV10Section : SpacetimeDB.TaggedEnum<( ExplicitNames ExplicitNames, System.Collections.Generic.List HttpHandlers, System.Collections.Generic.List HttpRoutes, - System.Collections.Generic.List ViewPrimaryKeys + System.Collections.Generic.List ViewPrimaryKeys, + System.Collections.Generic.List Submodules )>; } diff --git a/crates/bindings-csharp/Runtime/Internal/Autogen/RawSubmoduleV10.g.cs b/crates/bindings-csharp/Runtime/Internal/Autogen/RawSubmoduleV10.g.cs new file mode 100644 index 00000000000..c5e6172da9b --- /dev/null +++ b/crates/bindings-csharp/Runtime/Internal/Autogen/RawSubmoduleV10.g.cs @@ -0,0 +1,36 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +#nullable enable + +using System; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace SpacetimeDB.Internal +{ + [SpacetimeDB.Type] + [DataContract] + public sealed partial class RawSubmoduleV10 + { + [DataMember(Name = "namespace")] + public string Namespace; + [DataMember(Name = "module")] + public RawModuleDefV10 Module; + + public RawSubmoduleV10( + string Namespace, + RawModuleDefV10 Module + ) + { + this.Namespace = Namespace; + this.Module = Module; + } + + public RawSubmoduleV10() + { + this.Namespace = ""; + this.Module = new(); + } + } +} diff --git a/crates/bindings-typescript/src/lib/autogen/types.ts b/crates/bindings-typescript/src/lib/autogen/types.ts index 2c2b00143ff..3cee51f03d5 100644 --- a/crates/bindings-typescript/src/lib/autogen/types.ts +++ b/crates/bindings-typescript/src/lib/autogen/types.ts @@ -390,6 +390,9 @@ export const RawModuleDefV10Section = __t.enum('RawModuleDefV10Section', { get ViewPrimaryKeys() { return __t.array(RawViewPrimaryKeyDefV10); }, + get Submodules() { + return __t.array(RawSubmoduleV10); + }, }); export type RawModuleDefV10Section = __Infer; @@ -547,6 +550,14 @@ export const RawSequenceDefV9 = __t.object('RawSequenceDefV9', { }); export type RawSequenceDefV9 = __Infer; +export const RawSubmoduleV10 = __t.object('RawSubmoduleV10', { + namespace: __t.string(), + get module(): any { + return RawModuleDefV10; + }, +}); +export type RawSubmoduleV10 = __Infer; + export const RawTableDefV10 = __t.object('RawTableDefV10', { sourceName: __t.string(), productTypeRef: __t.u32(), diff --git a/crates/bindings-typescript/src/lib/query.ts b/crates/bindings-typescript/src/lib/query.ts index fda31f68f45..bb93b0e6ce3 100644 --- a/crates/bindings-typescript/src/lib/query.ts +++ b/crates/bindings-typescript/src/lib/query.ts @@ -2,6 +2,7 @@ import { ConnectionId } from './connection_id'; import { Identity } from './identity'; import type { ColumnIndex, IndexColumns, IndexOpts } from './indexes'; import type { UntypedSchemaDef } from './schema'; +import type { UntypedTableDef } from './table'; import type { UntypedTableSchema } from './table_schema'; import { Timestamp } from './timestamp'; import type { @@ -223,6 +224,26 @@ export type QueryBuilder = { > as Tbl['accessorName']]: TableRef & From; } & {}; +/** + * Like `QueryBuilder`, but with declared namespaces also exposed as sub-objects. + * This is the type of the `tables` argument in a `subscribe` query-builder callback. + * + * Root-level tables appear as direct properties (same as `QueryBuilder`). + * Declared namespaces appear as sub-objects — each is itself a `QueryBuilder` for that + * namespace's schema, so `tables..` is fully typed. + * + * When `SchemaDef['namespaces']` is absent or `{}`, no namespace properties appear — + * accessing an undeclared namespace is a compile error. + */ +export type NamespacedQueryBuilder = + QueryBuilder & { + readonly [NS in keyof NonNullable]: NonNullable< + SchemaDef['namespaces'] + >[NS] extends UntypedSchemaDef + ? QueryBuilder[NS]> + : never; + }; + /** * A runtime reference to a table. This materializes the RowExpr for us. * TODO: Maybe add the full SchemaDef to the type signature depending on how joins will work. @@ -335,6 +356,42 @@ export function makeQueryBuilder( return Object.freeze(qb) as QueryBuilder; } +/** + * Builds the namespace-aware `tables` object passed to `subscribe`'s query-builder callback. + * + * Tables whose `sourceName` contains no `.` are placed at the root. + * Tables with a dotted `sourceName` (e.g. `"namespace.table"`) are grouped under a + * sub-object keyed by the namespace alias, with the part after the dot as the + * property key within that namespace. + */ +export function makeFromBuilder( + tables: SchemaDef['tables'] +): NamespacedQueryBuilder { + const result: Record = Object.create(null); + const namespaces: Record> = Object.create( + null + ); + + for (const table of Object.values(tables) as UntypedTableDef[]) { + const dotIdx = table.sourceName.indexOf('.'); + if (dotIdx === -1) { + result[table.accessorName] = createTableRefFromDef(table as any); + } else { + const ns = table.sourceName.slice(0, dotIdx); + const key = table.sourceName.slice(dotIdx + 1); + (namespaces[ns] ??= Object.create(null))[key] = createTableRefFromDef( + table as any + ); + } + } + + for (const [ns, nsTables] of Object.entries(namespaces)) { + result[ns] = Object.freeze(nsTables); + } + + return Object.freeze(result) as unknown as NamespacedQueryBuilder; +} + function createRowExpr( tableDef: TableDef ): RowExpr { @@ -879,7 +936,10 @@ function literalValueToSql(value: unknown): string { } function quoteIdentifier(name: string): string { - return `"${name.replace(/"/g, '""')}"`; + return name + .split('.') + .map(part => `"${part.replace(/"/g, '""')}"`) + .join('.'); } function isLiteralExpr( diff --git a/crates/bindings-typescript/src/lib/reducers.ts b/crates/bindings-typescript/src/lib/reducers.ts index 0eae2adc2a9..27c57f0721b 100644 --- a/crates/bindings-typescript/src/lib/reducers.ts +++ b/crates/bindings-typescript/src/lib/reducers.ts @@ -98,6 +98,12 @@ export interface JwtClaims { readonly fullPayload: JsonObject; } +export type AliasViews = SchemaDef extends { + namespaces: infer NS extends Record; +} + ? { readonly [K in keyof NS]: ReducerCtx } + : {}; + /** * Reducer context parametrized by the inferred Schema */ @@ -113,4 +119,5 @@ export type ReducerCtx = Readonly<{ newUuidV4(): Uuid; newUuidV7(): Uuid; random: Random; + as: AliasViews; }>; diff --git a/crates/bindings-typescript/src/lib/schema.ts b/crates/bindings-typescript/src/lib/schema.ts index 1229aed82b8..eb3821f6606 100644 --- a/crates/bindings-typescript/src/lib/schema.ts +++ b/crates/bindings-typescript/src/lib/schema.ts @@ -7,6 +7,7 @@ import { } from './algebraic_type'; import type { CaseConversionPolicy, + RawSubmoduleV10, RawModuleDefV10, RawModuleDefV10Section, RawScopedTypeNameV10, @@ -42,6 +43,7 @@ export type TableNamesOf = Values< */ export type UntypedSchemaDef = { tables: Record; + namespaces?: Record; }; /** @@ -202,6 +204,7 @@ export class ModuleContext { explicitNames: { entries: [], }, + submodules: [], }; get moduleDef(): ModuleDef { @@ -266,9 +269,19 @@ export class ModuleContext { value: module.caseConversionPolicy, } ); + push( + module.submodules && { + tag: 'Submodules', + value: module.submodules, + } + ); return { sections }; } + addSubmodule(submodule: RawSubmoduleV10) { + this.#moduleDef.submodules.push(submodule); + } + /** * Set the case conversion policy for this module. * Called by the settings mechanism. diff --git a/crates/bindings-typescript/src/sdk/db_connection_impl.ts b/crates/bindings-typescript/src/sdk/db_connection_impl.ts index f462c88a56e..f7b3c60e048 100644 --- a/crates/bindings-typescript/src/sdk/db_connection_impl.ts +++ b/crates/bindings-typescript/src/sdk/db_connection_impl.ts @@ -35,7 +35,7 @@ import type { } from './message_types.ts'; import type { ReducerEvent } from './reducer_event.ts'; import { type UntypedRemoteModule } from './spacetime_module.ts'; -import { makeQueryBuilder } from '../lib/query'; +import { makeFromBuilder, type NamespacedQueryBuilder } from '../lib/query'; import { type TableCache, type Operation, @@ -57,6 +57,7 @@ import type { } from './reducers.ts'; import type { ClientDbView } from './db_view.ts'; import type { RowType, UntypedTableDef } from '../lib/table.ts'; +import type { UntypedSchemaDef } from '../lib/schema'; import type { ProceduresView } from './procedures.ts'; import type { Values } from '../lib/type_util.ts'; import type { TransactionUpdate } from './client_api/types.ts'; @@ -484,8 +485,12 @@ export class DbConnectionImpl return new SubscriptionBuilderImpl(this); }; - getTablesMap(): any { - return makeQueryBuilder({ tables: this.#remoteModule.tables } as any); + getFromBuilder< + SchemaDef extends UntypedSchemaDef, + >(): NamespacedQueryBuilder { + return makeFromBuilder( + this.#remoteModule.tables as SchemaDef['tables'] + ); } registerSubscription( diff --git a/crates/bindings-typescript/src/sdk/subscription_builder_impl.ts b/crates/bindings-typescript/src/sdk/subscription_builder_impl.ts index 1ac436c8c9c..f7e271b4e85 100644 --- a/crates/bindings-typescript/src/sdk/subscription_builder_impl.ts +++ b/crates/bindings-typescript/src/sdk/subscription_builder_impl.ts @@ -6,8 +6,13 @@ import type { } from './event_context'; import { EventEmitter } from './event_emitter'; import type { UntypedRemoteModule } from './spacetime_module'; -import { isRowTypedQuery, toSql, type RowTypedQuery } from '../lib/query'; -import type { Values } from '../lib/type_util'; +import { + isRowTypedQuery, + toSql, + type NamespacedQueryBuilder, + type RowTypedQuery, +} from '../lib/query'; +import type { UntypedSchemaDef } from '../lib/schema'; export class SubscriptionBuilderImpl { #onApplied?: (ctx: SubscriptionEventContextInterface) => void = @@ -86,9 +91,14 @@ export class SubscriptionBuilderImpl { subscribe( query_sql: Array> ): SubscriptionHandleImpl; + /** + * @param queryFn - Receives the query builder for all tables (root and namespaced), e.g. + * `tables => tables.players.build()` or, for a submodule table, + * `tables => tables.inventory.items.build()`. May return multiple queries as an array. + */ subscribe( queryFn: ( - tables: Values + tables: NamespacedQueryBuilder ) => RowTypedQuery | RowTypedQuery[] ): SubscriptionHandleImpl; subscribe( @@ -100,8 +110,8 @@ export class SubscriptionBuilderImpl { ): SubscriptionHandleImpl { let queries: Array>; if (typeof query_sql === 'function') { - const tablesMap = this.db.getTablesMap?.(); - const result = query_sql(tablesMap); + const tables = this.db.getFromBuilder(); + const result = query_sql(tables); queries = Array.isArray(result) ? result : [result]; } else { queries = Array.isArray(query_sql) ? query_sql : [query_sql]; diff --git a/crates/bindings-typescript/src/server/db_view.ts b/crates/bindings-typescript/src/server/db_view.ts index 9e0350ff863..847a74f957b 100644 --- a/crates/bindings-typescript/src/server/db_view.ts +++ b/crates/bindings-typescript/src/server/db_view.ts @@ -9,7 +9,11 @@ export type ReadonlyDbView = { readonly [Tbl in Values< SchemaDef['tables'] > as Tbl['accessorName']]: ReadonlyTable; -}; +} & (SchemaDef extends { + namespaces: infer NS extends Record; +} + ? { readonly [K in keyof NS]: ReadonlyDbView } + : {}); /** * A type representing the database view, mapping table names to their corresponding Table handles. @@ -18,4 +22,8 @@ export type DbView = { readonly [Tbl in Values< SchemaDef['tables'] > as Tbl['accessorName']]: Table; -}; +} & (SchemaDef extends { + namespaces: infer NS extends Record; +} + ? { readonly [K in keyof NS]: DbView } + : {}); diff --git a/crates/bindings-typescript/src/server/http_handlers.ts b/crates/bindings-typescript/src/server/http_handlers.ts index a092f513611..68d42a267f8 100644 --- a/crates/bindings-typescript/src/server/http_handlers.ts +++ b/crates/bindings-typescript/src/server/http_handlers.ts @@ -211,11 +211,19 @@ export class Request { } } +export type HandlerAliasViews = + SchemaDef extends { + namespaces: infer NS extends Record; + } + ? { readonly [K in keyof NS]: HandlerContext } + : {}; + export interface HandlerContext { readonly timestamp: Timestamp; readonly http: HttpClient; readonly identity: Identity; readonly random: Random; + readonly as: HandlerAliasViews; withTx(body: (ctx: TransactionCtx) => T): T; newUuidV4(): Uuid; newUuidV7(): Uuid; @@ -228,7 +236,8 @@ export type HandlerFn = ( export interface HttpHandlerExport< S extends UntypedSchemaDef = UntypedSchemaDef, -> extends ModuleExport { +> extends HandlerFn, + ModuleExport { [httpHandlerFn]: HandlerFn; } @@ -354,24 +363,27 @@ export function makeHttpHandlerExport( opts: HttpHandlerOpts | undefined, fn: HandlerFn ): HttpHandlerExport { - const handlerExport = { - [httpHandlerFn]: fn, - [exportContext]: ctx, - [registerExport](ctx: SchemaInner, exportName: string) { - if (exportedHttpHandlerObjects.has(handlerExport)) { - throw new TypeError( - `HTTP handler '${exportName}' was exported more than once` + const handlerExport: HttpHandlerExport = Object.assign( + (...args: Parameters>) => fn(...args), + { + [httpHandlerFn]: fn, + [exportContext]: ctx, + [registerExport](ctx: SchemaInner, exportName: string) { + if (exportedHttpHandlerObjects.has(handlerExport)) { + throw new TypeError( + `HTTP handler '${exportName}' was exported more than once` + ); + } + exportedHttpHandlerObjects.add(handlerExport); + registerHttpHandler(ctx, exportName, fn, opts); + ctx.httpHandlerExports.set( + handlerExport as HttpHandlerExport, + exportName ); - } - exportedHttpHandlerObjects.add(handlerExport); - registerHttpHandler(ctx, exportName, fn, opts); - ctx.httpHandlerExports.set( - handlerExport as HttpHandlerExport, - exportName - ); - }, - }; - return handlerExport as HttpHandlerExport; + }, + } + ); + return handlerExport; } export function makeHttpRouterExport( diff --git a/crates/bindings-typescript/src/server/index.ts b/crates/bindings-typescript/src/server/index.ts index a840be4a59d..ae084f672d4 100644 --- a/crates/bindings-typescript/src/server/index.ts +++ b/crates/bindings-typescript/src/server/index.ts @@ -33,5 +33,6 @@ export { type ResponseInit, } from './http'; export type { HandlerContext, HttpHandlerExport } from './http'; +export { ScheduleAt } from '../lib/schedule_at'; import './polyfills'; // Ensure polyfills are loaded diff --git a/crates/bindings-typescript/src/server/procedures.ts b/crates/bindings-typescript/src/server/procedures.ts index 5f1867d986c..2dec68467c0 100644 --- a/crates/bindings-typescript/src/server/procedures.ts +++ b/crates/bindings-typescript/src/server/procedures.ts @@ -24,10 +24,18 @@ import { Uuid } from '../lib/uuid'; import { httpClient, type HttpClient } from './http_internal'; import type { DbView } from './db_view'; import { makeRandom, type Random } from './rng'; -import { callUserFunction, ReducerCtxImpl, runWithTx, sys } from './runtime'; +import { + assignTxAliasViews, + buildProcedureAliasCtxMap, + callUserFunction, + ReducerCtxImpl, + runWithTx, + sys, +} from './runtime'; import { exportContext, registerExport, + type SubmoduleDispatchInfo, type ModuleExport, type SchemaInner, } from './schema'; @@ -92,6 +100,13 @@ export type ProcedureOptsWithOptionalName< Ret extends TypeBuilder = TypeBuilder, > = Omit, 'name'> & { name?: string }; +export type ProcedureAliasViews = + SchemaDef extends { + namespaces: infer NS extends Record; + } + ? { readonly [K in keyof NS]: ProcedureCtx } + : {}; + export interface ProcedureCtx { readonly sender: Identity; readonly databaseIdentity: Identity; @@ -101,6 +116,7 @@ export interface ProcedureCtx { readonly connectionId: ConnectionId | null; readonly http: HttpClient; readonly random: Random; + readonly as: ProcedureAliasViews; withTx(body: (ctx: TransactionCtx) => T): T; newUuidV4(): Uuid; newUuidV7(): Uuid; @@ -173,23 +189,27 @@ export type Procedures = Array<{ }>; export function callProcedure( - moduleCtx: SchemaInner, + procedures: Procedures, id: number, sender: Identity, connectionId: ConnectionId | null, timestamp: Timestamp, argsBuf: Uint8Array, - dbView: () => DbView + dbView: () => DbView, + dispatches: SubmoduleDispatchInfo[] = [], + parentPrefix = '' ): Uint8Array { const { fn, deserializeArgs, serializeReturn, returnTypeBaseSize } = - moduleCtx.procedures[id]; + procedures[id]; const args = deserializeArgs(new BinaryReader(argsBuf)); const ctx: ProcedureCtx = new ProcedureCtxImpl( sender, timestamp, connectionId, - dbView + dbView, + dispatches, + parentPrefix ); const ret = callUserFunction(fn, ctx, args); @@ -206,14 +226,21 @@ const ProcedureCtxImpl = class ProcedureCtx #uuidCounter: { value: 0 } | undefined; #random: Random | undefined; #dbView: () => DbView; + #dispatches: SubmoduleDispatchInfo[]; + #parentPrefix: string; + #asViews: object | undefined; constructor( readonly sender: Identity, readonly timestamp: Timestamp, readonly connectionId: ConnectionId | null, - dbView: () => DbView + dbView: () => DbView, + dispatches: SubmoduleDispatchInfo[] = [], + parentPrefix = '' ) { this.#dbView = dbView; + this.#dispatches = dispatches; + this.#parentPrefix = parentPrefix; } get databaseIdentity() { @@ -232,17 +259,27 @@ const ProcedureCtxImpl = class ProcedureCtx return httpClient; } + get as() { + return (this.#asViews ??= buildProcedureAliasCtxMap( + this, + this.#dispatches, + this.#parentPrefix + )) as any; + } + withTx(body: (ctx: TransactionCtx) => T): T { - return runWithTx( - timestamp => - new TransactionCtxImpl( - this.sender, - timestamp, - this.connectionId, - this.#dbView() - ) as TransactionCtx, - body - ); + const dispatches = this.#dispatches; + const parentPrefix = this.#parentPrefix; + return runWithTx(timestamp => { + const tx = new TransactionCtxImpl( + this.sender, + timestamp, + this.connectionId, + this.#dbView() + ); + assignTxAliasViews(tx, dispatches, parentPrefix); + return tx as unknown as TransactionCtx; + }, body); } newUuidV4(): Uuid { diff --git a/crates/bindings-typescript/src/server/runtime.ts b/crates/bindings-typescript/src/server/runtime.ts index e5d782a71e3..3ecf9f560df 100644 --- a/crates/bindings-typescript/src/server/runtime.ts +++ b/crates/bindings-typescript/src/server/runtime.ts @@ -10,6 +10,8 @@ import { import { RawModuleDef, ViewResultHeader, + type RawProcedureDefV10, + type RawReducerDefV10, type RawTableDefV10, type Typespace, } from '../lib/autogen/types'; @@ -27,6 +29,8 @@ import { type UniqueIndex, } from '../lib/indexes'; import { callProcedure } from './procedures'; +import type { Procedures } from './procedures'; +import type { Reducers } from './reducers'; import { type HandlerContext, Request, @@ -40,21 +44,37 @@ import { serializeHeaders, } from './http_shared'; import { + type AliasViews, type AuthCtx, type JsonObject, type JwtClaims, type ReducerCtx as IReducerCtx, } from '../lib/reducers'; import { type UntypedSchemaDef } from '../lib/schema'; -import { type RowType, type Table, type TableMethods } from '../lib/table'; +import { + type RowType, + type Table, + type TableMethods, + type UntypedTableDef, +} from '../lib/table'; import { bsatnBaseSize, hasOwn } from '../lib/util'; -import { type AnonymousViewCtx, type ViewCtx } from './views'; -import { isRowTypedQuery, makeQueryBuilder, toSql } from './query'; -import type { DbView } from './db_view'; +import { + type AnonymousViewCtx, + type AnonViews, + type ViewCtx, + type Views, +} from './views'; +import { + isRowTypedQuery, + makeQueryBuilder, + toSql, + type QueryBuilder, +} from './query'; +import type { DbView, ReadonlyDbView } from './db_view'; import { getErrorConstructor, SenderError } from './errors'; import { Range, type Bound } from './range'; import { makeRandom, type Random } from './rng'; -import type { SchemaInner } from './schema'; +import type { SubmoduleDispatchInfo, SchemaInner } from './schema'; import { HttpRequest, HttpResponse } from '../lib/autogen/types'; const { freeze } = Object; @@ -226,18 +246,21 @@ export const ReducerCtxImpl = class ReducerCtx< timestamp: Timestamp; connectionId: ConnectionId | null; db: DbView; + as: AliasViews; constructor( sender: Identity, timestamp: Timestamp, connectionId: ConnectionId | null, - dbView: DbView + dbView: DbView, + asViews: object = {} ) { Object.seal(this); this.sender = sender; this.timestamp = timestamp; this.connectionId = connectionId; - this.db = dbView; + this.db = dbView as unknown as DbView; + this.as = asViews as AliasViews; } /** Reset the `ReducerCtx` to be used for a new transaction */ @@ -245,13 +268,21 @@ export const ReducerCtxImpl = class ReducerCtx< me: InstanceType, sender: Identity, timestamp: Timestamp, - connectionId: ConnectionId | null + connectionId: ConnectionId | null, + dbView?: DbView, + asViews?: object ) { me.sender = sender; me.timestamp = timestamp; me.connectionId = connectionId; me.#uuidCounter = undefined; me.#senderAuth = undefined; + if (dbView !== undefined) { + me.db = dbView; + } + if (asViews !== undefined) { + me.as = asViews as AliasViews; + } } get databaseIdentity() { @@ -337,34 +368,145 @@ export function runWithTx( } } +type FlatSubmoduleDispatch = { + reducerFns: Reducers; + reducerDefs: RawReducerDefV10[]; + procedureFns: Procedures; + procedureDefs: RawProcedureDefV10[]; + anonViewFns: AnonViews; + viewFns: Views; + tables: Array<{ accessorName: string; tableDef: RawTableDefV10 }>; + schemaTables: Record; + typespace: Typespace; + dbView_: DbView | undefined; + queryBuilder_: QueryBuilder | undefined; + /** e.g. "alias." for a submodule with namespace alias "alias" */ + namePrefix: string; + subDispatches: SubmoduleDispatchInfo[]; +}; + +function flattenSubmoduleDispatches( + dispatches: SubmoduleDispatchInfo[], + parentPrefix = '' +): FlatSubmoduleDispatch[] { + const result: FlatSubmoduleDispatch[] = []; + for (const d of dispatches) { + const namePrefix = parentPrefix + d.namespace + '.'; + result.push({ + reducerFns: d.reducerFns, + reducerDefs: d.reducerDefs, + procedureFns: d.procedureFns, + procedureDefs: d.procedureDefs, + anonViewFns: d.anonViewFns, + viewFns: d.viewFns, + tables: d.tables, + schemaTables: d.schemaTables, + typespace: d.typespace, + dbView_: undefined, + queryBuilder_: undefined, + namePrefix, + subDispatches: d.subDispatches, + }); + result.push(...flattenSubmoduleDispatches(d.subDispatches, namePrefix)); + } + return result; +} + export const makeHooks = (schema: SchemaInner): ModuleHooks => new ModuleHooksImpl(schema); class ModuleHooksImpl implements ModuleHooks { #schema: SchemaInner; #dbView_: DbView | undefined; + #consumerAs_: object | undefined; #reducerArgsDeserializers; - /** Cache the `ReducerCtx` object to avoid allocating anew for ever reducer call. */ + #consumerReducerCount: number; + #consumerProcedureCount: number; + #flatSubmodules: FlatSubmoduleDispatch[]; + #consumerAnonViewCount: number; + #consumerViewCount: number; + /** Cache the `ReducerCtx` object to avoid allocating anew for every reducer call. */ #reducerCtx_: InstanceType | undefined; + /** Per-submodule alias ctx maps, cached lazily (parallel to #flatSubmodules). */ + #submoduleAsViews_: (object | undefined)[] = []; constructor(schema: SchemaInner) { this.#schema = schema; - this.#reducerArgsDeserializers = schema.moduleDef.reducers.map( - ({ params }) => ProductType.makeDeserializer(params, schema.typespace) + this.#consumerReducerCount = schema.reducers.length; + this.#consumerProcedureCount = schema.procedures.length; + this.#consumerAnonViewCount = schema.anonViews.length; + this.#consumerViewCount = schema.views.length; + this.#flatSubmodules = flattenSubmoduleDispatches( + schema.submoduleDispatchInfos + ); + + const consumerDeserializers = schema.moduleDef.reducers.map(({ params }) => + ProductType.makeDeserializer(params, schema.typespace) + ); + const submoduleDeserializers = this.#flatSubmodules.flatMap( + ({ reducerDefs, typespace }) => + reducerDefs.map(({ params }) => + ProductType.makeDeserializer(params, typespace) + ) ); + this.#reducerArgsDeserializers = [ + ...consumerDeserializers, + ...submoduleDeserializers, + ]; } get #dbView() { - return (this.#dbView_ ??= freeze( + if (this.#dbView_ !== undefined) return this.#dbView_; + const rootTables = Object.values(this.#schema.schemaType.tables).map( + table => [ + table.accessorName, + makeTableView(this.#schema.typespace, table.tableDef), + ] + ); + const submoduleNs = this.#schema.submoduleDispatchInfos.map(dispatch => [ + dispatch.namespace, + buildDbViewForDispatch(dispatch, dispatch.namespace + '.'), + ]); + this.#dbView_ = freeze( + Object.fromEntries([...rootTables, ...submoduleNs]) + ) as DbView; + return this.#dbView_; + } + + #getSubmoduleDbView(submoduleIdx: number): DbView { + const m = this.#flatSubmodules[submoduleIdx]; + return (m.dbView_ ??= freeze( Object.fromEntries( - Object.values(this.#schema.schemaType.tables).map(table => [ - table.accessorName, - makeTableView(this.#schema.typespace, table.tableDef), + m.tables.map(({ accessorName, tableDef }) => [ + accessorName, + makeTableView(m.typespace, tableDef, m.namePrefix), ]) - ) + ) as DbView + )); + } + + #getSubmoduleAsViews(submoduleIdx: number): object { + return (this.#submoduleAsViews_[submoduleIdx] ??= buildAliasCtxMap( + this.#reducerCtx, + this.#flatSubmodules[submoduleIdx].subDispatches, + this.#flatSubmodules[submoduleIdx].namePrefix )); } + /** Query builder scoped to the submodule's own tables, with namespace-prefixed + * source names so generated SQL targets the mounted table names. */ + #getSubmoduleQueryBuilder(submoduleIdx: number): QueryBuilder { + const m = this.#flatSubmodules[submoduleIdx]; + return (m.queryBuilder_ ??= makeQueryBuilder({ + tables: Object.fromEntries( + Object.entries(m.schemaTables).map(([key, def]) => [ + key, + { ...def, sourceName: m.namePrefix + def.sourceName }, + ]) + ), + })); + } + get #reducerCtx() { return (this.#reducerCtx_ ??= new ReducerCtxImpl( Identity.zero(), @@ -374,6 +516,14 @@ class ModuleHooksImpl implements ModuleHooks { )); } + get #consumerAs() { + return (this.#consumerAs_ ??= buildAliasCtxMap( + this.#reducerCtx, + this.#schema.submoduleDispatchInfos, + '' + )); + } + __describe_module__() { const writer = new BinaryWriter(128); RawModuleDef.serialize( @@ -398,19 +548,46 @@ class ModuleHooksImpl implements ModuleHooks { timestamp: bigint, argsBuf: DataView ): void { - const moduleCtx = this.#schema; const deserializeArgs = this.#reducerArgsDeserializers[reducerId]; BINARY_READER.reset(argsBuf); const args = deserializeArgs(BINARY_READER); const senderIdentity = new Identity(sender); + + let fn: ((...args: any[]) => any) | undefined; + let dbView: DbView; + let asViews: object; + + if (reducerId < this.#consumerReducerCount) { + fn = this.#schema.reducers[reducerId]; + dbView = this.#dbView; + asViews = this.#consumerAs; + } else { + let offset = this.#consumerReducerCount; + for (let i = 0; i < this.#flatSubmodules.length; i++) { + const m = this.#flatSubmodules[i]; + if (reducerId < offset + m.reducerFns.length) { + fn = m.reducerFns[reducerId - offset]; + dbView = this.#getSubmoduleDbView(i); + asViews = this.#getSubmoduleAsViews(i); + break; + } + offset += m.reducerFns.length; + } + if (fn === undefined) { + throw new RangeError(`unknown reducerId ${reducerId}`); + } + } + const ctx = this.#reducerCtx; ReducerCtxImpl.reset( ctx, senderIdentity, new Timestamp(timestamp), - ConnectionId.nullIfZero(new ConnectionId(connId)) + ConnectionId.nullIfZero(new ConnectionId(connId)), + dbView!, + asViews! ); - callUserFunction(moduleCtx.reducers[reducerId], ctx, args); + callUserFunction(fn, ctx, args); } __call_view__( @@ -419,15 +596,40 @@ class ModuleHooksImpl implements ModuleHooks { argsBuf: Uint8Array ): { data: Uint8Array } { const moduleCtx = this.#schema; + let viewFns: Views; + let localId: number; + let dbView: ReadonlyDbView; + let from: QueryBuilder; + + if (id < this.#consumerViewCount) { + viewFns = moduleCtx.views; + localId = id; + dbView = this.#dbView as ReadonlyDbView; + from = makeQueryBuilder(moduleCtx.schemaType); + } else { + let offset = this.#consumerViewCount; + let found = false; + for (let i = 0; i < this.#flatSubmodules.length; i++) { + const m = this.#flatSubmodules[i]; + if (id < offset + m.viewFns.length) { + viewFns = m.viewFns; + localId = id - offset; + dbView = this.#getSubmoduleDbView(i) as ReadonlyDbView; + from = this.#getSubmoduleQueryBuilder(i); + found = true; + break; + } + offset += m.viewFns.length; + } + if (!found) throw new RangeError(`unknown viewId ${id}`); + } + const { fn, deserializeParams, serializeReturn, returnTypeBaseSize } = - moduleCtx.views[id]; + viewFns![localId!]; const ctx: ViewCtx = freeze({ sender: new Identity(sender), - // this is the non-readonly DbView, but the typing for the user will be - // the readonly one, and if they do call mutating functions it will fail - // at runtime - db: this.#dbView, - from: makeQueryBuilder(moduleCtx.schemaType), + db: dbView!, + from: from!, }); const args = deserializeParams(new BinaryReader(argsBuf)); const ret = callUserFunction(fn, ctx, args); @@ -444,14 +646,39 @@ class ModuleHooksImpl implements ModuleHooks { __call_view_anon__(id: u32, argsBuf: Uint8Array): { data: Uint8Array } { const moduleCtx = this.#schema; + let anonViewFns: AnonViews; + let localId: number; + let dbView: ReadonlyDbView; + let from: QueryBuilder; + + if (id < this.#consumerAnonViewCount) { + anonViewFns = moduleCtx.anonViews; + localId = id; + dbView = this.#dbView as ReadonlyDbView; + from = makeQueryBuilder(moduleCtx.schemaType); + } else { + let offset = this.#consumerAnonViewCount; + let found = false; + for (let i = 0; i < this.#flatSubmodules.length; i++) { + const m = this.#flatSubmodules[i]; + if (id < offset + m.anonViewFns.length) { + anonViewFns = m.anonViewFns; + localId = id - offset; + dbView = this.#getSubmoduleDbView(i) as ReadonlyDbView; + from = this.#getSubmoduleQueryBuilder(i); + found = true; + break; + } + offset += m.anonViewFns.length; + } + if (!found) throw new RangeError(`unknown anonViewId ${id}`); + } + const { fn, deserializeParams, serializeReturn, returnTypeBaseSize } = - moduleCtx.anonViews[id]; + anonViewFns![localId!]; const ctx: AnonymousViewCtx = freeze({ - // this is the non-readonly DbView, but the typing for the user will be - // the readonly one, and if they do call mutating functions it will fail - // at runtime - db: this.#dbView, - from: makeQueryBuilder(moduleCtx.schemaType), + db: dbView!, + from: from!, }); const args = deserializeParams(new BinaryReader(argsBuf)); const ret = callUserFunction(fn, ctx, args); @@ -473,15 +700,43 @@ class ModuleHooksImpl implements ModuleHooks { timestamp: bigint, args: Uint8Array ): Uint8Array { - return callProcedure( - this.#schema, - id, - new Identity(sender), - ConnectionId.nullIfZero(new ConnectionId(connection_id)), - new Timestamp(timestamp), - args, - () => this.#dbView - ); + const senderIdentity = new Identity(sender); + const connId = ConnectionId.nullIfZero(new ConnectionId(connection_id)); + const ts = new Timestamp(timestamp); + + if (id < this.#consumerProcedureCount) { + return callProcedure( + this.#schema.procedures, + id, + senderIdentity, + connId, + ts, + args, + () => this.#dbView as DbView, + this.#schema.submoduleDispatchInfos + ); + } + + let offset = this.#consumerProcedureCount; + for (let i = 0; i < this.#flatSubmodules.length; i++) { + const m = this.#flatSubmodules[i]; + if (id < offset + m.procedureFns.length) { + return callProcedure( + m.procedureFns, + id - offset, + senderIdentity, + connId, + ts, + args, + () => this.#getSubmoduleDbView(i), + m.subDispatches, + m.namePrefix + ); + } + offset += m.procedureFns.length; + } + + throw new RangeError(`unknown procedureId ${id}`); } __call_http_handler__( @@ -494,7 +749,8 @@ class ModuleHooksImpl implements ModuleHooks { const handler = moduleCtx.httpHandlers[id]; const ctx = new HandlerContextImpl( new Timestamp(timestamp), - () => this.#dbView + () => this.#dbView, + this.#schema.submoduleDispatchInfos ); const requestMetadata = HttpRequest.deserialize(new BinaryReader(request)); const response = callUserFunction( @@ -521,14 +777,18 @@ class HandlerContextImpl #uuidCounter: { value: number } | undefined; #random: Random | undefined; #dbView: () => DbView; + #dispatches: SubmoduleDispatchInfo[]; + #asViews: object | undefined; readonly http = httpClient; constructor( readonly timestamp: Timestamp, - dbView: () => DbView + dbView: () => DbView, + dispatches: SubmoduleDispatchInfo[] = [] ) { this.#dbView = dbView; + this.#dispatches = dispatches; } get identity() { @@ -539,12 +799,28 @@ class HandlerContextImpl return (this.#random ??= makeRandom(this.timestamp)); } + get as() { + return (this.#asViews ??= buildHandlerAliasCtxMap( + this, + this.#dispatches, + '' + )) as any; + } + withTx(body: (ctx: any) => T): T { - return runWithTx( - timestamp => - new ReducerCtxImpl(Identity.zero(), timestamp, null, this.#dbView()), - body - ); + const dispatches = this.#dispatches; + return runWithTx(timestamp => { + const tx = new ReducerCtxImpl( + Identity.zero(), + timestamp, + null, + this.#dbView() + ); + if (dispatches.length > 0) { + tx.as = buildAliasCtxMap(tx, dispatches, '') as any; + } + return tx; + }, body); } newUuidV4(): Uuid { @@ -559,11 +835,247 @@ class HandlerContextImpl } } +function buildDbViewForDispatch( + dispatch: SubmoduleDispatchInfo, + namePrefix: string +): object { + const tableEntries = dispatch.tables.map(({ accessorName, tableDef }) => [ + accessorName, + makeTableView(dispatch.typespace, tableDef, namePrefix), + ]); + const subNsEntries = dispatch.subDispatches.map(sub => [ + sub.namespace, + buildDbViewForDispatch(sub, namePrefix + sub.namespace + '.'), + ]); + return freeze(Object.fromEntries([...tableEntries, ...subNsEntries])); +} + +function buildAliasCtx( + parent: InstanceType, + dispatch: SubmoduleDispatchInfo, + namePrefix: string +): object { + const nsDb = buildDbViewForDispatch(dispatch, namePrefix); + const subAs = buildAliasCtxMap(parent, dispatch.subDispatches, namePrefix); + return { + get sender() { + return parent.sender; + }, + get databaseIdentity() { + return parent.databaseIdentity; + }, + get identity() { + return parent.identity; + }, + get timestamp() { + return parent.timestamp; + }, + get connectionId() { + return parent.connectionId; + }, + get senderAuth() { + return parent.senderAuth; + }, + get random() { + return parent.random; + }, + newUuidV4() { + return parent.newUuidV4(); + }, + newUuidV7() { + return parent.newUuidV7(); + }, + db: nsDb, + as: subAs, + }; +} + +function buildAliasCtxMap( + parent: InstanceType, + dispatches: SubmoduleDispatchInfo[], + parentPrefix: string +): object { + return freeze( + Object.fromEntries( + dispatches.map(d => [ + d.namespace, + buildAliasCtx(parent, d, parentPrefix + d.namespace + '.'), + ]) + ) + ); +} + +function buildHandlerAliasCtx( + parent: HandlerContextImpl, + dispatch: SubmoduleDispatchInfo, + namePrefix: string +): object { + // nsDb is built lazily inside withTx so that sys.table_id_from_name is called + // only after a transaction has been started (sys.procedure_start_mut_tx). + let nsDb_: DbView | undefined; + const subAs = buildHandlerAliasCtxMap( + parent, + dispatch.subDispatches, + namePrefix + ); + return { + get timestamp() { + return parent.timestamp; + }, + get http() { + return parent.http; + }, + get identity() { + return parent.identity; + }, + get random() { + return parent.random; + }, + as: subAs, + withTx(body: any) { + return runWithTx((ts: Timestamp) => { + const tx = new ReducerCtxImpl( + Identity.zero(), + ts, + null, + (nsDb_ ??= buildDbViewForDispatch( + dispatch, + namePrefix + ) as DbView) + ); + assignTxAliasViews(tx, dispatch.subDispatches, namePrefix); + return tx; + }, body); + }, + newUuidV4() { + return parent.newUuidV4(); + }, + newUuidV7() { + return parent.newUuidV7(); + }, + }; +} + +function buildHandlerAliasCtxMap( + parent: HandlerContextImpl, + dispatches: SubmoduleDispatchInfo[], + parentPrefix: string +): object { + return freeze( + Object.fromEntries( + dispatches.map(d => [ + d.namespace, + buildHandlerAliasCtx(parent, d, parentPrefix + d.namespace + '.'), + ]) + ) + ); +} + +type ProcCtxRef = { + sender: Identity; + connectionId: ConnectionId | null; + timestamp: Timestamp; + get databaseIdentity(): Identity; + get identity(): Identity; + get http(): typeof httpClient; + get random(): Random; + newUuidV4(): Uuid; + newUuidV7(): Uuid; +}; + +function buildProcedureAliasCtx( + parent: ProcCtxRef, + dispatch: SubmoduleDispatchInfo, + namePrefix: string +): object { + // nsDb is built lazily inside withTx so that sys.table_id_from_name is called + // only after a transaction has been started (sys.procedure_start_mut_tx). + let nsDb_: DbView | undefined; + const subAs = buildProcedureAliasCtxMap( + parent, + dispatch.subDispatches, + namePrefix + ); + return { + get sender() { + return parent.sender; + }, + get databaseIdentity() { + return parent.databaseIdentity; + }, + get identity() { + return parent.identity; + }, + get timestamp() { + return parent.timestamp; + }, + get connectionId() { + return parent.connectionId; + }, + get http() { + return parent.http; + }, + get random() { + return parent.random; + }, + as: subAs, + withTx(body: any) { + return runWithTx((ts: Timestamp) => { + const tx = new ReducerCtxImpl( + parent.sender, + ts, + parent.connectionId, + (nsDb_ ??= buildDbViewForDispatch( + dispatch, + namePrefix + ) as DbView) + ); + assignTxAliasViews(tx, dispatch.subDispatches, namePrefix); + return tx; + }, body); + }, + newUuidV4() { + return parent.newUuidV4(); + }, + newUuidV7() { + return parent.newUuidV7(); + }, + }; +} + +export function buildProcedureAliasCtxMap( + parent: ProcCtxRef, + dispatches: SubmoduleDispatchInfo[], + parentPrefix: string +): object { + return freeze( + Object.fromEntries( + dispatches.map(d => [ + d.namespace, + buildProcedureAliasCtx(parent, d, parentPrefix + d.namespace + '.'), + ]) + ) + ); +} + +/** Builds and assigns reducer-style alias views onto a freshly created TransactionCtx. + * Must be called while inside a transaction (after sys.procedure_start_mut_tx). */ +export function assignTxAliasViews( + tx: InstanceType, + dispatches: SubmoduleDispatchInfo[], + parentPrefix = '' +): void { + if (dispatches.length > 0) { + tx.as = buildAliasCtxMap(tx, dispatches, parentPrefix) as any; + } +} + export function makeTableView( typespace: Typespace, - table: RawTableDefV10 + table: RawTableDefV10, + namePrefix = '' ): Table { - const table_id = sys.table_id_from_name(table.sourceName); + const table_id = sys.table_id_from_name(namePrefix + table.sourceName); const rowType = typespace.types[table.productTypeRef]; if (rowType.tag !== 'Product') { throw 'impossible'; @@ -659,7 +1171,7 @@ export function makeTableView( for (const indexDef of table.indexes) { const accessorName = indexDef.accessorName!; - const index_id = sys.index_id_from_name(indexDef.sourceName!); + const index_id = sys.index_id_from_name(namePrefix + indexDef.sourceName!); let column_ids: number[]; let isHashIndex = false; diff --git a/crates/bindings-typescript/src/server/schema.ts b/crates/bindings-typescript/src/server/schema.ts index cd99697826c..c399a66d31e 100644 --- a/crates/bindings-typescript/src/server/schema.ts +++ b/crates/bindings-typescript/src/server/schema.ts @@ -3,6 +3,11 @@ import { CaseConversionPolicy, Lifecycle, type MethodOrAny, + type RawModuleDefV10, + type RawProcedureDefV10, + type RawReducerDefV10, + type RawTableDefV10, + type Typespace, } from '../lib/autogen/types'; import { type ParamsAsObject, @@ -18,6 +23,7 @@ import { } from '../lib/schema'; import type { UntypedTableSchema } from '../lib/table_schema'; import { TypeBuilder, type ColumnBuilder } from '../lib/type_builders'; +import { hasOwn } from '../lib/util'; import { Router, type HandlerFn, @@ -68,10 +74,27 @@ type UntypedScheduledFunctionExport = | ReducerExport | ProcedureExport; +export type SubmoduleDispatchInfo = { + namespace: string; + reducerFns: Reducers; + reducerDefs: RawReducerDefV10[]; + procedureFns: Procedures; + procedureDefs: RawProcedureDefV10[]; + anonViewFns: AnonViews; + viewFns: Views; + typespace: Typespace; + tables: Array<{ accessorName: string; tableDef: RawTableDefV10 }>; + /** The submodule's own schemaType tables, used to build namespace-scoped query builders. */ + schemaTables: Record; + subDispatches: SubmoduleDispatchInfo[]; +}; + export class SchemaInner< S extends UntypedSchemaDef = UntypedSchemaDef, > extends ModuleContext { schemaType: S; + exportsRegistered = false; + schedulesResolved = false; existingFunctions = new Set(); existingHttpHandlers = new Set(); reducers: Reducers = []; @@ -89,6 +112,7 @@ export class SchemaInner< new Map(); pendingSchedules: PendingSchedule[] = []; pendingHttpRoutes: PendingHttpRoute[] = []; + submoduleDispatchInfos: SubmoduleDispatchInfo[] = []; constructor(getSchemaType: (ctx: SchemaInner) => S) { super(); @@ -114,6 +138,10 @@ export class SchemaInner< } resolveSchedules() { + if (this.schedulesResolved) { + return; + } + this.schedulesResolved = true; // Pending schedules come from two API paths: // - legacy table({ scheduled }) schedules already know their tableName and // scheduleAtCol because schema() resolves them while iterating table keys @@ -249,23 +277,9 @@ export class Schema implements ModuleDefaultExport { } [moduleHooks](exports: object) { - // if (!(hasOwn(exports, 'default') && exports.default instanceof Schema)) { - // throw new TypeError('must export schema as default export'); - // } - const registeredSchema = this.#ctx; - for (const [name, moduleExport] of Object.entries(exports)) { - if (name === 'default') continue; - if (!isModuleExport(moduleExport)) { - throw new TypeError( - 'exporting something that is not a spacetime export' - ); - } - checkExportContext(moduleExport, registeredSchema); - moduleExport[registerExport](registeredSchema, name); - } - registeredSchema.resolveSchedules(); - registeredSchema.resolveHttpRoutes(); - return makeHooks(registeredSchema); + this.buildRawModuleDefV10(exports); + this.#ctx.resolveHttpRoutes(); + return makeHooks(this.#ctx); } get schemaType(): S { @@ -280,6 +294,56 @@ export class Schema implements ModuleDefaultExport { return this.#ctx.typespace; } + get submoduleDispatchInfos(): SubmoduleDispatchInfo[] { + return this.#ctx.submoduleDispatchInfos; + } + + /** Internal: register exports and materialize the RawModuleDefV10 for upload. */ + buildRawModuleDefV10( + exports: object, + opts?: { ignoreNonModuleExports?: boolean } + ): RawModuleDefV10 { + registerModuleExports(this.#ctx, exports, { + ignoreNonModuleExports: opts?.ignoreNonModuleExports ?? false, + }); + this.#ctx.resolveSchedules(); + return this.#ctx.rawModuleDefV10(); + } + + /** + * @internal – called by schema() when processing a submodule namespace entry. + * Registers the library's exports and returns both the serialized module def + * and the runtime dispatch info needed by ModuleHooksImpl for __call_reducer__. + */ + buildSubmoduleDispatch(exports: object): { + rawDef: RawModuleDefV10; + dispatch: SubmoduleDispatchInfo; + } { + const rawDef = this.buildRawModuleDefV10(exports, { + ignoreNonModuleExports: true, + }); + this.#ctx.resolveHttpRoutes(); + return { + rawDef, + dispatch: { + namespace: '', + reducerFns: [...this.#ctx.reducers], + reducerDefs: [...this.#ctx.moduleDef.reducers], + procedureFns: [...this.#ctx.procedures], + procedureDefs: [...this.#ctx.moduleDef.procedures], + anonViewFns: [...this.#ctx.anonViews], + viewFns: [...this.#ctx.views], + typespace: this.#ctx.moduleDef.typespace, + tables: Object.values(this.#ctx.schemaType.tables).map(t => ({ + accessorName: t.accessorName, + tableDef: t.tableDef, + })), + schemaTables: this.#ctx.schemaType.tables, + subDispatches: [...this.#ctx.submoduleDispatchInfos], + }, + }; + } + /** * Defines a SpacetimeDB reducer function. * @@ -698,18 +762,101 @@ export interface ModuleSettings { CASE_CONVERSION_POLICY?: CaseConversionPolicy; } -export function schema>( - tables: H, +type SubmoduleNamespace = { + default: Schema; + [key: string]: unknown; +}; + +type SchemaEntry = UntypedTableSchema | SubmoduleNamespace; + +type ExtractTableEntries> = { + [K in keyof H as H[K] extends UntypedTableSchema ? K : never]: Extract< + H[K], + UntypedTableSchema + >; +}; + +type ExtractSubmoduleSchemas> = { + [K in keyof H as H[K] extends { default: Schema } + ? K + : never]: H[K] extends { default: Schema } + ? S + : never; +}; + +type SchemaDefForEntries> = + TablesToSchema> & { + namespaces: ExtractSubmoduleSchemas; + }; + +function isUntypedTableSchema(x: unknown): x is UntypedTableSchema { + return typeof x === 'object' && x !== null && hasOwn(x, 'tableDef'); +} + +function isSubmoduleNamespace(x: unknown): x is SubmoduleNamespace { + return ( + typeof x === 'object' && + x !== null && + hasOwn(x, 'default') && + x.default instanceof Schema + ); +} + +function registerModuleExports( + schema: SchemaInner, + exports: object, + opts?: { ignoreNonModuleExports?: boolean } +) { + if (schema.exportsRegistered) { + return; + } + schema.exportsRegistered = true; + + for (const [name, moduleExport] of Object.entries(exports)) { + if (name === 'default') continue; + if (!isModuleExport(moduleExport)) { + if (opts?.ignoreNonModuleExports) { + continue; + } + throw new TypeError('exporting something that is not a spacetime export'); + } + checkExportContext(moduleExport, schema); + moduleExport[registerExport](schema, name); + } +} + +export function schema>( + entries: H, moduleSettings?: ModuleSettings -): Schema> { - const ctx = new SchemaInner>(ctx => { +): Schema> { + const ctx = new SchemaInner>(ctx => { // Apply module settings. if (moduleSettings?.CASE_CONVERSION_POLICY != null) { ctx.setCaseConversionPolicy(moduleSettings.CASE_CONVERSION_POLICY); } const tableSchemas: Record = {}; - for (const [accName, table] of Object.entries(tables)) { + for (const [accName, entry] of Object.entries(entries)) { + if (entry instanceof Schema) { + throw new TypeError( + `schema entry '${accName}' looks like a default import; use \`import * as ${accName} from '...'\` so the submodule can see the library's named reducer exports.` + ); + } + if (isSubmoduleNamespace(entry)) { + const { rawDef, dispatch } = + entry.default.buildSubmoduleDispatch(entry); + dispatch.namespace = accName; + ctx.addSubmodule({ namespace: accName, module: rawDef }); + ctx.submoduleDispatchInfos.push(dispatch); + continue; + } + if (!isUntypedTableSchema(entry)) { + throw new TypeError( + `schema entry '${accName}' must be a table or a submodule namespace object` + ); + } + + const table = entry; const tableDef = table.tableDef(ctx, accName); tableSchemas[accName] = tableToSchema(accName, table, tableDef); const tableSourceNames = ctx.tableSourceNames.get(table); @@ -737,7 +884,7 @@ export function schema>( }); } } - return { tables: tableSchemas } as TablesToSchema; + return { tables: tableSchemas } as SchemaDefForEntries; }); return new Schema(ctx); diff --git a/crates/bindings-typescript/src/server/views.ts b/crates/bindings-typescript/src/server/views.ts index 06b3b94dedc..c7cb9ca0b4d 100644 --- a/crates/bindings-typescript/src/server/views.ts +++ b/crates/bindings-typescript/src/server/views.ts @@ -51,7 +51,7 @@ export function makeViewExport< fn.bind() as ViewExport; viewExport[exportContext] = ctx; viewExport[registerExport] = (ctx, exportName) => { - registerView(ctx, opts, exportName, false, params, ret, fn); + registerView(ctx, opts, exportName, params, ret, fn); }; return viewExport; } @@ -73,7 +73,7 @@ export function makeAnonViewExport< fn.bind() as ViewExport; viewExport[exportContext] = ctx; viewExport[registerExport] = (ctx, exportName) => { - registerView(ctx, opts, exportName, true, params, ret, fn); + registerAnonymousView(ctx, opts, exportName, params, ret, fn); }; return viewExport; } @@ -194,19 +194,57 @@ export type ViewReturnTypeBuilder = export function registerView< S extends UntypedSchemaDef, - const Anonymous extends boolean, Params extends ParamsObj, Ret extends ViewReturnTypeBuilder, >( ctx: SchemaInner, opts: ViewOpts, exportName: string, - anon: Anonymous, params: Params, ret: Ret, - fn: Anonymous extends true - ? AnonymousViewFn - : ViewFn + fn: ViewFn +) { + const described = describeView(ctx, opts, exportName, false, params, ret); + // `ctx.views` is schema-erased. `ViewCtx` and `ViewCtx` describe the same + // shape, but TypeScript cannot relate two instantiations of the mapped type + // `ReadonlyDbView` while the schema is still a type parameter, so erasing `S` here + // needs an assertion. It is sound because the runtime builds the context it passes + // back in from this very schema. + ctx.views.push(buildViewInfo(ctx, described, fn as AnyViewFn)); +} + +export function registerAnonymousView< + S extends UntypedSchemaDef, + Params extends ParamsObj, + Ret extends ViewReturnTypeBuilder, +>( + ctx: SchemaInner, + opts: ViewOpts, + exportName: string, + params: Params, + ret: Ret, + fn: AnonymousViewFn +) { + const described = describeView(ctx, opts, exportName, true, params, ret); + // Schema-erased for the same reason as `registerView` above. + ctx.anonViews.push(buildViewInfo(ctx, described, fn as AnyAnonymousViewFn)); +} + +/** + * The flavor-independent part of registering a view: register its types, record + * it in the module def, and validate its primary key. Everything here is the + * same for regular and anonymous views, so it is factored out of both. + */ +function describeView< + Params extends ParamsObj, + Ret extends ViewReturnTypeBuilder, +>( + ctx: SchemaInner, + opts: ViewOpts, + exportName: string, + anon: boolean, + params: Params, + ret: Ret ) { ctx.defineFunction(exportName); const paramsBuilder = new RowBuilder(params, toPascalCase(exportName)); @@ -214,8 +252,6 @@ export function registerView< // Register return types if they are product types let returnType = ctx.registerTypesRecursively(ret).algebraicType; - const { typespace } = ctx; - const { value: paramType } = ctx.resolveType( ctx.registerTypesRecursively(paramsBuilder) ); @@ -255,24 +291,39 @@ export function registerView< }); } - // If it is an option, we wrap the function to make the return look like an array. + // An option-returning view is presented to the host as an array of zero or one + // rows, so its function needs wrapping and its return type rewriting. + const wrapOption = returnType.tag == 'Sum'; + // Tested directly rather than via `wrapOption` so that TypeScript narrows `returnType`. if (returnType.tag == 'Sum') { - const originalFn = fn; - fn = ((ctx: ViewCtx, args: InferTypeOfRow) => { - const ret = originalFn(ctx, args); - return ret == null ? [] : [ret]; - }) as any; returnType = AlgebraicType.Array( returnType.value.variants[0].algebraicType ); } - (anon ? ctx.anonViews : ctx.views).push({ - fn, + return { paramType, returnType, wrapOption }; +} + +// Build the stored `ViewInfo` for `fn`. Generic over the stored function type so +// the regular and anonymous cases each keep their own context shape here, instead +// of being collapsed into one type by a cast. +function buildViewInfo any>( + ctx: SchemaInner, + { paramType, returnType, wrapOption }: ReturnType, + fn: F +): ViewInfo { + const { typespace } = ctx; + return { + fn: wrapOption + ? (((viewCtx, args) => { + const ret = fn(viewCtx, args); + return ret == null ? [] : [ret]; + }) as F) + : fn, deserializeParams: ProductType.makeDeserializer(paramType, typespace), serializeReturn: AlgebraicType.makeSerializer(returnType, typespace), returnTypeBaseSize: bsatnBaseSize(typespace, returnType), - }); + }; } // Inspect the returned row builder and collect the column property names marked diff --git a/crates/bindings-typescript/tests/ctx_as.test.ts b/crates/bindings-typescript/tests/ctx_as.test.ts new file mode 100644 index 00000000000..e764d881b76 --- /dev/null +++ b/crates/bindings-typescript/tests/ctx_as.test.ts @@ -0,0 +1,97 @@ +import { beforeAll, describe, expect, it, vi } from 'vitest'; + +vi.mock( + 'spacetime:sys@2.0', + () => ({ + moduleHooks: Symbol('moduleHooks'), + table_id_from_name: () => 1, + index_id_from_name: () => 1, + row_iter_bsatn_close: () => {}, + }), + { virtual: true } +); + +describe('ctx.as alias proxy', () => { + let schema: typeof import('../src/server/schema').schema; + let table: typeof import('../src/lib/table').table; + let t: typeof import('../src/lib/type_builders').t; + let moduleHooks: symbol; + + beforeAll(async () => { + ({ schema } = await import('../src/server/schema')); + ({ table } = await import('../src/lib/table')); + ({ t } = await import('../src/lib/type_builders')); + ({ moduleHooks } = (await import('spacetime:sys@2.0')) as any); + }); + + it('ctx.as. provides a narrowed ctx with library db and delegating sender', async () => { + const sessions = table( + { name: 'sessions' }, + { id: t.u64().primaryKey().autoInc() } + ); + const authSchema = schema({ sessions }); + const authLib = { default: authSchema }; + + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + const consumer = schema({ players, myauth: authLib }); + + let capturedCtx: any; + const myReducer = consumer.reducer((ctx: any) => { + capturedCtx = ctx; + }); + + const hooks = (consumer as any)[moduleHooks]({ myReducer }); + hooks.__call_reducer__(0, 0n, 0n, 0n, new DataView(new ArrayBuffer(0))); + + expect(capturedCtx.as).toBeDefined(); + expect(capturedCtx.as.myauth).toBeDefined(); + expect(capturedCtx.as.myauth.db).toBeDefined(); + expect(capturedCtx.as.myauth.db.sessions).toBeDefined(); + expect(capturedCtx.as.myauth.sender).toBe(capturedCtx.sender); + expect(capturedCtx.as.myauth.timestamp).toBe(capturedCtx.timestamp); + }); + + it('ctx.as is empty object when there are no submodules', async () => { + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + const consumer = schema({ players }); + + let capturedCtx: any; + const myReducer = consumer.reducer((ctx: any) => { + capturedCtx = ctx; + }); + + const hooks = (consumer as any)[moduleHooks]({ myReducer }); + hooks.__call_reducer__(0, 0n, 0n, 0n, new DataView(new ArrayBuffer(0))); + + expect(capturedCtx.as).toBeDefined(); + expect(Object.keys(capturedCtx.as)).toHaveLength(0); + }); + + it('ctx.as..as carries nested submodule aliases', async () => { + const bazTable = table({ name: 'baz_items' }, { id: t.u32().primaryKey() }); + const bazSchema = schema({ bazTable }); + const bazLib = { default: bazSchema }; + + const sessions = table( + { name: 'sessions' }, + { id: t.u64().primaryKey().autoInc() } + ); + const authSchema = schema({ sessions, baz: bazLib }); + const authLib = { default: authSchema }; + + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + const consumer = schema({ players, myauth: authLib }); + + let capturedCtx: any; + const myReducer = consumer.reducer((ctx: any) => { + capturedCtx = ctx; + }); + + const hooks = (consumer as any)[moduleHooks]({ myReducer }); + hooks.__call_reducer__(0, 0n, 0n, 0n, new DataView(new ArrayBuffer(0))); + + expect(capturedCtx.as.myauth.as.baz).toBeDefined(); + expect(capturedCtx.as.myauth.as.baz.db.bazTable).toBeDefined(); + expect(capturedCtx.as.myauth.as.baz.sender).toBe(capturedCtx.sender); + }); +}); diff --git a/crates/bindings-typescript/tests/db_connection.test.ts b/crates/bindings-typescript/tests/db_connection.test.ts index e7fb4215079..ae4fa365032 100644 --- a/crates/bindings-typescript/tests/db_connection.test.ts +++ b/crates/bindings-typescript/tests/db_connection.test.ts @@ -419,6 +419,38 @@ describe('DbConnection', () => { expect(wsAdapter.closed).toBeFalsy(); }); + test('subscribe() accepts a query-builder callback and sends the built SQL', async () => { + const wsAdapter = new WebsocketTestAdapter(); + const client = DbConnection.builder() + .withUri('ws://127.0.0.1:1234') + .withDatabaseName('db') + .withWSFn(wsAdapter.openWebSocket) + .build(); + + await client['wsPromise']; + wsAdapter.acceptConnection(); + wsAdapter.sendToClient( + ServerMessage.InitialConnection({ + identity: anIdentity, + token: 'a-token', + connectionId: ConnectionId.random(), + }) + ); + + client.subscriptionBuilder().subscribe(tables => tables.player.build()); + + await Promise.resolve(); + const subscribeMessage = wsAdapter.outgoingMessages.find( + message => message.tag === 'Subscribe' + ); + if (subscribeMessage?.tag !== 'Subscribe') { + throw new Error('No Subscribe message found in messageQueue.'); + } + expect(subscribeMessage.value.queryStrings).toEqual([ + 'SELECT * FROM "player"', + ]); + }); + test('fires row callbacks after reducer resolution in ReducerResult', async () => { const wsAdapter = new WebsocketTestAdapter(); const onConnectPromise = new Deferred(); diff --git a/crates/bindings-typescript/tests/query.test.ts b/crates/bindings-typescript/tests/query.test.ts index 929506b28b1..be9858578f4 100644 --- a/crates/bindings-typescript/tests/query.test.ts +++ b/crates/bindings-typescript/tests/query.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { Identity } from '../src/lib/identity'; import { makeQueryBuilder, + makeFromBuilder, and, or, not, @@ -393,3 +394,41 @@ describe('TableScan.toSql', () => { ); }); }); + +describe('makeFromBuilder', () => { + // Exercises the query builder used by `subscriptionBuilder().subscribe(tables => ...)`: + // root tables (no dot in `sourceName`) are exposed directly, while namespaced/submodule + // tables (dotted `sourceName`, e.g. `"lib.orders"`) are grouped under a sub-object keyed + // by the namespace, so two tables with the same accessor name in different namespaces + // don't collide. + it('exposes root-level tables (no dot in sourceName) directly', () => { + const from = makeFromBuilder(schemaDef.tables); + const sql = toSql(from.person.build()); + + expect(sql).toBe('SELECT * FROM "person"'); + }); + + it('groups a namespaced table (dotted sourceName) under its namespace', () => { + const namespacedTables = { + ...schemaDef.tables, + orders: { ...schemaDef.tables.orders, sourceName: 'lib.orders' }, + }; + const from = makeFromBuilder(namespacedTables) as any; + + expect(from.orders).toBeUndefined(); + const sql = toSql(from.lib.orders.build()); + expect(sql).toBe('SELECT * FROM "lib"."orders"'); + }); + + it('keeps a root table with the same accessor name distinct from a namespaced one', () => { + const namespacedTables = { + ...schemaDef.tables, + orders: { ...schemaDef.tables.orders, sourceName: 'lib.orders' }, + }; + const from = makeFromBuilder(namespacedTables) as any; + + // Root `person` is unaffected by the `lib.orders` namespace grouping. + expect(toSql(from.person.build())).toBe('SELECT * FROM "person"'); + expect(toSql(from.lib.orders.build())).toBe('SELECT * FROM "lib"."orders"'); + }); +}); diff --git a/crates/bindings-typescript/tests/schema_submodules.test.ts b/crates/bindings-typescript/tests/schema_submodules.test.ts new file mode 100644 index 00000000000..7c430ec3dc1 --- /dev/null +++ b/crates/bindings-typescript/tests/schema_submodules.test.ts @@ -0,0 +1,220 @@ +import { beforeAll, describe, expect, it, vi } from 'vitest'; + +vi.mock( + 'spacetime:sys@2.0', + () => ({ + moduleHooks: Symbol('moduleHooks'), + }), + { virtual: true } +); + +vi.mock('../src/server/runtime', () => ({ + makeHooks: () => ({}), + callProcedure: () => new Uint8Array(), + callUserFunction: (fn: (...args: any[]) => any, ...args: any[]) => + fn(...args), + ReducerCtxImpl: class {}, + sys: { + row_iter_bsatn_close: () => {}, + }, +})); + +describe('schema submodules', () => { + let schema: typeof import('../src/server/schema').schema; + let table: typeof import('../src/lib/table').table; + let t: typeof import('../src/lib/type_builders').t; + + beforeAll(async () => { + ({ schema } = await import('../src/server/schema')); + ({ table } = await import('../src/lib/table')); + ({ t } = await import('../src/lib/type_builders')); + }); + + it('emits submodule module defs and resolves submodule schedules', () => { + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + + const sessionCleanupTick = table( + { + name: 'session_cleanup_tick', + scheduled: (): any => cleanExpiredSessions, + }, + { + scheduledId: t.u64().primaryKey().autoInc(), + scheduledAt: t.scheduleAt(), + } + ); + + const sessions = table( + { name: 'sessions' }, + { + id: t.u64().primaryKey().autoInc(), + } + ); + + const authSchema = schema({ + sessions, + sessionCleanupTick, + }); + + const cleanExpiredSessions = authSchema.reducer(() => {}); + const authLib = { + default: authSchema, + cleanExpiredSessions, + }; + + const consumer = schema({ + players, + myauth: authLib, + }); + + const raw = consumer.buildRawModuleDefV10({}); + const submodules = raw.sections.find( + section => section.tag === 'Submodules' + )?.value; + + expect(submodules).toHaveLength(1); + expect(submodules?.[0]?.namespace).toBe('myauth'); + + const submoduleSections = submodules?.[0]?.module.sections ?? []; + const submoduleReducers = submoduleSections.find( + section => section.tag === 'Reducers' + )?.value; + const submoduleSchedules = submoduleSections.find( + section => section.tag === 'Schedules' + )?.value; + + expect(submoduleReducers).toEqual( + expect.arrayContaining([ + expect.objectContaining({ sourceName: 'cleanExpiredSessions' }), + ]) + ); + expect(submoduleSchedules).toEqual([ + expect.objectContaining({ + tableName: 'sessionCleanupTick', + functionName: 'cleanExpiredSessions', + }), + ]); + }); + + it('rejects default-import style submodules with a clear error', () => { + const sessions = table( + { name: 'sessions' }, + { + id: t.u64().primaryKey().autoInc(), + } + ); + + const authSchema = schema({ sessions }); + + expect(() => + schema({ + myauth: authSchema as any, + }) + ).toThrow(/looks like a default import/); + }); + + it('populates submoduleDispatchInfos with reducer fns and table metadata', () => { + const sessions = table( + { name: 'sessions' }, + { id: t.u64().primaryKey().autoInc() } + ); + + const authSchema = schema({ sessions }); + const cleanExpiredSessions = authSchema.reducer(() => {}); + const authLib = { default: authSchema, cleanExpiredSessions }; + + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + const consumer = schema({ players, myauth: authLib }); + + const infos = consumer.submoduleDispatchInfos; + expect(infos).toHaveLength(1); + + const info = infos[0]; + expect(info.reducerFns).toHaveLength(1); + expect(info.reducerDefs).toHaveLength(1); + expect(info.reducerDefs[0].sourceName).toBe('cleanExpiredSessions'); + expect(info.tables).toHaveLength(1); + expect(info.tables[0].accessorName).toBe('sessions'); + expect(info.subDispatches).toHaveLength(0); + }); + + it('flattens nested submodule dispatches depth-first', () => { + // baz library: 1 reducer + const bazTable = table({ name: 'baz_items' }, { id: t.u32().primaryKey() }); + const bazSchema = schema({ bazTable }); + const bazReducer = bazSchema.reducer(() => {}); + const bazLib = { default: bazSchema, bazReducer }; + + // auth library: 1 own reducer, uses baz as a submodule + const sessions = table( + { name: 'sessions' }, + { id: t.u64().primaryKey().autoInc() } + ); + const authSchema = schema({ sessions, baz: bazLib }); + const authReducer = authSchema.reducer(() => {}); + const authLib = { default: authSchema, authReducer }; + + // consumer: 1 own reducer, uses auth as a submodule + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + const consumer = schema({ players, myauth: authLib }); + const consumerReducer = consumer.reducer(() => {}); + + // Verify depth-first structure: + // consumer.submoduleDispatchInfos[0] = myauth (authReducer) + // consumer.submoduleDispatchInfos[0].subDispatches[0] = myauth.baz (bazReducer) + const infos = consumer.submoduleDispatchInfos; + expect(infos).toHaveLength(1); + + const authInfo = infos[0]; + expect(authInfo.reducerFns).toHaveLength(1); + expect(authInfo.reducerDefs[0].sourceName).toBe('authReducer'); + expect(authInfo.subDispatches).toHaveLength(1); + + const bazInfo = authInfo.subDispatches[0]; + expect(bazInfo.reducerFns).toHaveLength(1); + expect(bazInfo.reducerDefs[0].sourceName).toBe('bazReducer'); + expect(bazInfo.subDispatches).toHaveLength(0); + + // Unused variable check + void consumerReducer; + }); + + it('submoduleDispatchInfos carry namespace and nested namespace dispatches propagate', () => { + const sessions = table( + { name: 'sessions' }, + { id: t.u64().primaryKey().autoInc() } + ); + const authSchema = schema({ sessions }); + const authLib = { default: authSchema }; + + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + const consumer = schema({ players, myauth: authLib }); + + const infos = consumer.submoduleDispatchInfos; + expect(infos).toHaveLength(1); + expect(infos[0].namespace).toBe('myauth'); + expect(infos[0].tables[0].accessorName).toBe('sessions'); + }); + + it('nested submodules carry their own namespace on subDispatches', () => { + const bazTable = table({ name: 'baz_items' }, { id: t.u32().primaryKey() }); + const bazSchema = schema({ bazTable }); + const bazLib = { default: bazSchema }; + + const sessions = table( + { name: 'sessions' }, + { id: t.u64().primaryKey().autoInc() } + ); + const authSchema = schema({ sessions, baz: bazLib }); + const authLib = { default: authSchema }; + + const players = table({ name: 'players' }, { id: t.u32().primaryKey() }); + const consumer = schema({ players, myauth: authLib }); + + const authInfo = consumer.submoduleDispatchInfos[0]; + expect(authInfo.namespace).toBe('myauth'); + expect(authInfo.subDispatches).toHaveLength(1); + expect(authInfo.subDispatches[0].namespace).toBe('baz'); + expect(authInfo.subDispatches[0].tables[0].accessorName).toBe('bazTable'); + }); +}); diff --git a/crates/cli/src/api.rs b/crates/cli/src/api.rs index d40b03ee87e..22cdd6c2ae1 100644 --- a/crates/cli/src/api.rs +++ b/crates/cli/src/api.rs @@ -4,9 +4,11 @@ use std::ops::Add; use reqwest::{header, Client, RequestBuilder}; use serde::Deserialize; +use spacetimedb_lib::db::raw_def::v10::RawModuleDefV10; use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; use spacetimedb_lib::de::serde::DeserializeWrapper; use spacetimedb_lib::Identity; +use spacetimedb_schema::def::ModuleDef; use crate::util::{AuthHeader, ResponseExt}; @@ -61,13 +63,30 @@ impl ClientApi { } /// Reads the `ModuleDef` from the `schema` endpoint. - pub async fn module_def(&self) -> anyhow::Result { + /// + /// Requests schema version 10 (required to see submodules). If the server rejects + /// that with a client error (e.g. servers predating V10 don't recognize the version) + /// we fall back to version 9 and upgrade the result. + pub async fn module_def(&self) -> anyhow::Result { let res = self .client .get(self.con.db_uri("schema")) - .query(&[("version", "9")]) + .query(&[("version", "10")]) .send() .await?; + if res.status().is_client_error() { + let res9 = self + .client + .get(self.con.db_uri("schema")) + .query(&[("version", "9")]) + .send() + .await?; + if res9.status().is_success() { + let DeserializeWrapper::(module_def) = res9.json_or_error().await?; + let module_def: ModuleDef = module_def.try_into()?; + return Ok(module_def.into()); + } + } let DeserializeWrapper(module_def) = res.json_or_error().await?; Ok(module_def) } diff --git a/crates/cli/src/subcommands/build.rs b/crates/cli/src/subcommands/build.rs index 63cf665150a..17106d55e45 100644 --- a/crates/cli/src/subcommands/build.rs +++ b/crates/cli/src/subcommands/build.rs @@ -3,7 +3,9 @@ use crate::util::find_module_path; use crate::Config; use clap::ArgAction::SetTrue; use clap::{Arg, ArgMatches}; +use colored::Colorize; use std::ffi::OsString; +use std::io::IsTerminal; use std::path::{Path, PathBuf}; pub fn cli() -> clap::Command { @@ -98,6 +100,25 @@ pub fn run_build( native_aot, dotnet_version, )?; + + // For TypeScript modules, extract the schema from the bundle and emit + // advisory warnings (e.g. HTTP handlers in mounted sub-modules are unsupported). + // If the standalone binary is unavailable or extraction fails, skip silently. + if result.1 == "Js" + && let Ok(module_def) = crate::subcommands::generate::extract_descriptions(&result.0) + { + let tty = std::io::stderr().is_terminal(); + for warning in module_def.collect_warnings() { + let prefix = if tty { + "warning:".yellow().to_string() + } else { + "warning:".to_string() + }; + eprintln!("{prefix} {warning}"); + println!(); + } + } + println!("Build finished successfully."); Ok(result) diff --git a/crates/cli/src/subcommands/call.rs b/crates/cli/src/subcommands/call.rs index 3e5f8840aa8..9dfb44bc643 100644 --- a/crates/cli/src/subcommands/call.rs +++ b/crates/cli/src/subcommands/call.rs @@ -45,14 +45,8 @@ enum CallDef<'a> { impl<'a> CallDef<'a> { fn params(&self) -> &'a sats::ProductType { match self { - CallDef::Reducer(reducer_def) => &reducer_def.params, - CallDef::Procedure(procedure_def) => &procedure_def.params, - } - } - fn name(&self) -> &str { - match self { - CallDef::Reducer(reducer_def) => &reducer_def.name, - CallDef::Procedure(procedure_def) => &procedure_def.name, + CallDef::Reducer(r) => &r.params, + CallDef::Procedure(p) => &p.params, } } fn kind(&self) -> &str { @@ -103,10 +97,13 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), Error> { let module_def: ModuleDef = api.module_def().await?.try_into()?; - let call_def = match module_def.reducer(&**reducer_procedure_name) { - Some(reducer_def) => CallDef::Reducer(reducer_def), - None => match module_def.procedure(&**reducer_procedure_name) { - Some(procedure_def) => CallDef::Procedure(procedure_def), + // Dot-qualified names (e.g. `lib.my_reducer`) route to submodules. + // Keep the owning module def around: type refs in the params must be + // resolved against the owning module's typespace, not the consumer's. + let (call_def, owning_def) = match module_def.reducer_by_name_with_module(reducer_procedure_name) { + Some((_, reducer_def, owning_def)) => (CallDef::Reducer(reducer_def), owning_def), + None => match module_def.procedure_by_name_with_module(reducer_procedure_name) { + Some((_, procedure_def, owning_def)) => (CallDef::Procedure(procedure_def), owning_def), None => { return Err(anyhow::Error::msg(no_such_reducer_or_procedure( &database_identity, @@ -152,7 +149,14 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), Error> { if response_text.starts_with("no such reducer") || response_text.starts_with("no such procedure") { no_such_reducer_or_procedure(&database_identity, database, reducer_procedure_name, &module_def) } else if response_text.starts_with("invalid arguments") { - invalid_arguments(&database_identity, database, &response_text, &module_def, call_def) + invalid_arguments( + &database_identity, + database, + &response_text, + owning_def.typespace(), + reducer_procedure_name, + call_def, + ) } else { return error; }; @@ -169,11 +173,20 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), Error> { } /// Returns an error message for when `reducer` is called with wrong arguments. -fn invalid_arguments(identity: &Identity, db: &str, text: &str, module_def: &ModuleDef, call_def: CallDef) -> String { +/// +/// `full_name` is the (possibly dot-qualified) name the user invoked. +fn invalid_arguments( + identity: &Identity, + db: &str, + text: &str, + typespace: &Typespace, + full_name: &str, + call_def: CallDef, +) -> String { let mut error = format!( "Invalid arguments provided for {} `{}` for database `{}` resolving to identity `{}`.", call_def.kind(), - call_def.name(), + full_name, db, identity ); @@ -190,7 +203,10 @@ fn invalid_arguments(identity: &Identity, db: &str, text: &str, module_def: &Mod error, "\n\nThe {} has the following signature:\n\t{}", call_def.kind(), - CallSignature(module_def.typespace().with_type(&call_def)) + CallSignature { + name: full_name, + call: typespace.with_type(&call_def) + } ) .unwrap(); @@ -218,13 +234,16 @@ fn split_at_first_substring<'t>(text: &'t str, substring: &str) -> Option<(&'t s /// Provided the `schema_json` for the database, /// returns the signature for a reducer OR procedure with `name`. -struct CallSignature<'a>(sats::WithTypespace<'a, CallDef<'a>>); +struct CallSignature<'a> { + name: &'a str, + call: sats::WithTypespace<'a, CallDef<'a>>, +} impl std::fmt::Display for CallSignature<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let call_def = self.0.ty(); - let typespace = self.0.typespace(); + let call_def = self.call.ty(); + let typespace = self.call.typespace(); - write!(f, "{}(", call_def.name())?; + write!(f, "{}(", self.name)?; // Print the arguments to `args`. let mut comma = false; @@ -258,17 +277,24 @@ const CALL_PRINT_LIMIT: usize = 10; /// Provided the schema for the database, /// decorate `error` with more helpful info about reducers and procedures. +/// +/// Reducers and procedures from submodules are listed under their +/// full dot-qualified names (e.g. `lib.my_reducer`). fn add_reducer_procedure_ctx_to_err(error: &mut String, module_def: &ModuleDef, reducer_name: &str) { - let reducers = module_def - .reducers() - .filter(|reducer| reducer.lifecycle.is_none()) - .map(|reducer| &*reducer.name) + let reducer_names = module_def + .all_reducers_with_prefix() + .into_iter() + .filter(|(_, _, r)| r.lifecycle.is_none()) + .map(|(_, _, r)| r.name.to_string()) .collect::>(); + let reducers = reducer_names.iter().map(String::as_str).collect::>(); - let procedures = module_def - .procedures() - .map(|reducer| &*reducer.name) + let procedure_names = module_def + .all_procedures_with_prefix() + .into_iter() + .map(|(prefix, _, p)| format!("{prefix}{}", &*p.name)) .collect::>(); + let procedures = procedure_names.iter().map(String::as_str).collect::>(); if let Some(best) = find_best_match_for_name(&reducers, reducer_name, None) { write!(error, "\n\nA reducer with a similar name exists: `{best}`").unwrap(); diff --git a/crates/cli/src/subcommands/describe.rs b/crates/cli/src/subcommands/describe.rs index e774224855c..1f841187428 100644 --- a/crates/cli/src/subcommands/describe.rs +++ b/crates/cli/src/subcommands/describe.rs @@ -6,7 +6,9 @@ use crate::util::UNSTABLE_WARNING; use crate::util::{database_identity, get_auth_header}; use anyhow::Context; use clap::{Arg, ArgAction, ArgMatches}; +use spacetimedb_lib::db::raw_def::v10::{RawReducerDefV10, RawTableDefV10}; use spacetimedb_lib::sats; +use spacetimedb_schema::def::ModuleDef; pub fn cli() -> clap::Command { clap::Command::new("describe") @@ -97,30 +99,31 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error }; let api = ClientApi::new(conn); - let module_def = api.module_def().await?; + let raw = api.module_def().await?; if json { fn sats_to_json(v: &T) -> serde_json::Result { serde_json::to_string_pretty(sats::serde::SerdeWrapper::from_ref(v)) } let json = match entity { + // Entity lookups go through the validated `ModuleDef`, which resolves + // canonical names and dot-qualified names from submodules + // (e.g. `lib.my_reducer`). Some((EntityType::Reducer, reducer_name)) => { - let reducer = module_def - .reducers - .iter() - .find(|r| *r.name == *reducer_name) - .context("no such reducer")?; - sats_to_json(reducer)? + let module_def: ModuleDef = raw.try_into()?; + let (_, reducer) = module_def.reducer_by_name(reducer_name).context("no such reducer")?; + sats_to_json(&RawReducerDefV10::from(reducer.clone()))? } Some((EntityType::Table, table_name)) => { - let table = module_def - .tables - .iter() - .find(|t| *t.name == *table_name) + let module_def: ModuleDef = raw.try_into()?; + let (_, _, table) = module_def + .all_tables_with_prefix() + .into_iter() + .find(|(prefix, _, t)| format!("{}{}", prefix, &*t.name) == *table_name) .context("no such table")?; - sats_to_json(table)? + sats_to_json(&RawTableDefV10::from(table.clone()))? } - None => sats_to_json(&module_def)?, + None => sats_to_json(&raw)?, }; // TODO: validate the JSON output diff --git a/crates/cli/src/subcommands/subscribe.rs b/crates/cli/src/subcommands/subscribe.rs index 8fc6f18abda..8b26700167e 100644 --- a/crates/cli/src/subcommands/subscribe.rs +++ b/crates/cli/src/subcommands/subscribe.rs @@ -7,12 +7,13 @@ use reqwest::Url; use serde_json::Value; use spacetimedb_client_api_messages::websocket::{common as ws_common, v1 as ws_v1, v2 as ws_v2, v3 as ws_v3}; use spacetimedb_data_structures::map::HashMap; -use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9; use spacetimedb_lib::de::serde::{DeserializeWrapper, SeedWrapper}; use spacetimedb_lib::de::DeserializeSeed as BsatnDeserializeSeed; use spacetimedb_lib::sats::WithTypespace; use spacetimedb_lib::ser::serde::SerializeWrapper; use spacetimedb_lib::{bsatn, AlgebraicType}; +use spacetimedb_schema::def::ModuleDef; +use spacetimedb_schema::identifier::NamespacePath; use std::collections::VecDeque; use std::io; use std::sync::atomic::AtomicU32; @@ -116,7 +117,7 @@ impl SubscribeConnection { } /// Wait for the initial subscription result and optionally print it. - async fn await_initial_update(&mut self, module_def: Option<&RawModuleDefV9>) -> Result<(), Error> { + async fn await_initial_update(&mut self, module_def: Option<&ModuleDef>) -> Result<(), Error> { match self { Self::V3 { ws, pending } => await_initial_update_v3(ws, pending, module_def).await, Self::V1 { ws } => await_initial_update_v1(ws, module_def).await, @@ -127,7 +128,7 @@ impl SubscribeConnection { async fn consume_transaction_updates( &mut self, num: Option, - module_def: &RawModuleDefV9, + module_def: &ModuleDef, num_received: &UpdateCounter, ) -> Result<(), Error> { match self { @@ -183,7 +184,7 @@ pub async fn exec(config: Config, args: &ArgMatches) -> Result<(), anyhow::Error database: resolved.database.clone(), }; let api = ClientApi::new(conn); - let module_def = api.module_def().await?; + let module_def: ModuleDef = api.module_def().await?.try_into()?; let mut conn = connect_with_fallback(&api, confirmed).await?; let num_received = UpdateCounter::new(); @@ -450,7 +451,7 @@ fn parse_msg_json(msg: &WsMessage) -> Option(ws: &mut S, module_def: Option<&RawModuleDefV9>) -> Result<(), Error> +async fn await_initial_update_v1(ws: &mut S, module_def: Option<&ModuleDef>) -> Result<(), Error> where S: TryStream + Unpin, { @@ -494,7 +495,7 @@ where async fn await_initial_update_v3( ws: &mut S, pending: &mut VecDeque, - module_def: Option<&RawModuleDefV9>, + module_def: Option<&ModuleDef>, ) -> Result<(), Error> where S: TryStream + Unpin, @@ -530,7 +531,7 @@ where async fn consume_transaction_updates_v1( ws: &mut S, num: Option, - module_def: &RawModuleDefV9, + module_def: &ModuleDef, num_received: &UpdateCounter, ) -> Result<(), Error> where @@ -575,7 +576,7 @@ async fn consume_transaction_updates_v3( ws: &mut S, pending: &mut VecDeque, num: Option, - module_def: &RawModuleDefV9, + module_def: &ModuleDef, num_received: &UpdateCounter, ) -> Result<(), Error> where @@ -665,25 +666,19 @@ fn decode_server_payload(msg: Bytes, pending: &mut VecDeque, - schema: &RawModuleDefV9, -) -> Result { +fn format_output_json_v1(msg: &ws_v1::DatabaseUpdate, schema: &ModuleDef) -> Result { let formatted = reformat_update_v1(msg, schema).map_err(|source| Error::Reformat { source })?; format_output_json_from_tables(&formatted) } /// Format initial v3 subscription rows using the CLI's existing JSON output shape. -fn format_output_json_query_rows(msg: &ws_v2::QueryRows, schema: &RawModuleDefV9) -> Result { +fn format_output_json_query_rows(msg: &ws_v2::QueryRows, schema: &ModuleDef) -> Result { let formatted = reformat_query_rows(msg, schema).map_err(|source| Error::Reformat { source })?; format_output_json_from_tables(&formatted) } /// Format a v3 transaction update using the CLI's existing JSON output shape. -fn format_output_json_transaction_update( - msg: &ws_v2::TransactionUpdate, - schema: &RawModuleDefV9, -) -> Result { +fn format_output_json_transaction_update(msg: &ws_v2::TransactionUpdate, schema: &ModuleDef) -> Result { let formatted = reformat_transaction_update(msg, schema).map_err(|source| Error::Reformat { source })?; format_output_json_from_tables(&formatted) } @@ -694,19 +689,41 @@ fn format_output_json_from_tables(formatted: &HashMap<&str, SubscriptionTable>) Ok(output) } +/// Resolve the row type for a table or view by its wire name. +/// +/// Wire names of tables and views from submodules are dot-qualified +/// (e.g. `lib.my_table`). The returned type is resolved against the owning +/// module's typespace, since that is where its type refs point. +fn type_for_table_like<'a>(module_def: &'a ModuleDef, name: &str) -> Option> { + let matches_wire_name = |prefix: &NamespacePath, plain_name: &str| format!("{prefix}{plain_name}") == name; + + let (owning_def, type_ref) = module_def + .all_tables_with_prefix() + .into_iter() + .find_map(|(prefix, owning_def, table)| { + matches_wire_name(&prefix, &table.name).then_some((owning_def, table.product_type_ref)) + }) + .or_else(|| { + module_def + .all_views_with_prefix() + .into_iter() + .find_map(|(prefix, owning_def, view)| { + matches_wire_name(&prefix, &view.name).then_some((owning_def, view.product_type_ref)) + }) + })?; + + Some(owning_def.typespace().resolve(type_ref)) +} + /// Convert a v1 JSON-format database update to the normalized table output map. fn reformat_update_v1<'a>( msg: &'a ws_v1::DatabaseUpdate, - schema: &RawModuleDefV9, + schema: &ModuleDef, ) -> anyhow::Result> { msg.tables .iter() .map(|upd| { - let table_ty = schema.typespace.resolve( - schema - .type_ref_for_table_like(&upd.table_name) - .context("table not found in schema")?, - ); + let table_ty = type_for_table_like(schema, &upd.table_name).context("table not found in schema")?; let reformat_row = |row: &str| -> anyhow::Result { // TODO: can the following two calls be merged into a single call to reduce allocations? @@ -736,16 +753,12 @@ fn reformat_update_v1<'a>( /// Convert v3 initial subscription rows to the normalized table output map. fn reformat_query_rows<'a>( msg: &'a ws_v2::QueryRows, - schema: &RawModuleDefV9, + schema: &ModuleDef, ) -> anyhow::Result> { let mut formatted = HashMap::default(); for table in &msg.tables { - let table_ty = schema.typespace.resolve( - schema - .type_ref_for_table_like(&table.table) - .context("table not found in schema")?, - ); + let table_ty = type_for_table_like(schema, &table.table).context("table not found in schema")?; let table_output = formatted.entry(&*table.table).or_insert_with(|| SubscriptionTable { deletes: Vec::new(), inserts: Vec::new(), @@ -759,17 +772,13 @@ fn reformat_query_rows<'a>( /// Convert a v3 transaction update to the normalized table output map. fn reformat_transaction_update<'a>( msg: &'a ws_v2::TransactionUpdate, - schema: &RawModuleDefV9, + schema: &ModuleDef, ) -> anyhow::Result> { let mut formatted = HashMap::default(); for query_set in &msg.query_sets { for table in &query_set.tables { - let table_ty = schema.typespace.resolve( - schema - .type_ref_for_table_like(&table.table_name) - .context("table not found in schema")?, - ); + let table_ty = type_for_table_like(schema, &table.table_name).context("table not found in schema")?; let table_output = formatted .entry(&*table.table_name) .or_insert_with(|| SubscriptionTable { diff --git a/crates/codegen/examples/regen-typescript-moduledef.rs b/crates/codegen/examples/regen-typescript-moduledef.rs index 6ebad479090..a3fb1b30f76 100644 --- a/crates/codegen/examples/regen-typescript-moduledef.rs +++ b/crates/codegen/examples/regen-typescript-moduledef.rs @@ -72,6 +72,14 @@ fn main() -> anyhow::Result<()> { let code = regex_replace!(&code, r"SubscriptionBuilderImpl as __SubscriptionBuilderImpl,", r""); let code = regex_replace!(&code, r"TableCache as __TableCache,", r""); let code = regex_replace!(&code, r"ClientCache as __ClientCache,", r""); + // Fix a TypeScript type inference error (TS7022/TS7023) caused by the recursive type + // RawModuleDefV10 -> RawSubmoduleV10 -> RawModuleDefV10. Annotating the `module` getter + // with `: any` breaks the cycle without affecting other generated types. + let code = regex_replace!( + &code, + r"(export const RawSubmoduleV10 = [\s\S]*?)get module\(\) \{", + r"${1}get module(): any {" + ); fs::write(dir.join(filename), code.as_bytes()) })?; diff --git a/crates/codegen/src/cpp.rs b/crates/codegen/src/cpp.rs index 9ddbeae8bb0..76d25e0f947 100644 --- a/crates/codegen/src/cpp.rs +++ b/crates/codegen/src/cpp.rs @@ -47,7 +47,7 @@ impl<'opts> Cpp<'opts> { } fn is_recursive_mount_module_field(&self, type_name: &str, field_name: &str) -> bool { - type_name == "RawModuleMountV10" && field_name == "module" + type_name == "RawSubmoduleV10" && field_name == "module" } fn write_header_comment(&self, output: &mut String) { @@ -180,7 +180,7 @@ impl<'opts> Cpp<'opts> { for (field_name, field_type) in &product.elements { write!(output, " ").unwrap(); if self.is_recursive_mount_module_field(type_name, field_name) { - // Temporary special-case to preserve the recursive RawModuleMountV10 -> + // Temporary special-case to preserve the recursive RawSubmoduleV10 -> // RawModuleDefV10 shape while breaking the include cycle in generated C++. write!(output, "std::shared_ptr<{}::RawModuleDefV10>", self.namespace).unwrap(); } else { @@ -213,7 +213,7 @@ impl<'opts> Cpp<'opts> { writeln!(output, " }}").unwrap(); // Generate equality method - if type_name == "RawModuleMountV10" { + if type_name == "RawSubmoduleV10" { // Pointer equality is sufficient for this internal autogen type. Mounts are not // emitted by the C++ module path yet; this exists to keep the schema shape aligned. writeln!(output, " SPACETIMEDB_PRODUCT_TYPE_EQUALITY(namespace_, module)").unwrap(); @@ -522,7 +522,7 @@ impl Lang for Cpp<'_> { let name = type_def.accessor_name.name(); // Special handling for AlgebraicType due to circular dependencies - if name.to_string() == "AlgebraicType" { + if name == "AlgebraicType" { return vec![OutputFile { filename: format!("{name}.g.h"), code: self.generate_algebraic_type_special(), @@ -541,13 +541,13 @@ impl Lang for Cpp<'_> { let type_name = name.to_string(); for dep in deps { - if dep != type_name && !(type_name == "RawModuleMountV10" && dep == "RawModuleDefV10") { + if dep != type_name && !(type_name == "RawSubmoduleV10" && dep == "RawModuleDefV10") { writeln!(output, "#include \"{}.g.h\"", dep).unwrap(); } } writeln!(output).unwrap(); - if type_name == "RawModuleMountV10" { + if type_name == "RawSubmoduleV10" { writeln!(output, "namespace {} {{", self.namespace).unwrap(); writeln!(output, "struct RawModuleDefV10;").unwrap(); writeln!(output, "}} // namespace {}", self.namespace).unwrap(); diff --git a/crates/codegen/src/lib.rs b/crates/codegen/src/lib.rs index 28d4fb8a5a4..3cdc59142a8 100644 --- a/crates/codegen/src/lib.rs +++ b/crates/codegen/src/lib.rs @@ -1,4 +1,6 @@ +use spacetimedb_lib::db::raw_def::v9::TableAccess; use spacetimedb_schema::def::{ModuleDef, ProcedureDef, ReducerDef, TableDef, TypeDef, ViewDef}; +use spacetimedb_schema::identifier::NamespacePath; use spacetimedb_schema::schema::{Schema, TableSchema}; mod code_indenter; pub mod cpp; @@ -33,10 +35,35 @@ pub fn generate(module: &ModuleDef, lang: &dyn Lang, options: &CodegenOptions) - itertools::chain!( util::iter_tables(module, options.visibility).map(|tbl| lang.generate_table_file(module, tbl)), module.views().map(|view| lang.generate_view_file(module, view)), + // Public tables from submodules + module + .all_tables_with_prefix() + .into_iter() + .filter(|(prefix, _, table)| !prefix.is_empty() && table.table_access == TableAccess::Public) + .map(|(prefix, owning_def, table)| lang.generate_submodule_table_file(owning_def, &prefix, table)), + // Views from submodules (views are currently always public) + module + .all_views_with_prefix() + .into_iter() + .filter(|(prefix, _, _)| !prefix.is_empty()) + .map(|(prefix, owning_def, view)| lang.generate_submodule_view_file(owning_def, &prefix, view)), module.types().flat_map(|typ| lang.generate_type_files(module, typ)), util::iter_reducers(module, options.visibility).map(|reducer| lang.generate_reducer_file(module, reducer)), util::iter_procedures(module, options.visibility) .map(|procedure| lang.generate_procedure_file(module, procedure)), + // Reducers from submodules + module + .all_reducers_with_prefix() + .into_iter() + .filter(|(prefix, _, reducer)| !prefix.is_empty() && !reducer.visibility.is_private()) + .map(|(prefix, owning_def, reducer)| lang.generate_submodule_reducer_file(owning_def, &prefix, reducer)), + // Procedures from submodules + module + .all_procedures_with_prefix() + .into_iter() + .filter(|(prefix, _, procedure)| !prefix.is_empty() && !procedure.visibility.is_private()) + .map(|(prefix, owning_def, procedure)| lang + .generate_submodule_procedure_file(owning_def, &prefix, procedure)), lang.generate_global_files(module, options), ) .collect() @@ -68,4 +95,69 @@ pub trait Lang { .expect("Failed to generate table due to validation errors"); self.generate_table_file_from_schema(module, &tbl, schema) } + + /// Generate a row-type file for a public table from a submodule. + /// Uses `owning_def`'s typespace for type resolution. + /// Filename goes in a subdirectory named after the namespace: + /// e.g. `alias/table_name_table.ts` for namespace `"alias."`, table `tableName`. + fn generate_submodule_table_file( + &self, + owning_def: &ModuleDef, + namespace: &NamespacePath, + table: &TableDef, + ) -> OutputFile { + let schema = TableSchema::from_module_def(owning_def, table, (), 0.into()) + .validated() + .expect("Failed to generate submodule table file"); + let mut file = self.generate_table_file_from_schema(owning_def, table, schema); + let ns_path = namespace.join_segments("/"); + file.filename = format!("{}/{}", ns_path, file.filename); + file + } + + /// Generate a row-type file for a view from a submodule. + /// Filename goes in a subdirectory named after the namespace prefix. + fn generate_submodule_view_file( + &self, + owning_def: &ModuleDef, + namespace: &NamespacePath, + view: &ViewDef, + ) -> OutputFile { + let tbl = TableDef::from(view.clone()); + let schema = TableSchema::from_view_def_for_codegen(owning_def, view) + .validated() + .expect("Failed to generate submodule view file"); + let mut file = self.generate_table_file_from_schema(owning_def, &tbl, schema); + let ns_path = namespace.join_segments("/"); + file.filename = format!("{}/{}", ns_path, file.filename); + file + } + + /// Generate an arg-schema file for a reducer from a submodule. + /// Filename goes in a subdirectory named after the namespace prefix. + fn generate_submodule_reducer_file( + &self, + owning_def: &ModuleDef, + prefix: &NamespacePath, + reducer: &ReducerDef, + ) -> OutputFile { + let mut file = self.generate_reducer_file(owning_def, reducer); + let ns_path = prefix.join_segments("/"); + file.filename = format!("{}/{}", ns_path, file.filename); + file + } + + /// Generate an arg-schema file for a procedure from a submodule. + /// Filename goes in a subdirectory named after the namespace prefix. + fn generate_submodule_procedure_file( + &self, + owning_def: &ModuleDef, + prefix: &NamespacePath, + procedure: &ProcedureDef, + ) -> OutputFile { + let mut file = self.generate_procedure_file(owning_def, procedure); + let ns_path = prefix.join_segments("/"); + file.filename = format!("{}/{}", ns_path, file.filename); + file + } } diff --git a/crates/codegen/src/typescript.rs b/crates/codegen/src/typescript.rs index a2d1cbf8e6c..e2ee468eba4 100644 --- a/crates/codegen/src/typescript.rs +++ b/crates/codegen/src/typescript.rs @@ -6,17 +6,20 @@ use crate::{CodegenOptions, OutputFile}; use super::util::{collect_case, print_auto_generated_file_comment, type_ref_name}; -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::fmt::{self, Write}; use std::iter; use std::ops::Deref; use convert_case::{Case, Casing}; +use spacetimedb_lib::db::raw_def::v9::TableAccess; use spacetimedb_lib::sats::layout::PrimitiveType; use spacetimedb_lib::sats::AlgebraicTypeRef; use spacetimedb_primitives::ColId; -use spacetimedb_schema::def::{ColumnDef, ConstraintDef, IndexDef, ModuleDef, ReducerDef, TableDef, TypeDef}; -use spacetimedb_schema::identifier::Identifier; +use spacetimedb_schema::def::{ + ColumnDef, ConstraintDef, IndexDef, ModuleDef, ProcedureDef, ReducerDef, TableDef, TypeDef, ViewDef, +}; +use spacetimedb_schema::identifier::{Identifier, NamespacePath}; use spacetimedb_schema::reducer_name::ReducerName; use spacetimedb_schema::schema::TableSchema; use spacetimedb_schema::type_for_generate::{AlgebraicTypeDef, AlgebraicTypeUse}; @@ -201,6 +204,67 @@ impl Lang for TypeScript { writeln!(out, "import {table_name_pascalcase}Row from \"./{table_module_name}\";"); } + // Import row types for submodule namespace tables (public only) + let ns_tables: Vec<_> = module + .all_tables_with_prefix() + .into_iter() + .filter(|(prefix, _, table)| !prefix.is_empty() && table.table_access == TableAccess::Public) + .collect(); + let ns_views: Vec<_> = module + .all_views_with_prefix() + .into_iter() + .filter(|(prefix, _, _)| !prefix.is_empty()) + .collect(); + let ns_reducers: Vec<_> = module + .all_reducers_with_prefix() + .into_iter() + .filter(|(prefix, _, reducer)| !prefix.is_empty() && !reducer.visibility.is_private()) + .collect(); + let ns_procedures: Vec<_> = module + .all_procedures_with_prefix() + .into_iter() + .filter(|(prefix, _, procedure)| !prefix.is_empty() && !procedure.visibility.is_private()) + .collect(); + if !ns_tables.is_empty() || !ns_views.is_empty() { + writeln!(out); + writeln!(out, "// Import namespace table schema definitions"); + for (prefix, _, table) in &ns_tables { + let ns_path = submodule_ns_path(prefix); + let file_stem = table_module_name(&table.accessor_name); + let row_type = submodule_row_type_name(prefix, table.accessor_name.deref()); + writeln!(out, "import {row_type}Row from \"./{ns_path}/{file_stem}\";"); + } + for (prefix, _, view) in &ns_views { + let ns_path = submodule_ns_path(prefix); + let file_stem = table_module_name(&view.accessor_name); + let row_type = submodule_row_type_name(prefix, view.accessor_name.deref()); + writeln!(out, "import {row_type}Row from \"./{ns_path}/{file_stem}\";"); + } + } + if !ns_reducers.is_empty() { + writeln!(out); + writeln!(out, "// Import namespace reducer arg schemas"); + for (prefix, _, reducer) in &ns_reducers { + if !is_reducer_invokable(reducer) { + continue; + } + let ns_path = submodule_ns_path(prefix); + let module_name = reducer_module_name(&reducer.accessor_name); + let args_type = submodule_reducer_args_type_name(prefix, &reducer.accessor_name); + writeln!(out, "import {args_type} from \"./{ns_path}/{module_name}\";"); + } + } + if !ns_procedures.is_empty() { + writeln!(out); + writeln!(out, "// Import namespace procedure arg schemas"); + for (prefix, _, procedure) in &ns_procedures { + let ns_path = submodule_ns_path(prefix); + let module_name = procedure_module_name(&procedure.accessor_name); + let args_type = submodule_procedure_args_type_name(prefix, &procedure.accessor_name); + writeln!(out, "import * as {args_type} from \"./{ns_path}/{module_name}\";"); + } + } + writeln!(out); writeln!(out, "/** Type-only namespace exports for generated type groups. */"); @@ -224,7 +288,7 @@ impl Lang for TypeScript { module, out, type_ref, - &table.name, + table.name.deref(), iter_indexes(table), iter_constraints(table), table.is_event, @@ -242,16 +306,61 @@ impl Lang for TypeScript { } writeln!(out, "{view_accessor}: __table({{"); out.indent(1); - write_table_opts(module, out, type_ref, &view.name, iter::empty(), iter::empty(), false); + write_table_opts( + module, + out, + type_ref, + view.name.deref(), + iter::empty(), + iter::empty(), + false, + ); out.dedent(1); writeln!(out, "}}, {}Row),", view_name_pascalcase); } + // Namespace tables from submodules + for (prefix, owning_def, table) in &ns_tables { + let source_name = submodule_source_name(prefix, table.name.deref()); + let row_type = submodule_row_type_name(prefix, table.accessor_name.deref()); + let type_ref = table.product_type_ref; + writeln!(out, "\"{source_name}\": __table({{"); + out.indent(1); + write_table_opts( + owning_def, + out, + type_ref, + &source_name, + iter_indexes(table), + iter_constraints(table), + table.is_event, + ); + out.dedent(1); + writeln!(out, "}}, {row_type}Row),"); + } + // Namespace views from submodules. + // The source name uses the canonical `view.name` (not the accessor) to match the + // backing table name registered in the database by `create_view_with_prefix`. + for (prefix, owning_def, view) in &ns_views { + let source_name = submodule_source_name(prefix, view.name.deref()); + let row_type = submodule_row_type_name(prefix, view.accessor_name.deref()); + let type_ref = view.product_type_ref; + writeln!(out, "\"{source_name}\": __table({{"); + out.indent(1); + write_table_opts( + owning_def, + out, + type_ref, + &source_name, + iter::empty(), + iter::empty(), + false, + ); + out.dedent(1); + writeln!(out, "}}, {row_type}Row),"); + } out.dedent(1); writeln!(out, "}});"); - table_accessor_aliases.retain(|(deprecated_accessor, _)| !table_accessor_names.contains(deprecated_accessor)); - let has_table_accessor_aliases = !table_accessor_aliases.is_empty(); - writeln!(out); writeln!(out, "/** The schema information for all reducers in this module. This is defined the same way as the reducers would have been defined in the server, except the body of the reducer is omitted in code generation. */"); writeln!(out, "const reducersSchema = __reducers("); @@ -264,6 +373,15 @@ impl Lang for TypeScript { let args_type = reducer_args_type_name(&reducer.accessor_name); writeln!(out, "__reducerSchema(\"{}\", {}),", reducer.name, args_type); } + for (prefix, _, reducer) in &ns_reducers { + if !is_reducer_invokable(reducer) { + continue; + } + // `reducer.name` is already qualified; do not prefix it again. + let wire_name = reducer.name.to_string(); + let args_type = submodule_reducer_args_type_name(prefix, &reducer.accessor_name); + writeln!(out, "__reducerSchema(\"{wire_name}\", {args_type}),"); + } out.dedent(1); writeln!(out, ");"); @@ -282,9 +400,20 @@ impl Lang for TypeScript { procedure.name, ); } + for (prefix, _, procedure) in &ns_procedures { + let wire_name = format!("{}{}", prefix, procedure.name); + let args_type = submodule_procedure_args_type_name(prefix, &procedure.accessor_name); + writeln!( + out, + "__procedureSchema(\"{wire_name}\", {args_type}.params, {args_type}.returnType)," + ); + } out.dedent(1); writeln!(out, ");"); + table_accessor_aliases.retain(|(deprecated_accessor, _)| !table_accessor_names.contains(deprecated_accessor)); + let has_table_accessor_aliases = !table_accessor_aliases.is_empty(); + if has_table_accessor_aliases { writeln!(out); writeln!( @@ -448,36 +577,95 @@ impl Lang for TypeScript { writeln!(out); writeln!(out, "/** The tables available in this remote SpacetimeDB module. Each table reference doubles as a query builder. */"); - if has_table_accessor_aliases { - writeln!( - out, - "const tablesBase: __TablesBase = __makeQueryBuilder(tablesSchema.schemaType);" - ); + if ns_tables.is_empty() && ns_views.is_empty() { + if has_table_accessor_aliases { + writeln!( + out, + "const tablesBase: __TablesBase = __makeQueryBuilder(tablesSchema.schemaType);" + ); + writeln!( + out, + "export const tables: Tables = __withTableAccessorAliases(tablesBase, true) as Tables;" + ); + } else { + writeln!( + out, + "export const tables: __QueryBuilder = __makeQueryBuilder(tablesSchema.schemaType);" + ); + } + } else { + writeln!(out, "const __qb = __makeQueryBuilder(tablesSchema.schemaType);"); + writeln!(out, "export const tables = {{"); + out.indent(1); + // Root tables (use camelCase accessor, matching tablesSchema keys) + for table in iter_tables(module, options.visibility) { + let key = table.accessor_name.deref().to_case(Case::Camel); + writeln!(out, "{key}: __qb.{key},"); + } + // Root views + for view in iter_views(module) { + let key = view.accessor_name.deref().to_case(Case::Camel); + writeln!(out, "{key}: __qb.{key},"); + } + // Build and emit namespace tree + let tree = build_ns_tree(&ns_tables, &ns_views); + emit_ns_tree(out, &tree); + out.dedent(1); + writeln!(out, "}} as const;"); + } + writeln!(out); + writeln!(out, "/** The reducers available in this remote SpacetimeDB module. */"); + if ns_reducers.is_empty() { writeln!( out, - "export const tables: Tables = __withTableAccessorAliases(tablesBase, true) as Tables;" + "export const reducers = __convertToAccessorMap(reducersSchema.reducersType.reducers);" ); } else { writeln!( out, - "export const tables: __QueryBuilder = __makeQueryBuilder(tablesSchema.schemaType);" + "const __reducerAccessors = __convertToAccessorMap(reducersSchema.reducersType.reducers);" ); + writeln!(out, "export const reducers = {{"); + out.indent(1); + for reducer in iter_reducers(module, options.visibility) { + if !is_reducer_invokable(reducer) { + continue; + } + let key = reducer.accessor_name.deref().to_case(Case::Camel); + writeln!(out, "{key}: __reducerAccessors.{key},"); + } + let tree = build_reducer_ns_tree(&ns_reducers); + emit_fn_ns_tree(out, "__reducerAccessors", &tree); + out.dedent(1); + writeln!(out, "}} as const;"); } - writeln!(out); - writeln!(out, "/** The reducers available in this remote SpacetimeDB module. */"); - writeln!( - out, - "export const reducers = __convertToAccessorMap(reducersSchema.reducersType.reducers);" - ); + writeln!(out); writeln!( out, "/** The procedures available in this remote SpacetimeDB module. */" ); - writeln!( - out, - "export const procedures = __convertToAccessorMap(proceduresSchema.procedures);" - ); + if ns_procedures.is_empty() { + writeln!( + out, + "export const procedures = __convertToAccessorMap(proceduresSchema.procedures);" + ); + } else { + writeln!( + out, + "const __procedureAccessors = __convertToAccessorMap(proceduresSchema.procedures);" + ); + writeln!(out, "export const procedures = {{"); + out.indent(1); + for procedure in iter_procedures(module, options.visibility) { + let key = procedure.accessor_name.deref().to_case(Case::Camel); + writeln!(out, "{key}: __procedureAccessors.{key},"); + } + let tree = build_procedure_ns_tree(&ns_procedures); + emit_fn_ns_tree(out, "__procedureAccessors", &tree); + out.dedent(1); + writeln!(out, "}} as const;"); + } // Write type aliases for EventContext, ReducerEventContext, SubscriptionEventContext, ErrorContext writeln!(out); @@ -619,7 +807,20 @@ impl Lang for TypeScript { let procedures_file = generate_procedures_file(module, options); let types_file = generate_types_file(module); - vec![index_file, reducers_file, procedures_file, types_file] + let mut files = vec![index_file, reducers_file, procedures_file, types_file]; + + // Generate types.ts for each submodule namespace so that the + // namespace-scoped reducer/procedure/table files can resolve their + // `import { … } from "./types"` imports. + let mut submodule_namespaces: BTreeMap = BTreeMap::new(); + collect_submodule_namespaces(module, &NamespacePath::root(), &mut submodule_namespaces); + for (prefix, owning_def) in submodule_namespaces.values() { + let ns_path = submodule_ns_path(prefix); + let filename = format!("{ns_path}/types.ts"); + files.push(generate_types_file_with_path(owning_def, filename)); + } + + files } } @@ -694,6 +895,10 @@ fn generate_procedures_file(module: &ModuleDef, options: &CodegenOptions) -> Out } fn generate_types_file(module: &ModuleDef) -> OutputFile { + generate_types_file_with_path(module, "types.ts".to_string()) +} + +fn generate_types_file_with_path(module: &ModuleDef, filename: String) -> OutputFile { let mut output = CodeIndenter::new(String::new(), INDENT); let out = &mut output; @@ -727,11 +932,26 @@ fn generate_types_file(module: &ModuleDef) -> OutputFile { } OutputFile { - filename: "types.ts".to_string(), + filename, code: output.into_inner(), } } +/// Recursively collect all submodule namespaces in depth-first order. +/// Keys are dot-terminated prefix strings (e.g. `"lib."`, `"lib.sublib."`). +/// Values are references to the `ModuleDef` that owns that namespace. +fn collect_submodule_namespaces<'a>( + module: &'a ModuleDef, + prefix: &NamespacePath, + out: &mut BTreeMap, +) { + for (ns, submodule_def) in module.submodules() { + let full_prefix = prefix.child(ns.clone()); + out.insert(full_prefix.to_string(), (full_prefix.clone(), submodule_def)); + collect_submodule_namespaces(submodule_def, &full_prefix, out); + } +} + fn print_index_imports(out: &mut Indenter) { // All library imports are prefixed with `__` to avoid // clashing with the names of user generated types. @@ -859,13 +1079,13 @@ fn write_table_opts<'a>( module: &ModuleDef, out: &mut Indenter, type_ref: AlgebraicTypeRef, - name: &Identifier, + name: &str, indexes: impl Iterator, constraints: impl Iterator, is_event: bool, ) { let product_def = module.typespace_for_generate()[type_ref].as_product().unwrap(); - writeln!(out, "name: '{}',", name.deref()); + writeln!(out, "name: '{}',", name); writeln!(out, "indexes: ["); out.indent(1); for index_def in indexes { @@ -1108,6 +1328,22 @@ fn table_module_name(table_name: &Identifier) -> String { table_name.deref().to_case(Case::Snake) + "_table" } +/// Source name (wire name) for a submodule namespace table/view. +/// +/// This is the *canonical* name, not the accessor name: it is what the host stores and +/// what appears on the wire. E.g. namespace="lib.", name="fruit_basket" → "lib.fruit_basket". +fn submodule_source_name(namespace: &NamespacePath, canonical_name: &str) -> String { + format!("{}{}", namespace, canonical_name) +} + +/// TypeScript import symbol for a submodule namespace table/view row type. +/// Uses `_` separator to avoid colliding with root tables that share the same PascalCase prefix. +/// E.g. namespace="lib.", accessor_name="library_table" → "Lib_LibraryTable" +fn submodule_row_type_name(namespace: &NamespacePath, accessor_name: &str) -> String { + let ns_part = namespace.join_segments("_").to_case(Case::Pascal); + format!("{}_{}", ns_part, accessor_name.to_case(Case::Pascal)) +} + fn reducer_args_type_name(reducer_name: &ReducerName) -> String { reducer_name.deref().to_case(Case::Pascal) + "Reducer" } @@ -1124,6 +1360,156 @@ fn procedure_module_name(procedure_name: &Identifier) -> String { procedure_name.deref().to_case(Case::Snake) + "_procedure" } +/// Converts a dot-terminated namespace like `"lib."` or `"lib.sublib."` to a path like `"lib"` or `"lib/sublib"`. +fn submodule_ns_path(namespace: &NamespacePath) -> String { + namespace.join_segments("/") +} + +/// TypeScript import symbol for a submodule namespace reducer/procedure. +/// Uses `_` separator to avoid colliding with root reducers/procedures sharing the same prefix. +/// E.g. prefix="lib.", accessor_name="library_reducer" → "Lib_LibraryReducer" +fn submodule_fn_type_name(prefix: &NamespacePath, accessor_name: &str) -> String { + let ns_part = prefix.join_segments("_").to_case(Case::Pascal); + format!("{}_{}", ns_part, accessor_name.to_case(Case::Pascal)) +} + +fn submodule_reducer_args_type_name(prefix: &NamespacePath, accessor_name: &ReducerName) -> String { + submodule_fn_type_name(prefix, accessor_name.deref()) + "Reducer" +} + +fn submodule_procedure_args_type_name(prefix: &NamespacePath, accessor_name: &Identifier) -> String { + submodule_fn_type_name(prefix, accessor_name.deref()) + "Procedure" +} + +/// A node in the recursive namespace tree used to emit the nested `tables` export. +struct NsTree { + /// (combined_qb_key, local_ts_key) for table/view entries at this level. + entries: Vec<(String, String)>, + /// Child namespace nodes keyed by namespace segment. + children: BTreeMap, +} + +impl NsTree { + fn new() -> Self { + NsTree { + entries: Vec::new(), + children: BTreeMap::new(), + } + } + + fn insert(&mut self, path_segs: &[&str], combined_qb_key: String, local_ts_key: String) { + if path_segs.is_empty() { + self.entries.push((combined_qb_key, local_ts_key)); + } else { + self.children + .entry(path_segs[0].to_string()) + .or_insert_with(NsTree::new) + .insert(&path_segs[1..], combined_qb_key, local_ts_key); + } + } +} + +/// Build the namespace tree from all submodule tables and views. +fn build_ns_tree<'a>( + ns_tables: &[(NamespacePath, &'a ModuleDef, &'a TableDef)], + ns_views: &[(NamespacePath, &'a ModuleDef, &'a ViewDef)], +) -> BTreeMap { + let mut tree: BTreeMap = BTreeMap::new(); + for (prefix, _, table) in ns_tables { + let source_name = submodule_source_name(prefix, table.name.deref()); + let local = table.accessor_name.deref().to_case(Case::Camel); + let segs: Vec<&str> = prefix.segments().iter().map(|s| &**s).collect(); + if let Some((first, rest)) = segs.split_first() { + tree.entry(first.to_string()) + .or_insert_with(NsTree::new) + .insert(rest, source_name, local); + } + } + for (prefix, _, view) in ns_views { + // Canonical name: must match the tablesSchema key and the DB backing table name. + let source_name = submodule_source_name(prefix, view.name.deref()); + let local = view.accessor_name.deref().to_case(Case::Camel); + let segs: Vec<&str> = prefix.segments().iter().map(|s| &**s).collect(); + if let Some((first, rest)) = segs.split_first() { + tree.entry(first.to_string()) + .or_insert_with(NsTree::new) + .insert(rest, source_name, local); + } + } + tree +} + +/// Recursively emit the namespace tree as nested TypeScript object blocks. +fn emit_ns_tree(out: &mut Indenter, tree: &BTreeMap) { + for (ns, node) in tree { + writeln!(out, "{ns}: {{"); + out.indent(1); + for (qb_key, local_key) in &node.entries { + writeln!(out, "{local_key}: __qb[\"{qb_key}\"],"); + } + emit_ns_tree(out, &node.children); + out.dedent(1); + writeln!(out, "}},"); + } +} + +/// Build namespace tree for submodule reducers (uses `.` path separator). +/// `flat_key` matches SDK's `accessorName = toCamelCase(wireName)`. +/// SDK toCamelCase only splits on `_`/`-`, so `/` is kept verbatim: +/// `"lib.library_reducer"` → `"lib.libraryReducer"`. Bracket notation is required. +fn build_reducer_ns_tree<'a>( + ns_reducers: &[(NamespacePath, &'a ModuleDef, &'a ReducerDef)], +) -> BTreeMap { + let mut tree: BTreeMap = BTreeMap::new(); + for (prefix, _, reducer) in ns_reducers { + if !is_reducer_invokable(reducer) { + continue; + } + let flat_key = format!("{}{}", prefix, reducer.accessor_name.deref().to_case(Case::Camel)); + let local = reducer.accessor_name.deref().to_case(Case::Camel); + let segs: Vec<&str> = prefix.segments().iter().map(|s| &**s).collect(); + if let Some((first, rest)) = segs.split_first() { + tree.entry(first.to_string()) + .or_insert_with(NsTree::new) + .insert(rest, flat_key, local); + } + } + tree +} + +/// Build namespace tree for submodule procedures (uses `.` path separator). +fn build_procedure_ns_tree<'a>( + ns_procedures: &[(NamespacePath, &'a ModuleDef, &'a ProcedureDef)], +) -> BTreeMap { + let mut tree: BTreeMap = BTreeMap::new(); + for (prefix, _, procedure) in ns_procedures { + let flat_key = format!("{}{}", prefix, procedure.accessor_name.deref().to_case(Case::Camel)); + let local = procedure.accessor_name.deref().to_case(Case::Camel); + let segs: Vec<&str> = prefix.segments().iter().map(|s| &**s).collect(); + if let Some((first, rest)) = segs.split_first() { + tree.entry(first.to_string()) + .or_insert_with(NsTree::new) + .insert(rest, flat_key, local); + } + } + tree +} + +/// Emit a namespace tree for reducers/procedures using bracket notation. +/// Flat keys contain `/` (e.g. `"lib/libraryReducer"`) so dot notation is invalid JS. +fn emit_fn_ns_tree(out: &mut Indenter, map_var: &str, tree: &BTreeMap) { + for (ns, node) in tree { + writeln!(out, "{ns}: {{"); + out.indent(1); + for (flat_key, local_key) in &node.entries { + writeln!(out, "{local_key}: {map_var}[\"{flat_key}\"],"); + } + emit_fn_ns_tree(out, map_var, &node.children); + out.dedent(1); + writeln!(out, "}},"); + } +} + pub fn type_name(module: &ModuleDef, ty: &AlgebraicTypeUse) -> String { let mut s = String::new(); write_type(module, &mut s, ty, None, None).unwrap(); diff --git a/crates/codegen/tests/codegen.rs b/crates/codegen/tests/codegen.rs index 0b2c4e483f2..b2a04309478 100644 --- a/crates/codegen/tests/codegen.rs +++ b/crates/codegen/tests/codegen.rs @@ -60,3 +60,29 @@ fn test_typescript_table_handles_are_camel_case() { r#"/** @deprecated Use `loggedOutPlayer` instead. This alias will be removed in the next major version. */"# )); } + +/// A submodule reducer's wire name must be qualified exactly once. +/// +/// `ReducerDef::name` is fully qualified, so any code that also prepends the namespace +/// path would emit `lib.lib.libInsert`. Nothing in the snapshot fixtures mounts a +/// submodule, so this checks the TypeScript output of one that does. +#[test] +fn submodule_reducer_wire_name_is_qualified_once() { + let module = CompiledModule::compile("module-test-ts", CompilationMode::Debug).extract_schema_blocking(); + let code = generate(&module, &TypeScript, &CodegenOptions::default()) + .into_iter() + .map(|f| f.code) + .collect::>() + .join("\n"); + + let reducer_lines: Vec<_> = code.lines().filter(|l| l.contains("__reducerSchema(")).collect(); + assert!( + code.contains(r#"__reducerSchema("lib.lib_insert""#), + "expected a singly-qualified wire name for the submodule reducer; got:\n{}", + reducer_lines.join("\n") + ); + assert!( + !code.contains("lib.lib."), + "namespace was applied twice somewhere in the generated bindings" + ); +} diff --git a/crates/core/src/client/message_handlers_v1.rs b/crates/core/src/client/message_handlers_v1.rs index 60d8eafdb28..32421dd808a 100644 --- a/crates/core/src/client/message_handlers_v1.rs +++ b/crates/core/src/client/message_handlers_v1.rs @@ -49,7 +49,7 @@ pub async fn handle(client: &ClientConnection, message: DataMessage, timer: Inst res.map_err(|e| { ( Some(reducer), - mod_info.module_def.reducer_full(&**reducer).map(|(id, _)| id), + mod_info.module_def.reducer_by_name(reducer).map(|(id, _)| id), e.into(), ) }) diff --git a/crates/core/src/client/messages.rs b/crates/core/src/client/messages.rs index 10183467029..2de3a676bc0 100644 --- a/crates/core/src/client/messages.rs +++ b/crates/core/src/client/messages.rs @@ -17,6 +17,7 @@ use spacetimedb_lib::ser::serde::SerializeWrapper; use spacetimedb_lib::{AlgebraicValue, ConnectionId, TimeDuration, Timestamp}; use spacetimedb_primitives::TableId; use spacetimedb_sats::bsatn; +use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_schema::table_name::TableName; use std::sync::Arc; use std::time::Instant; @@ -608,7 +609,7 @@ impl ToProtocol for SubscriptionMessage { query_id, rows: ws_v1::SubscribeRows { table_id: result.table_id, - table_name: result.table_name.into(), + table_name: RawIdentifier::new(&*result.table_name), table_rows, }, } @@ -621,7 +622,7 @@ impl ToProtocol for SubscriptionMessage { query_id, rows: ws_v1::SubscribeRows { table_id: result.table_id, - table_name: result.table_name.into(), + table_name: RawIdentifier::new(&*result.table_name), table_rows, }, } @@ -639,7 +640,7 @@ impl ToProtocol for SubscriptionMessage { query_id, rows: ws_v1::SubscribeRows { table_id: result.table_id, - table_name: result.table_name.into(), + table_name: RawIdentifier::new(&*result.table_name), table_rows, }, } @@ -652,7 +653,7 @@ impl ToProtocol for SubscriptionMessage { query_id, rows: ws_v1::SubscribeRows { table_id: result.table_id, - table_name: result.table_name.into(), + table_name: RawIdentifier::new(&*result.table_name), table_rows, }, } diff --git a/crates/core/src/host/instance_env.rs b/crates/core/src/host/instance_env.rs index 982ab6dcbff..332aff3bbb0 100644 --- a/crates/core/src/host/instance_env.rs +++ b/crates/core/src/host/instance_env.rs @@ -26,7 +26,7 @@ use spacetimedb_sats::{ buffer::CountWriter, AlgebraicValue, ProductValue, }; -use spacetimedb_schema::identifier::Identifier; +use spacetimedb_schema::identifier::NamespacedIdentifier; use spacetimedb_table::indexes::RowPointer; use spacetimedb_table::table::RowRef; use std::fmt::Display; @@ -48,7 +48,7 @@ pub struct InstanceEnv { /// The type of the last, including current, function to be executed by this environment. pub func_type: FuncCallType, /// The name of the last, including current, function to be executed by this environment. - pub func_name: Option, + pub func_name: Option, /// Are we in an anonymous tx context? in_anon_tx: bool, /// A procedure's last known transaction offset. @@ -246,7 +246,7 @@ impl InstanceEnv { } /// Signal to this `InstanceEnv` that a function call is beginning. - pub fn start_funcall(&mut self, name: Identifier, ts: Timestamp, func_type: FuncCallType) { + pub fn start_funcall(&mut self, name: NamespacedIdentifier, ts: Timestamp, func_type: FuncCallType) { self.start_time = ts; self.start_instant = Instant::now(); self.func_type = func_type; diff --git a/crates/core/src/host/mod.rs b/crates/core/src/host/mod.rs index 68f6035df80..31aa62ec6bc 100644 --- a/crates/core/src/host/mod.rs +++ b/crates/core/src/host/mod.rs @@ -56,7 +56,7 @@ impl FunctionArgs { fn into_tuple(self, seed: ArgsSeed<'_, Def>) -> Result { self._into_tuple(seed).map_err(|err| InvalidFunctionArguments { err, - function_name: seed.name().clone(), + function_name: seed.name().clone().into(), }) } fn _into_tuple(self, seed: ArgsSeed<'_, Def>) -> anyhow::Result { @@ -116,7 +116,7 @@ impl Default for ArgsTuple { // TODO(noa): replace imports from this module with imports straight from primitives. pub use spacetimedb_primitives::ReducerId; -use spacetimedb_schema::identifier::Identifier; +use spacetimedb_schema::identifier::NamespacedIdentifier; /// Inner error type for [`InvalidReducerArguments`] and [`InvalidProcedureArguments`]. #[derive(thiserror::Error, Debug)] @@ -124,7 +124,8 @@ use spacetimedb_schema::identifier::Identifier; pub struct InvalidFunctionArguments { #[source] err: anyhow::Error, - function_name: Identifier, + /// Qualified for a function in a submodule. + function_name: NamespacedIdentifier, } /// Newtype over [`InvalidFunctionArguments`] which renders with the word "reducer". diff --git a/crates/core/src/host/module_host.rs b/crates/core/src/host/module_host.rs index 35f88434330..c307376bdd6 100644 --- a/crates/core/src/host/module_host.rs +++ b/crates/core/src/host/module_host.rs @@ -63,10 +63,10 @@ use spacetimedb_lib::{bsatn, ConnectionId, TimeDuration, Timestamp}; use spacetimedb_primitives::{HttpHandlerId, ProcedureId, TableId, ViewFnPtr, ViewId}; use spacetimedb_query::compile_subscription; use spacetimedb_sats::raw_identifier::RawIdentifier; -use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, ProductValue}; +use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, ProductValue, Typespace}; use spacetimedb_schema::auto_migrate::{AutoMigrateError, MigrationPolicy}; use spacetimedb_schema::def::{ModuleDef, ProcedureDef, ReducerDef, ViewDef}; -use spacetimedb_schema::identifier::Identifier; +use spacetimedb_schema::identifier::{Identifier, NamespacedIdentifier}; use spacetimedb_schema::reducer_name::ReducerName; use spacetimedb_schema::table_name::TableName; use std::collections::VecDeque; @@ -549,18 +549,6 @@ impl GenericModuleInstance for super::v8::JsProcedureInstance { } } -/// Creates the table for `view_def` in `stdb`. -pub fn create_table_from_view_def( - stdb: &RelationalDB, - tx: &mut MutTxId, - module_def: &ModuleDef, - view_def: &ViewDef, -) -> anyhow::Result<()> { - stdb.create_view(tx, module_def, view_def) - .with_context(|| format!("failed to create table for view {}", &view_def.name))?; - Ok(()) -} - /// Moves out the `trapped: bool` from `res`. fn extract_trapped(res: Result<(T, bool), E>) -> (Result, bool) { match res { @@ -595,21 +583,37 @@ fn init_database_inner( let auth_ctx = AuthCtx::for_current(owner_identity); let (tx, ()) = stdb .with_auto_rollback(tx, |tx| { - // Create all in-memory tables defined by the module, - // with IDs ordered lexicographically by the table names. - let mut table_defs: Vec<_> = module_def.tables().collect(); - table_defs.sort_by_key(|x| &x.name); - for def in table_defs { - logger.info(&format!("Creating table `{}`", &def.name)); - spacetimedb_engine::update::create_table_from_def(stdb, tx, module_def, def)?; + // Create all in-memory tables defined by the module (including submodules), + // with IDs ordered lexicographically by their full namespaced names. + let mut table_defs = module_def.all_tables_with_prefix(); + table_defs.sort_by(|(p1, _, d1), (p2, _, d2)| { + let n1 = format!("{}{}", p1, d1.name); + let n2 = format!("{}{}", p2, d2.name); + n1.cmp(&n2) + }); + for (prefix, owning_def, def) in table_defs { + let display_name = format!("{}{}", prefix, def.name); + logger.info(&format!("Creating table `{}`", display_name)); + spacetimedb_engine::update::create_table_from_def_with_prefix(stdb, tx, owning_def, def, &prefix)?; } - // Create all in-memory views defined by the module. - let mut view_defs: Vec<_> = module_def.views().collect(); - view_defs.sort_by_key(|x| &x.name); - for def in view_defs { - logger.info(&format!("Creating table for view `{}`", &def.name)); - create_table_from_view_def(stdb, tx, module_def, def)?; + // Create all in-memory views defined by the module (root + submodule). + let mut view_defs = module_def.all_views_with_prefix(); + view_defs.sort_by(|(p1, _, d1), (p2, _, d2)| { + let n1 = format!("{}{}", p1, d1.name); + let n2 = format!("{}{}", p2, d2.name); + n1.cmp(&n2) + }); + for (prefix, owning_def, def) in view_defs { + let display_name = format!("{}{}", prefix, def.name); + logger.info(&format!("Creating table for view `{}`", display_name)); + if prefix.is_empty() { + spacetimedb_engine::update::create_table_from_view_def(stdb, tx, owning_def, def)?; + } else { + spacetimedb_engine::update::create_table_from_view_def_with_prefix( + stdb, tx, owning_def, def, &prefix, + )?; + } } // Insert the late-bound row-level security expressions. @@ -678,7 +682,7 @@ pub fn call_identity_connected( let reducer_lookup = module.module_def.lifecycle_reducer(Lifecycle::OnConnect); let stdb = module.relational_db(); let workload = Workload::reducer_no_args( - ReducerName::new(Identifier::new_assume_valid("call_identity_connected".into())), + ReducerName::new(Identifier::new_unsafe_assume_valid("call_identity_connected".into())), caller_auth.claims.identity, caller_connection_id, ); @@ -707,7 +711,7 @@ pub fn call_identity_connected( // abort the connection: we can't really recover. let tx = Some(ScopeGuard::into_inner(mut_tx)); let params = ModuleHost::call_reducer_params( - module, + &module.module_def, caller_auth.claims.identity, Some(caller_connection_id), None, @@ -1075,7 +1079,7 @@ impl ProcedureResultTarget { } pub struct CallViewParams { - pub view_name: Identifier, + pub view_name: NamespacedIdentifier, pub view_id: ViewId, pub table_id: TableId, pub fn_ptr: ViewFnPtr, @@ -1088,12 +1092,26 @@ pub struct CallViewParams { pub args: ArgsTuple, pub row_type: AlgebraicTypeRef, pub timestamp: Timestamp, + /// The typespace of the module that owns this view. + /// For root views this equals the top-level typespace; + /// for submodule views this is the submodule's own typespace. + /// + /// Wrapped in an `Arc` so per-instance view calls don't deep-clone the typespace. + pub view_typespace: Arc, } pub(crate) struct ResolvedViewForRefresh<'a> { pub view_id: ViewId, pub table_id: TableId, pub view_def: &'a ViewDef, + /// The full namespaced view name as stored in `st_view` (e.g. `"lib.library_view"`). + pub view_name: NamespacedIdentifier, + /// The globally-offset fn_ptr expected by the guest dispatch layer. + /// For submodule views this differs from `view_def.fn_ptr`, which is local to the owning module. + pub global_fn_ptr: ViewFnPtr, + /// The `ModuleDef` that owns this view. Use this (not the root def) to resolve + /// type-index references in the `ViewDef`. + pub owning_def: &'a ModuleDef, } /// Lookup a module's [`ViewDef`] and check for consistency among @@ -1112,14 +1130,15 @@ pub(crate) fn resolve_view_for_refresh<'a>( .table_id .ok_or_else(|| anyhow::anyhow!("view {:?} does not have a backing table", view_id))?; - let view_name: Identifier = st_view.view_name.into(); - let view_def = module_def.view(&view_name).ok_or_else(|| { - anyhow::anyhow!( - "view `{}` for view id `{}` not found in current module", - view_name, - view_id - ) - })?; + let (global_fn_ptr, view_def, owning_def) = module_def + .view_by_name_with_global_fn_ptr(&st_view.view_name.clone().into()) + .ok_or_else(|| { + anyhow::anyhow!( + "view `{}` for view id `{}` not found in current module", + st_view.view_name, + view_id + ) + })?; let is_anonymous = view_def.is_anonymous; @@ -1128,7 +1147,7 @@ pub(crate) fn resolve_view_for_refresh<'a>( "found is_anonymous={} in st_view, but {} in module when updating view `{}`", st_view.is_anonymous, is_anonymous, - view_name, + st_view.view_name, )); } @@ -1136,6 +1155,9 @@ pub(crate) fn resolve_view_for_refresh<'a>( view_id, table_id, view_def, + view_name: st_view.view_name.into(), + global_fn_ptr, + owning_def, }) } @@ -2073,7 +2095,9 @@ impl ModuleHost { let reducer_name = reducer_lookup .as_ref() .map(|(_, def)| def.name.clone()) - .unwrap_or_else(|| ReducerName::new(Identifier::new_assume_valid("__identity_disconnected__".into()))); + .unwrap_or_else(|| { + ReducerName::new(Identifier::new_unsafe_assume_valid("__identity_disconnected__".into())) + }); let is_client_exist = |mut_tx: &MutTxId| mut_tx.st_client_row(caller_identity, caller_connection_id).is_some(); @@ -2123,7 +2147,7 @@ impl ModuleHost { // that `st_client` is updated appropriately. let tx = Some(mut_tx); let result = Self::call_reducer_params( - info, + &info.module_def, caller_identity, Some(caller_connection_id), None, @@ -2210,7 +2234,7 @@ impl ModuleHost { } fn call_reducer_params( - module: &ModuleInfo, + owning_def: &ModuleDef, caller_identity: Identity, caller_connection_id: Option, client: Option>, @@ -2221,7 +2245,7 @@ impl ModuleHost { args: FunctionArgs, ) -> Result { let args = args - .into_tuple_for_def(&module.module_def, reducer_def) + .into_tuple_for_def(owning_def, reducer_def) .map_err(InvalidReducerArguments)?; let caller_connection_id = caller_connection_id.unwrap_or(ConnectionId::ZERO); Ok(CallReducerParams { @@ -2246,10 +2270,10 @@ impl ModuleHost { reducer_name: &str, args: FunctionArgs, ) -> Result<(&'a ReducerDef, CallReducerParams), ReducerCallError> { - let (reducer_id, reducer_def) = self + let (reducer_id, reducer_def, owning_def) = self .info .module_def - .reducer_full(reducer_name) + .reducer_by_name_with_module(reducer_name) .ok_or(ReducerCallError::NoSuchReducer)?; if let Some(lifecycle) = reducer_def.lifecycle { return Err(ReducerCallError::LifecycleReducer(lifecycle)); @@ -2262,7 +2286,7 @@ impl ModuleHost { Ok(( reducer_def, Self::call_reducer_params( - &self.info, + owning_def, caller_identity, caller_connection_id, client, @@ -2759,10 +2783,10 @@ impl ModuleHost { procedure_name: &str, args: FunctionArgs, ) -> Result<(&'a ProcedureDef, CallProcedureParams), ProcedureCallError> { - let (procedure_id, procedure_def) = self + let (procedure_id, procedure_def, owning_def) = self .info .module_def - .procedure_full(procedure_name) + .procedure_by_name_with_module(procedure_name) .ok_or(ProcedureCallError::NoSuchProcedure)?; if procedure_def.visibility.is_private() && !self.is_database_owner(caller_identity) { @@ -2770,7 +2794,7 @@ impl ModuleHost { } let args = args - .into_tuple_for_def(&self.info.module_def, procedure_def) + .into_tuple_for_def(owning_def, procedure_def) .map_err(InvalidProcedureArguments)?; let caller_connection_id = caller_connection_id.unwrap_or(ConnectionId::ZERO); @@ -2877,7 +2901,7 @@ impl ModuleHost { view_collector.collect_views(&mut view_ids); for view_id in view_ids { let st_view_row = tx.lookup_st_view(view_id)?; - let view_name = st_view_row.view_name.into(); + let view_name: NamespacedIdentifier = st_view_row.view_name.into(); let view_id = st_view_row.view_id; let table_id = st_view_row.table_id.ok_or(ViewCallError::TableDoesNotExist(view_id))?; let is_anonymous = st_view_row.is_anonymous; @@ -2963,9 +2987,11 @@ impl ModuleHost { view_id, table_id, view_def, + view_name, + global_fn_ptr, + owning_def, } = resolved; - let view_name = &view_def.name; - let args = match FunctionArgs::Nullary.into_tuple_for_def(module_def, view_def) { + let args = match FunctionArgs::Nullary.into_tuple_for_def(owning_def, view_def) { Ok(args) => args, Err(err) => { outcome = ViewOutcome::Failed(format!("failed to build view args: {err}")); @@ -2976,15 +3002,16 @@ impl ModuleHost { let (result, trap) = Self::call_view_inner( instance, tx, - view_name, + &view_name, view_id, table_id, - view_def.fn_ptr, + global_fn_ptr, caller, sender, args, view_def.product_type_ref, timestamp, + Arc::new(owning_def.typespace().clone()), ); num_views_evaluated += 1; @@ -3019,7 +3046,7 @@ impl ModuleHost { fn call_view( instance: &mut RefInstance<'_, I>, tx: MutTxId, - view_name: &Identifier, + view_name: &NamespacedIdentifier, view_id: ViewId, table_id: TableId, args: FunctionArgs, @@ -3042,7 +3069,7 @@ impl ModuleHost { fn call_view_at( instance: &mut RefInstance<'_, I>, tx: MutTxId, - view_name: &Identifier, + view_name: &NamespacedIdentifier, view_id: ViewId, table_id: TableId, args: FunctionArgs, @@ -3051,22 +3078,34 @@ impl ModuleHost { timestamp: Timestamp, ) -> Result<(ViewCallResult, bool), ViewCallError> { let module_def = &instance.common.info().module_def; - let view_def = module_def.view(view_name).ok_or(ViewCallError::NoSuchView)?; - let fn_ptr = view_def.fn_ptr; + let (global_fn_ptr, view_def, owning_def) = module_def + .view_by_name_with_global_fn_ptr(view_name) + .ok_or(ViewCallError::NoSuchView)?; let row_type = view_def.product_type_ref; let args = args - .into_tuple_for_def(module_def, view_def) + .into_tuple_for_def(owning_def, view_def) .map_err(InvalidViewArguments)?; Ok(Self::call_view_inner( - instance, tx, view_name, view_id, table_id, fn_ptr, caller, sender, args, row_type, timestamp, + instance, + tx, + view_name, + view_id, + table_id, + global_fn_ptr, + caller, + sender, + args, + row_type, + timestamp, + Arc::new(owning_def.typespace().clone()), )) } fn call_view_inner( instance: &mut RefInstance<'_, I>, tx: MutTxId, - name: &Identifier, + name: &NamespacedIdentifier, view_id: ViewId, table_id: TableId, fn_ptr: ViewFnPtr, @@ -3075,6 +3114,7 @@ impl ModuleHost { args: ArgsTuple, row_type: AlgebraicTypeRef, timestamp: Timestamp, + view_typespace: Arc, ) -> (ViewCallResult, bool) { let view_name = name.clone(); let params = CallViewParams { @@ -3087,6 +3127,7 @@ impl ModuleHost { sender, args, row_type, + view_typespace, }; instance.common.call_view_with_tx(tx, params, instance.instance) @@ -3320,7 +3361,10 @@ impl ModuleHost { .map(PipelinedProject::from) .collect::>(); - let table_name = table_name.into(); + // The v1/v2 wire types carry table names as a plain `RawIdentifier`. A submodule + // table's name is namespaced, but the protocol has always sent it as an opaque + // string, so narrow here rather than changing those message types. + let table_name = RawIdentifier::new(&*table_name); let delta_tx = DeltaTx::from(tx); let params = ExecutionParams::from_auth(auth); let plan_fragments = optimized.iter(); diff --git a/crates/core/src/host/scheduler.rs b/crates/core/src/host/scheduler.rs index c5ec94796d5..86f2bcc39bd 100644 --- a/crates/core/src/host/scheduler.rs +++ b/crates/core/src/host/scheduler.rs @@ -306,7 +306,7 @@ impl ScheduledFunctionParams { } fn kind(&self, module: &ModuleInfo) -> ScheduledFunctionKind { - if module.module_def.procedure_full(self.function_name()).is_some() { + if module.module_def.procedure_by_name(self.function_name()).is_some() { ScheduledFunctionKind::Procedure } else { ScheduledFunctionKind::Reducer @@ -776,11 +776,14 @@ fn function_to_reducer_call_params( ) -> anyhow::Result<(Timestamp, Instant, CallReducerParams)> { let identity = module.database_identity; + // Find the reducer and deserialize the arguments. + // Use the owning module's typespace (not necessarily the root's) so that type-index + // references inside the def are resolved correctly for submodules. let module = &module.module_def; - let Some((id, def)) = module.reducer_full(name) else { + let Some((id, def, owning)) = module.reducer_by_name_with_module(name) else { return Err(anyhow!("Reducer `{name}` not found")); }; - let args = args.into_tuple_for_def(module, def).map_err(InvalidReducerArguments)?; + let args = args.into_tuple_for_def(owning, def).map_err(InvalidReducerArguments)?; let (ts, instant) = scheduled_call_time(at); Ok((ts, instant, CallReducerParams::from_system(ts, identity, id, args))) @@ -795,11 +798,11 @@ fn function_to_procedure_call_params( let identity = module.database_identity; let module = &module.module_def; - let Some((id, def)) = module.procedure_full(name) else { + let Some((id, def, owning)) = module.procedure_by_name_with_module(name) else { return Err(anyhow!("Procedure `{name}` not found")); }; let args = args - .into_tuple_for_def(module, def) + .into_tuple_for_def(owning, def) .map_err(InvalidProcedureArguments)?; let (ts, instant) = scheduled_call_time(at); diff --git a/crates/core/src/host/v8/mod.rs b/crates/core/src/host/v8/mod.rs index 52abf373e48..b27a24fce3a 100644 --- a/crates/core/src/host/v8/mod.rs +++ b/crates/core/src/host/v8/mod.rs @@ -104,7 +104,7 @@ use spacetimedb_datastore::traits::Program; use spacetimedb_lib::{ConnectionId, Identity, RawModuleDef, Timestamp}; use spacetimedb_schema::auto_migrate::MigrationPolicy; use spacetimedb_schema::def::ModuleDef; -use spacetimedb_schema::identifier::Identifier; +use spacetimedb_schema::identifier::NamespacedIdentifier; use spacetimedb_table::static_assert_size; use std::cell::Cell; use std::num::NonZeroUsize; @@ -381,7 +381,7 @@ impl JsInstanceEnv { /// /// Returns the handle used by reducers to read from `args` /// as well as the handle used to write the error message, if any. - fn start_funcall(&mut self, name: Identifier, ts: Timestamp, func_type: FuncCallType) { + fn start_funcall(&mut self, name: NamespacedIdentifier, ts: Timestamp, func_type: FuncCallType) { self.instance_env.start_funcall(name, ts, func_type); } @@ -1406,7 +1406,7 @@ fn handle_main_worker_request( handle_worker_request("call_reducer", reply_tx, || { let mut call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); let (res, trapped) = call_reducer(None, params); - (res, trapped) + (res.result, trapped) }) } JsMainWorkerRequest::CallReducerDetached { params, on_panic } => { @@ -1458,7 +1458,10 @@ fn handle_main_worker_request( caller_auth, caller_connection_id, } => handle_worker_request("call_identity_connected", reply_tx, || { - let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); + let call_reducer = |tx, params| { + let (res, trapped) = instance_common.call_reducer_with_tx(tx, params, inst); + (res.result, trapped) + }; let mut trapped = false; let res = call_identity_connected(caller_auth, caller_connection_id, &info, call_reducer, &mut trapped); (res, trapped) @@ -1468,7 +1471,10 @@ fn handle_main_worker_request( caller_identity, caller_connection_id, } => handle_worker_request("call_identity_disconnected", reply_tx, || { - let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); + let call_reducer = |tx, params| { + let (res, trapped) = instance_common.call_reducer_with_tx(tx, params, inst); + (res.result, trapped) + }; let mut trapped = false; let res = ModuleHost::call_identity_disconnected_inner( caller_identity, @@ -1481,7 +1487,10 @@ fn handle_main_worker_request( }), JsMainWorkerRequest::DisconnectClient { reply_tx, client_id } => { handle_worker_request("disconnect_client", reply_tx, || { - let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); + let call_reducer = |tx, params| { + let (res, trapped) = instance_common.call_reducer_with_tx(tx, params, inst); + (res.result, trapped) + }; let mut trapped = false; let res = ModuleHost::disconnect_client_inner(client_id, &info, call_reducer, &mut trapped); (res, trapped) @@ -1489,7 +1498,7 @@ fn handle_main_worker_request( } JsMainWorkerRequest::InitDatabase { reply_tx, program } => { handle_worker_request("init_database", reply_tx, || { - let call_reducer = |tx, params| instance_common.call_reducer_with_tx_offset(tx, params, inst); + let call_reducer = |tx, params| instance_common.call_reducer_with_tx(tx, params, inst); let (res, trapped): (Result, bool) = init_database(replica_ctx, &info.module_def, program, call_reducer); (res, trapped) diff --git a/crates/core/src/host/v8/syscall/common.rs b/crates/core/src/host/v8/syscall/common.rs index e6e287c545b..fac26a6072f 100644 --- a/crates/core/src/host/v8/syscall/common.rs +++ b/crates/core/src/host/v8/syscall/common.rs @@ -28,7 +28,7 @@ use spacetimedb_lib::{ConnectionId, Identity, RawModuleDef, Timestamp}; use spacetimedb_primitives::{ColId, IndexId, ProcedureId, TableId, ViewFnPtr}; use spacetimedb_sats::bsatn; use spacetimedb_schema::def::ModuleDef; -use spacetimedb_schema::identifier::Identifier; +use spacetimedb_schema::identifier::NamespacedIdentifier; use v8::{FunctionCallbackArguments, Isolate, Local, PinScope, Value}; /// Calls the `__call_procedure__` function `fun`. @@ -761,8 +761,8 @@ fn refresh_views( let table_id = resolved.table_id; let view_def = resolved.view_def; - let view_name = &view_def.name; - let fn_ptr = view_def.fn_ptr; + let view_name = &resolved.view_name; + let fn_ptr = resolved.global_fn_ptr; let sender = tx .as_ref() .expect("procedure tx missing while looking up refreshed view args") @@ -783,7 +783,7 @@ fn refresh_views( tx = Some(next_tx); let return_data = call_result?; - let typespace = module_def.typespace(); + let typespace = resolved.owning_def.typespace(); let row_product_type = typespace .resolve(view_def.product_type_ref) .resolve_refs() @@ -860,7 +860,7 @@ fn call_view( scope: &mut PinScope<'_, '_>, hooks: &HookFunctions<'_>, view_call: &ViewCallInfo, - view_name: &Identifier, + view_name: &NamespacedIdentifier, table_id: TableId, fn_ptr: ViewFnPtr, sender: Option, diff --git a/crates/core/src/host/wasm_common/module_host_actor.rs b/crates/core/src/host/wasm_common/module_host_actor.rs index 18fb6bd7e99..ba64274867e 100644 --- a/crates/core/src/host/wasm_common/module_host_actor.rs +++ b/crates/core/src/host/wasm_common/module_host_actor.rs @@ -6,18 +6,19 @@ use crate::db::sql::ast::SchemaViewer; use crate::energy::{EnergyMonitor, FunctionBudget, FunctionFingerprint}; use crate::error::DBError; use crate::host::host_controller::CallProcedureReturn; +use crate::host::host_controller::ReducerCallResultWithTxOffset; use crate::host::instance_env::{InstanceEnv, TxSlot}; use crate::host::module_common::{build_common_module_from_raw, ModuleCommon}; use crate::host::module_host::{ call_identity_connected, init_database, CallHttpHandlerParams, CallProcedureParams, CallReducerParams, - CallViewParams, ClientConnectedError, DatabaseUpdate, EventStatus, HttpHandlerCallError, ModuleEvent, - ModuleFunctionCall, ModuleInfo, RefInstance, SqlCommand, SqlCommandResult, ViewCallResult, ViewCommand, - ViewCommandResult, ViewOutcome, + CallViewParams, ClientConnectedError, DatabaseUpdate, EventStatus, HttpHandlerCallError, InitDatabaseResult, + ModuleEvent, ModuleFunctionCall, ModuleInfo, RefInstance, SqlCommand, SqlCommandResult, ViewCallResult, + ViewCommand, ViewCommandResult, ViewOutcome, }; use crate::host::scheduler::{CallScheduledFunctionResult, ScheduledFunctionParams}; use crate::host::{ - ArgsTuple, InitDatabaseResult, ModuleHost, ProcedureCallError, ProcedureCallResult, ReducerCallError, - ReducerCallResult, ReducerCallResultWithTxOffset, ReducerId, ReducerOutcome, Scheduler, UpdateDatabaseResult, + ArgsTuple, ModuleHost, ProcedureCallError, ProcedureCallResult, ReducerCallError, ReducerCallResult, ReducerId, + ReducerOutcome, Scheduler, UpdateDatabaseResult, }; use crate::identity::Identity; use crate::messages::control_db::HostType; @@ -37,7 +38,7 @@ use spacetimedb_auth::identity::ConnectionAuthCtx; use spacetimedb_datastore::db_metrics::DB_METRICS; use spacetimedb_datastore::error::{DatastoreError, ViewError}; use spacetimedb_datastore::execution_context::{self, ReducerContext, Workload}; -use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId, ViewCallInfo}; +use spacetimedb_datastore::locking_tx_datastore::{FuncCallType, MutTxId, ViewCallInfo, ViewInstanceArgs}; use spacetimedb_datastore::traits::{IsolationLevel, Program}; use spacetimedb_execution::ExecutionParams; use spacetimedb_lib::buffer::DecodeError; @@ -52,7 +53,7 @@ use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, Deserialize, ProductValu use spacetimedb_schema::auto_migrate::{MigratePlan, MigrationPolicy, MigrationPolicyError}; use spacetimedb_schema::def::deserialize::FunctionDef; use spacetimedb_schema::def::{ModuleDef, ViewDef}; -use spacetimedb_schema::identifier::Identifier; +use spacetimedb_schema::identifier::NamespacedIdentifier; use spacetimedb_schema::reducer_name::ReducerName; use spacetimedb_subscription::SubscriptionPlan; use std::collections::HashMap; @@ -590,7 +591,9 @@ impl WasmModuleInstance { impl WasmModuleInstance { #[tracing::instrument(level = "trace", skip_all)] fn call_reducer_with_tx(&mut self, tx: Option, params: CallReducerParams) -> (ReducerCallResult, bool) { - let (res, trapped) = self.call_reducer_with_tx_offset(tx, params); + let (res, trapped) = crate::callgrind_flag::invoke_allowing_callgrind(|| { + self.common.call_reducer_with_tx(tx, params, &mut self.instance) + }); (res.result, trapped) } @@ -601,7 +604,7 @@ impl WasmModuleInstance { params: CallReducerParams, ) -> (ReducerCallResultWithTxOffset, bool) { crate::callgrind_flag::invoke_allowing_callgrind(|| { - self.common.call_reducer_with_tx_offset(tx, params, &mut self.instance) + self.common.call_reducer_with_tx(tx, params, &mut self.instance) }) } @@ -794,7 +797,7 @@ impl InstanceCommon { let op = ProcedureOp { id: procedure_id, - name: procedure_name.clone(), + name: procedure_name.clone().into(), caller_identity, caller_connection_id, timestamp, @@ -887,7 +890,7 @@ impl InstanceCommon { let op = HttpHandlerOp { id: handler_id, - name: handler_name.clone(), + name: handler_name.clone().into(), timestamp, request_bytes, request_body_bytes: request_body, @@ -956,16 +959,6 @@ impl InstanceCommon { tx: Option, params: CallReducerParams, inst: &mut I, - ) -> (ReducerCallResult, bool) { - let (res, trapped) = self.call_reducer_with_tx_offset(tx, params, inst); - (res.result, trapped) - } - - pub(crate) fn call_reducer_with_tx_offset( - &mut self, - tx: Option, - params: CallReducerParams, - inst: &mut I, ) -> (ReducerCallResultWithTxOffset, bool) { let CallReducerParams { timestamp, @@ -1316,6 +1309,7 @@ impl InstanceCommon { args, row_type, timestamp, + view_typespace, } = params; let _outer_span = start_call_function_span(&view_name, &caller, None); @@ -1374,15 +1368,14 @@ impl InstanceCommon { None => ViewCallInfo::anonymous(view_id), }; let result = ViewResult::from_return_data(raw).context("Error parsing view result")?; - let typespace = self.info.module_def.typespace(); - let row_product_type = typespace + let row_product_type = view_typespace .resolve(row_type) .resolve_refs()? .into_product() .map_err(|_| anyhow!("Error resolving row type for view"))?; let rows = match result { - ViewResult::Rows(bytes) => deserialize_view_rows(row_type, bytes, typespace) + ViewResult::Rows(bytes) => deserialize_view_rows(row_type, bytes, &view_typespace) .context("Error deserializing rows returned by view".to_string())?, ViewResult::RawSql(query) => self .run_query_for_view(&mut tx, &query, &row_product_type, &view_call) @@ -1520,57 +1513,67 @@ fn collect_subscribed_view_calls( ) -> Result, anyhow::Error> { let mut view_calls = Vec::new(); - for view in module_def.views() { + for (prefix, owning_def, view) in module_def.all_views_with_prefix() { let ViewDef { - name: view_name, + name: local_name, is_anonymous, - fn_ptr, product_type_ref, .. } = view; + // Full namespaced canonical name: matches both the st_view registration + // (create_view / create_view_with_prefix) and `view_by_name_with_global_fn_ptr`. + let view_name = prefix.join(local_name.clone()); + + let (global_fn_ptr, _, _) = module_def + .view_by_name_with_global_fn_ptr(&view_name) + .ok_or_else(|| anyhow::anyhow!("view {} not found in module_def", view_name))?; + let st_view = tx - .view_from_name(view_name)? - .ok_or_else(|| anyhow::anyhow!("view {} not found in database", &view_name))?; + .view_from_name(&view_name)? + .ok_or_else(|| anyhow::anyhow!("view {} not found in database", view_name))?; let view_id = st_view.view_id; let table_id = st_view .table_id - .ok_or_else(|| anyhow::anyhow!("view {} does not have a backing table in database", &view_name))?; - let view_instances = tx.materialized_view_instances_for_view(view_id); + .ok_or_else(|| anyhow::anyhow!("view {} does not have a backing table in database", view_name))?; + let subs = tx.materialized_view_instances_for_view(view_id); + let view_typespace = Arc::new(owning_def.typespace().clone()); if *is_anonymous { - if view_instances.is_empty() { + if subs.is_empty() { continue; } view_calls.push(CallViewParams { view_name: view_name.clone(), view_id, table_id, - fn_ptr: *fn_ptr, + fn_ptr: global_fn_ptr, caller: owner_identity, sender: None, args: ArgsTuple::nullary(), row_type: *product_type_ref, timestamp: Timestamp::now(), + view_typespace: view_typespace.clone(), }); continue; } - for args in view_instances { - let Some(sender) = args.sender() else { + for sub in subs { + let ViewInstanceArgs::Sender(identity) = sub else { continue; }; view_calls.push(CallViewParams { view_name: view_name.clone(), view_id, table_id, - fn_ptr: *fn_ptr, + fn_ptr: global_fn_ptr, caller: owner_identity, - sender: Some(sender), + sender: Some(identity), args: ArgsTuple::nullary(), row_type: *product_type_ref, timestamp: Timestamp::now(), + view_typespace: view_typespace.clone(), }); } } @@ -1599,7 +1602,7 @@ impl AllVmMetrics { let def = &info.module_def; let reducers = def.reducer_ids_and_defs(); let num_reducers = reducers.len() as u32; - let reducers = reducers.map(|(_, def)| def.name()); + let reducers = reducers.into_iter().map(|(_, def)| def.name()); // Pre-fetch the metrics for both: let reducer_counters = reducers @@ -1846,7 +1849,8 @@ fn lifecyle_modifications_to_tx( */ pub trait InstanceOp { - fn name(&self) -> &Identifier; + /// The full namespaced name of the function being called. + fn name(&self) -> &NamespacedIdentifier; fn timestamp(&self) -> Timestamp; fn call_type(&self) -> FuncCallType; } @@ -1854,7 +1858,7 @@ pub trait InstanceOp { /// Describes a view call in a cheaply shareable way. #[derive(Clone, Debug)] pub struct ViewOp<'a> { - pub name: &'a Identifier, + pub name: &'a NamespacedIdentifier, pub view_id: ViewId, pub table_id: TableId, pub fn_ptr: ViewFnPtr, @@ -1864,7 +1868,7 @@ pub struct ViewOp<'a> { } impl InstanceOp for ViewOp<'_> { - fn name(&self) -> &Identifier { + fn name(&self) -> &NamespacedIdentifier { self.name } @@ -1880,7 +1884,7 @@ impl InstanceOp for ViewOp<'_> { /// Describes an anonymous view call in a cheaply shareable way. #[derive(Clone, Debug)] pub struct AnonymousViewOp<'a> { - pub name: &'a Identifier, + pub name: &'a NamespacedIdentifier, pub view_id: ViewId, pub table_id: TableId, pub fn_ptr: ViewFnPtr, @@ -1889,7 +1893,7 @@ pub struct AnonymousViewOp<'a> { } impl InstanceOp for AnonymousViewOp<'_> { - fn name(&self) -> &Identifier { + fn name(&self) -> &NamespacedIdentifier { self.name } @@ -1915,8 +1919,8 @@ pub struct ReducerOp<'a> { } impl InstanceOp for ReducerOp<'_> { - fn name(&self) -> &Identifier { - self.name.as_identifier() + fn name(&self) -> &NamespacedIdentifier { + self.name.as_namespaced() } fn timestamp(&self) -> Timestamp { self.timestamp @@ -1951,7 +1955,7 @@ impl From> for execution_context::ReducerContext { #[derive(Clone, Debug)] pub struct ProcedureOp { pub id: ProcedureId, - pub name: Identifier, + pub name: NamespacedIdentifier, pub caller_identity: Identity, pub caller_connection_id: ConnectionId, pub timestamp: Timestamp, @@ -1959,7 +1963,7 @@ pub struct ProcedureOp { } impl InstanceOp for ProcedureOp { - fn name(&self) -> &Identifier { + fn name(&self) -> &NamespacedIdentifier { &self.name } fn timestamp(&self) -> Timestamp { @@ -1974,14 +1978,14 @@ impl InstanceOp for ProcedureOp { #[derive(Clone, Debug)] pub struct HttpHandlerOp { pub id: HttpHandlerId, - pub name: Identifier, + pub name: NamespacedIdentifier, pub timestamp: Timestamp, pub request_bytes: Bytes, pub request_body_bytes: Bytes, } impl InstanceOp for HttpHandlerOp { - fn name(&self) -> &Identifier { + fn name(&self) -> &NamespacedIdentifier { &self.name } fn timestamp(&self) -> Timestamp { @@ -2036,9 +2040,16 @@ mod tests { let mut tx = begin_mut_tx(&stdb); let (view_id, _table_id) = stdb.create_view(&mut tx, &module_def, view_def)?; - let view_call = ViewCallInfo::anonymous(view_id); - tx.subscribe_view(view_call.clone(), ViewInstanceArgs::Anonymous, Identity::ZERO)?; - tx.subscribe_view(view_call, ViewInstanceArgs::Anonymous, Identity::ONE)?; + tx.subscribe_view( + ViewCallInfo::anonymous(view_id), + ViewInstanceArgs::Anonymous, + Identity::ZERO, + )?; + tx.subscribe_view( + ViewCallInfo::anonymous(view_id), + ViewInstanceArgs::Anonymous, + Identity::ONE, + )?; // Two subscriber rows exist, but anonymous views should still be reevaluated once // because they share a single materialization. @@ -2066,10 +2077,16 @@ mod tests { let mut tx = begin_mut_tx(&stdb); let (view_id, _table_id) = stdb.create_view(&mut tx, &module_def, view_def)?; - let zero_args = ViewInstanceArgs::Sender(Identity::ZERO); - let one_args = ViewInstanceArgs::Sender(Identity::ONE); - tx.subscribe_view(ViewCallInfo::from_args(view_id, zero_args), zero_args, Identity::ZERO)?; - tx.subscribe_view(ViewCallInfo::from_args(view_id, one_args), one_args, Identity::ONE)?; + tx.subscribe_view( + ViewCallInfo::sender(view_id, Identity::ZERO), + ViewInstanceArgs::Sender(Identity::ZERO), + Identity::ZERO, + )?; + tx.subscribe_view( + ViewCallInfo::sender(view_id, Identity::ONE), + ViewInstanceArgs::Sender(Identity::ONE), + Identity::ONE, + )?; // Sender-backed views keep one materialization per sender, so reevaluation must // preserve both callers. diff --git a/crates/core/src/host/wasmtime/wasm_instance_env.rs b/crates/core/src/host/wasmtime/wasm_instance_env.rs index e429be2056e..da4a2f3987f 100644 --- a/crates/core/src/host/wasmtime/wasm_instance_env.rs +++ b/crates/core/src/host/wasmtime/wasm_instance_env.rs @@ -24,7 +24,7 @@ use spacetimedb_lib::{bsatn, ConnectionId, Identity, Timestamp}; use spacetimedb_primitives::errno::HOST_CALL_FAILURE; use spacetimedb_primitives::{errno, ColId, ViewFnPtr}; use spacetimedb_schema::def::ModuleDef; -use spacetimedb_schema::identifier::Identifier; +use spacetimedb_schema::identifier::NamespacedIdentifier; use std::future::Future; use std::num::NonZeroU32; use std::sync::Arc; @@ -333,7 +333,7 @@ impl WasmInstanceEnv { /// as well as the handle used to write the reducer error message or procedure return value. pub fn start_funcall( &mut self, - name: Identifier, + name: NamespacedIdentifier, args: bytes::Bytes, ts: Timestamp, func_type: FuncCallType, @@ -1775,8 +1775,8 @@ impl WasmInstanceEnv { let table_id = resolved.table_id; let view_def = resolved.view_def; - let view_name = &view_def.name; - let fn_ptr = view_def.fn_ptr; + let view_name = &resolved.view_name; + let fn_ptr = resolved.global_fn_ptr; let sender = tx .as_ref() .expect("procedure tx missing while looking up refreshed view args") @@ -1796,7 +1796,7 @@ impl WasmInstanceEnv { tx = Some(next_tx); let return_data = call_result?; - let typespace = module_def.typespace(); + let typespace = resolved.owning_def.typespace(); let row_product_type = typespace .resolve(view_def.product_type_ref) .resolve_refs()? @@ -1846,7 +1846,7 @@ impl WasmInstanceEnv { fn call_view<'a>( caller: &mut Caller<'a, Self>, view_call: &ViewCallInfo, - view_name: &Identifier, + view_name: &NamespacedIdentifier, fn_ptr: ViewFnPtr, sender: Option, ) -> anyhow::Result { diff --git a/crates/core/src/host/wasmtime/wasmtime_module.rs b/crates/core/src/host/wasmtime/wasmtime_module.rs index 25357dc876f..e34367ee0de 100644 --- a/crates/core/src/host/wasmtime/wasmtime_module.rs +++ b/crates/core/src/host/wasmtime/wasmtime_module.rs @@ -20,7 +20,7 @@ use spacetimedb_datastore::locking_tx_datastore::FuncCallType; use spacetimedb_lib::{bsatn, ConnectionId, Identity, RawModuleDef}; use spacetimedb_primitives::errno::HOST_CALL_FAILURE; use spacetimedb_schema::def::ModuleDef; -use spacetimedb_schema::identifier::Identifier; +use spacetimedb_schema::identifier::NamespacedIdentifier; use wasmtime::{ AsContext, AsContextMut, Caller, ExternType, Instance, InstancePre, Linker, Store, TypedFunc, WasmBacktrace, WasmParams, WasmResults, @@ -244,7 +244,7 @@ pub(super) fn call_view_export( mut ctx: impl AsContextMut, call_view: Option, call_view_anon: Option, - view_name: &Identifier, + view_name: &NamespacedIdentifier, fn_ptr: u32, sender: Option, args_source: u32, @@ -642,7 +642,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { // Prepare arguments to the reducer + the error sink & start timings. let args_bytes = op.args.get_bsatn().clone(); - let reducer_name = op.name.clone().into(); + let reducer_name: NamespacedIdentifier = op.name().clone(); let (args_source, errors_sink) = store .data_mut() @@ -686,7 +686,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { let (args_source, errors_sink) = store .data_mut() - .start_funcall(op.name.clone(), args_bytes, op.timestamp, op.call_type()); + .start_funcall(op.name().clone(), args_bytes, op.timestamp, op.call_type()); let call_result = call_view_export( &mut *store, @@ -723,7 +723,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { let (args_source, errors_sink) = store .data_mut() - .start_funcall(op.name.clone(), args_bytes, op.timestamp, op.call_type()); + .start_funcall(op.name().clone(), args_bytes, op.timestamp, op.call_type()); let call_result = call_view_export( &mut *store, @@ -769,7 +769,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { let (args_source, result_sink) = store .data_mut() - .start_funcall(op.name.clone(), op.arg_bytes, op.timestamp, FuncCallType::Procedure); + .start_funcall(op.name().clone(), op.arg_bytes, op.timestamp, FuncCallType::Procedure); let Some(call_procedure) = self.call_procedure.as_ref() else { let res = module_host_actor::ProcedureExecuteResult { @@ -837,7 +837,7 @@ impl module_host_actor::WasmInstance for WasmtimeInstance { let (request_source, response_sink) = store .data_mut() - .start_funcall(op.name.clone(), op.request_bytes, op.timestamp, call_type); + .start_funcall(op.name().clone(), op.request_bytes, op.timestamp, call_type); let request_body_source = store .data_mut() .create_extra_bytes_source(op.request_body_bytes) diff --git a/crates/core/src/subscription/mod.rs b/crates/core/src/subscription/mod.rs index 67881a456a5..92f76ab320f 100644 --- a/crates/core/src/subscription/mod.rs +++ b/crates/core/src/subscription/mod.rs @@ -12,6 +12,7 @@ use spacetimedb_lib::metrics::ExecutionMetrics; use spacetimedb_physical_plan::plan::ParamResolver; use spacetimedb_primitives::{ColList, TableId}; use spacetimedb_sats::bsatn::ToBsatn; +use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_sats::Serialize; use spacetimedb_schema::table_name::TableName; use std::sync::Arc; @@ -126,7 +127,8 @@ fn table_update_from_rows( ( ws_v1::TableUpdate::new( table_id, - table_name.into(), + // The v1 wire type takes a plain `RawIdentifier`; see `execute_one_off_query`. + RawIdentifier::new(&*table_name), ws_v1::SingleQueryUpdate { update, num_rows }, ), metrics, diff --git a/crates/core/src/subscription/module_subscription_manager.rs b/crates/core/src/subscription/module_subscription_manager.rs index 0fe707a5582..c82b5c311e9 100644 --- a/crates/core/src/subscription/module_subscription_manager.rs +++ b/crates/core/src/subscription/module_subscription_manager.rs @@ -31,6 +31,7 @@ use spacetimedb_expr::expr::CollectViews; use spacetimedb_lib::metrics::ExecutionMetrics; use spacetimedb_lib::{AlgebraicValue, ConnectionId, Identity, ProductValue}; use spacetimedb_primitives::{ColId, IndexId, TableId, ViewId}; +use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_schema::def::RawModuleDefVersion; use spacetimedb_schema::table_name::TableName; use spacetimedb_subscription::{JoinEdge, SubscriptionPlan}; @@ -1916,7 +1917,8 @@ impl SendWorker { .filter(|upd| !clients_with_errors.contains(&upd.id)) // Do the aggregation. .fold(client_table_id_updates, |mut tables, upd| { - let table_name = upd.table_name.into(); + // The v1 wire type takes a plain `RawIdentifier`; see `execute_one_off_query`. + let table_name = RawIdentifier::new(&*upd.table_name); match tables.entry((upd.id, upd.table_id)) { Entry::Occupied(mut entry) => match entry.get_mut().zip_mut(upd.update) { Bsatn((tbl_upd, update)) => tbl_upd.push(update), @@ -2084,7 +2086,7 @@ impl SendWorker { let table_updates: Vec = qs_updates .into_iter() .map(|((_, _, table_name), rows)| ws_v2::TableUpdate { - table_name: table_name.into(), + table_name: RawIdentifier::new(&*table_name), rows: rows.into_boxed_slice(), }) .collect(); diff --git a/crates/datastore/src/error.rs b/crates/datastore/src/error.rs index 596c0ca26de..912abca4fce 100644 --- a/crates/datastore/src/error.rs +++ b/crates/datastore/src/error.rs @@ -3,7 +3,7 @@ use spacetimedb_lib::db::raw_def::{v9::RawSql, RawIndexDefV8}; use spacetimedb_primitives::{ColId, ColList, IndexId, SequenceId, TableId, ViewId}; use spacetimedb_sats::buffer::DecodeError; use spacetimedb_sats::product_value::InvalidFieldError; -use spacetimedb_sats::raw_identifier::RawIdentifier; +use spacetimedb_sats::raw_identifier::RawNamespacedIdentifier; use spacetimedb_sats::{AlgebraicType, AlgebraicValue}; use spacetimedb_schema::def::error::LibError; use spacetimedb_snapshot::SnapshotError; @@ -40,7 +40,7 @@ pub enum DatastoreError { #[derive(Error, Debug)] pub enum ViewError { #[error("view '{0}' not found")] - NotFound(RawIdentifier), + NotFound(RawNamespacedIdentifier), #[error("Table backing View '{0}' not found")] TableNotFound(ViewId), #[error("failed to deserialize view arguments from row")] diff --git a/crates/datastore/src/execution_context.rs b/crates/datastore/src/execution_context.rs index f2e24a5876e..94386be3fda 100644 --- a/crates/datastore/src/execution_context.rs +++ b/crates/datastore/src/execution_context.rs @@ -81,7 +81,7 @@ impl TryFrom<&txdata::Inputs> for ReducerContext { let timestamp = bsatn::from_reader(args)?; let name = RawIdentifier::new(&**inputs.reducer_name); - let name = ReducerName::new(Identifier::new_assume_valid(name)); + let name = ReducerName::new(Identifier::new_unsafe_assume_valid(name)); Ok(Self { name, diff --git a/crates/datastore/src/locking_tx_datastore/datastore.rs b/crates/datastore/src/locking_tx_datastore/datastore.rs index ffc662f3ce5..a219c14d548 100644 --- a/crates/datastore/src/locking_tx_datastore/datastore.rs +++ b/crates/datastore/src/locking_tx_datastore/datastore.rs @@ -1051,7 +1051,7 @@ pub(crate) mod tests { use spacetimedb_sats::algebraic_value::ser::value_serialize; use spacetimedb_sats::bsatn::{to_vec, ToBsatn}; use spacetimedb_sats::layout::RowTypeLayout; - use spacetimedb_sats::raw_identifier::RawIdentifier; + use spacetimedb_sats::raw_identifier::RawNamespacedIdentifier; use spacetimedb_sats::{product, AlgebraicType, GroundSpacetimeType, SumTypeVariant, SumValue}; use spacetimedb_schema::def::BTreeAlgorithm; use spacetimedb_schema::identifier::Identifier; @@ -1179,7 +1179,7 @@ pub(crate) mod tests { Self { index_id: value.id.into(), table_id: value.table.into(), - index_name: RawIdentifier::new(value.name), + index_name: RawNamespacedIdentifier::new(value.name), index_algorithm: StIndexAlgorithm::BTree { columns: value.col }, } } @@ -1249,7 +1249,7 @@ pub(crate) mod tests { fn from(value: SequenceRow<'_>) -> Self { Self { sequence_id: value.id.into(), - sequence_name: RawIdentifier::new(value.name), + sequence_name: RawNamespacedIdentifier::new(value.name), table_id: value.table.into(), col_pos: value.col_pos.into(), increment: 1, @@ -1265,7 +1265,7 @@ pub(crate) mod tests { fn from(value: SequenceRow<'_>) -> Self { Self { sequence_id: value.id.into(), - sequence_name: RawIdentifier::new(value.name), + sequence_name: RawNamespacedIdentifier::new(value.name), table_id: value.table.into(), col_pos: value.col_pos.into(), increment: 1, @@ -1286,7 +1286,7 @@ pub(crate) mod tests { fn from(value: ConstraintRow<'_>) -> Self { Self { constraint_id: value.constraint_id.into(), - constraint_name: RawIdentifier::new(value.constraint_name), + constraint_name: RawNamespacedIdentifier::new(value.constraint_name), table_id: value.table_id.into(), constraint_data: StConstraintData::Unique { columns: value.unique_columns.into(), @@ -3205,7 +3205,7 @@ pub(crate) mod tests { table_id: TableId::SENTINEL, schedule_id: ScheduleId::SENTINEL, schedule_name: Identifier::for_test("schedule"), - function_name: Identifier::for_test("reducer"), + function_name: Identifier::for_test("reducer").into(), at_column: 1.into(), }; let sum_ty = AlgebraicType::sum([("foo", AlgebraicType::Bool), ("bar", AlgebraicType::U16)]); diff --git a/crates/datastore/src/locking_tx_datastore/mut_tx.rs b/crates/datastore/src/locking_tx_datastore/mut_tx.rs index d230c40c7fe..3ba90142faf 100644 --- a/crates/datastore/src/locking_tx_datastore/mut_tx.rs +++ b/crates/datastore/src/locking_tx_datastore/mut_tx.rs @@ -51,12 +51,15 @@ use spacetimedb_primitives::{ col_list, ColId, ColList, ColSet, ConstraintId, IndexId, ScheduleId, SequenceId, TableId, ViewId, }; use spacetimedb_sats::{ - bsatn::to_writer, memory_usage::MemoryUsage, raw_identifier::RawIdentifier, ser::Serialize, AlgebraicValue, - ProductType, ProductValue, + bsatn::to_writer, + memory_usage::MemoryUsage, + raw_identifier::{RawIdentifier, RawNamespacedIdentifier}, + ser::Serialize, + AlgebraicValue, ProductType, ProductValue, }; use spacetimedb_schema::{ def::{ModuleDef, ViewColumnDef, ViewDef, ViewParamDef}, - identifier::Identifier, + identifier::{Identifier, NamespacePath, NamespacedIdentifier}, reducer_name::ReducerName, schema::{ ColumnSchema, ConstraintSchema, IndexSchema, RowLevelSecuritySchema, SequenceSchema, TableSchema, @@ -454,7 +457,10 @@ pub struct MutTxId { pub(crate) _not_send: PhantomData>, } -static_assert_size!(MutTxId, 560); +// Grew by one word when `ReducerName` became fully qualified: it now holds a +// `NamespacedIdentifier` (segments + joined rendering) rather than a single `Identifier`. +// One per transaction, not per row. +static_assert_size!(MutTxId, 576); impl MutTxId { /// Record that a view performs a table scan in this transaction's read set @@ -779,7 +785,7 @@ impl MutTxId { .. } = view_def; - let view_name: RawIdentifier = name.clone().into(); + let view_name: RawNamespacedIdentifier = RawIdentifier::from(name.clone()).into(); // `create_table` inserts into `st_view` and updates the table schema. let view_id = self @@ -794,6 +800,52 @@ impl MutTxId { Ok((view_id, table_id)) } + /// Like [`create_view`] but registers the view under `name_prefix + view_def.name` + /// (e.g. `"lib.library_view"`), using `owning_def` for type resolution. + /// + /// The canonical `view_def.name` is used (not `accessor_name`) so that submodule views + /// follow the same convention as root views ([`Self::create_view`]) and match the keys + /// used by `ModuleDef::view_by_name_with_global_fn_ptr` and the auto-migrate plan. + /// + /// Used for submodule views whose canonical names are dot-namespaced. + pub fn create_view_with_prefix( + &mut self, + owning_def: &ModuleDef, + view_def: &ViewDef, + name_prefix: &NamespacePath, + ) -> Result<(ViewId, TableId)> { + let mut table_schema = TableSchema::from_view_def_for_datastore(owning_def, view_def); + let full_view_name = name_prefix.join(view_def.name.clone()); + table_schema.table_name = TableName::from(full_view_name.clone()); + + // Namespace the alias rather than dropping it: a bare accessor name would collide + // when two mounts have views with the same local name, but discarding it loses the + // mapping codegen needs. + table_schema.alias = table_schema.alias.map(|alias| name_prefix.join_namespaced(&alias)); + + // Prefix index and constraint names so they remain globally unique across mounts. + for index in &mut table_schema.indexes { + index.index_name = name_prefix.join_raw(&index.index_name); + } + for constraint in &mut table_schema.constraints { + constraint.constraint_name = name_prefix.join_raw(&constraint.constraint_name); + } + + let table_id = self.create_table(table_schema)?; + + let view_name = RawNamespacedIdentifier::from(full_view_name); + let view_id = self + .view_id_from_name(&view_name)? + .ok_or(ViewError::NotFound(view_name))?; + + self.insert_into_st_view_param(view_id, &view_def.param_columns)?; + self.insert_into_st_view_column(view_id, &view_def.return_columns)?; + + self.committed_state_write_lock.ephemeral_tables.insert(table_id); + + Ok((view_id, table_id)) + } + /// Drop the backing table of a view and update the system tables. pub fn drop_view(&mut self, view_id: ViewId) -> Result<()> { let st_view_row = self.lookup_st_view(view_id)?; @@ -943,7 +995,7 @@ impl MutTxId { } /// Insert a row into `st_table_accessor` for `table_name`, if an alias is present. - fn insert_st_table_accessor(&mut self, table_name: &TableName, alias: Option<&Identifier>) -> Result<()> { + fn insert_st_table_accessor(&mut self, table_name: &TableName, alias: Option<&NamespacedIdentifier>) -> Result<()> { let Some(accessor_name) = alias.cloned() else { return Ok(()); }; @@ -975,7 +1027,11 @@ impl MutTxId { } /// Insert a row into `st_index_accessor` for `index_name`, if an alias is present. - fn insert_st_index_accessor(&mut self, index_name: &RawIdentifier, alias: Option<&RawIdentifier>) -> Result<()> { + fn insert_st_index_accessor( + &mut self, + index_name: &RawNamespacedIdentifier, + alias: Option<&RawNamespacedIdentifier>, + ) -> Result<()> { let Some(accessor_name) = alias.cloned() else { return Ok(()); }; @@ -1136,7 +1192,7 @@ impl MutTxId { } /// Drops rows in `st_index_accessor` for this canonical `index_name`. - fn drop_st_index_accessor(&mut self, index_name: &RawIdentifier) -> Result<()> { + fn drop_st_index_accessor(&mut self, index_name: &RawNamespacedIdentifier) -> Result<()> { let value = index_name.as_ref().into(); self.delete_col_eq(ST_INDEX_ACCESSOR_ID, StIndexAccessorFields::IndexName.col_id(), &value) } @@ -1550,7 +1606,7 @@ impl MutTxId { fn create_table_and_update_seq( &mut self, table_schema: TableSchema, - seq_values: HashMap, + seq_values: HashMap, ) -> Result { let table_id = self.create_table(table_schema)?; let table_schema = self.schema_for_table(table_id)?; diff --git a/crates/datastore/src/system_tables.rs b/crates/datastore/src/system_tables.rs index c1eeb6aa6ea..2635093cc2d 100644 --- a/crates/datastore/src/system_tables.rs +++ b/crates/datastore/src/system_tables.rs @@ -24,12 +24,12 @@ use spacetimedb_sats::algebraic_value::de::ValueDeserializer; use spacetimedb_sats::algebraic_value::ser::value_serialize; use spacetimedb_sats::hash::Hash; use spacetimedb_sats::product_value::InvalidFieldError; -use spacetimedb_sats::raw_identifier::RawIdentifier; +use spacetimedb_sats::raw_identifier::{RawIdentifier, RawNamespacedIdentifier}; use spacetimedb_sats::{impl_deserialize, impl_serialize, impl_st, u256, AlgebraicType, AlgebraicValue, ArrayValue}; use spacetimedb_schema::def::{ BTreeAlgorithm, ConstraintData, DirectAlgorithm, HashAlgorithm, IndexAlgorithm, ModuleDef, UniqueConstraintData, }; -use spacetimedb_schema::identifier::Identifier; +use spacetimedb_schema::identifier::{Identifier, NamespacedIdentifier}; use spacetimedb_schema::schema::{ ColumnSchema, ConstraintSchema, IndexSchema, RowLevelSecuritySchema, ScheduleSchema, Schema, SequenceSchema, TableSchema, @@ -250,7 +250,7 @@ pub trait StFields: Copy + Sized { /// Returns the column name of the system table field as a [`RawIdentifier`]. #[inline] fn col_name(self) -> Identifier { - Identifier::new_assume_valid(self.name().into()) + Identifier::new_unsafe_assume_valid(self.name().into()) } /// Return all fields of this type, in order. @@ -1197,7 +1197,8 @@ pub struct StViewArgRow { pub struct StIndexRow { pub index_id: IndexId, pub table_id: TableId, - pub index_name: RawIdentifier, + /// Namespaced for submodule tables (e.g. `"lib.sessions_id_idx_btree"`). + pub index_name: RawNamespacedIdentifier, pub index_algorithm: StIndexAlgorithm, } @@ -1298,7 +1299,8 @@ impl From for StIndexRow { #[sats(crate = spacetimedb_lib)] pub struct StSequenceRow { pub sequence_id: SequenceId, - pub sequence_name: RawIdentifier, + /// Namespaced for submodule tables (e.g. `"lib.sessions_id_seq"`). + pub sequence_name: RawNamespacedIdentifier, pub table_id: TableId, pub col_pos: ColId, pub increment: i128, @@ -1349,7 +1351,8 @@ impl From for SequenceSchema { #[sats(crate = spacetimedb_lib)] pub struct StConstraintRow { pub(crate) constraint_id: ConstraintId, - pub(crate) constraint_name: RawIdentifier, + /// Namespaced for submodule tables. + pub(crate) constraint_name: RawNamespacedIdentifier, pub table_id: TableId, pub(crate) constraint_data: StConstraintData, } @@ -1746,7 +1749,9 @@ pub struct StScheduledRow { /// Note that, despite the column name, this may refer to either a reducer or a procedure. /// We cannot change the schema of existing system tables, /// so we are unable to rename this column. - pub(crate) reducer_name: Identifier, + /// Namespaced for submodule tables (e.g. `"lib.library_scheduled_procedure"`), + /// since that is how the scheduler resolves it. + pub(crate) reducer_name: NamespacedIdentifier, pub(crate) schedule_name: Identifier, pub(crate) at_column: ColId, } @@ -1808,7 +1813,8 @@ impl From for ProductValue { #[sats(crate = spacetimedb_lib)] pub struct StTableAccessorRow { pub table_name: TableName, - pub accessor_name: Identifier, + /// Namespaced for submodule tables, matching `table_name`. + pub accessor_name: NamespacedIdentifier, } impl TryFrom> for StTableAccessorRow { @@ -1828,8 +1834,9 @@ impl From for ProductValue { #[derive(Debug, Clone, PartialEq, Eq, SpacetimeType)] #[sats(crate = spacetimedb_lib)] pub struct StIndexAccessorRow { - pub index_name: RawIdentifier, - pub accessor_name: RawIdentifier, + /// Namespaced for submodule tables. + pub index_name: RawNamespacedIdentifier, + pub accessor_name: RawNamespacedIdentifier, } impl TryFrom> for StIndexAccessorRow { diff --git a/crates/datastore/src/traits.rs b/crates/datastore/src/traits.rs index 22e12b98271..3296baef545 100644 --- a/crates/datastore/src/traits.rs +++ b/crates/datastore/src/traits.rs @@ -207,7 +207,11 @@ pub struct TxDataTableEntry { pub ephemeral: bool, } -static_assert_size!(TxDataTableEntry, 56); +// Grew from 56 when `TableName` became a `NamespacedIdentifier`: it keeps both the +// validated segments and their dot-joined rendering (2 words each) where it previously +// held a single `RawIdentifier` (1 word). The extra word is per table per transaction, +// which is not a hot allocation; keeping the segments typed is worth it. +static_assert_size!(TxDataTableEntry, 72); impl TxDataTableEntry { /// Create a new, empty `TxDataTableEntry` for `table_name`. diff --git a/crates/engine/src/relational_db.rs b/crates/engine/src/relational_db.rs index 7edb9fe20fc..90c22ea282c 100644 --- a/crates/engine/src/relational_db.rs +++ b/crates/engine/src/relational_db.rs @@ -51,6 +51,7 @@ use spacetimedb_sats::memory_usage::MemoryUsage; use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_sats::{AlgebraicType, AlgebraicValue, ProductType, ProductValue}; use spacetimedb_schema::def::{ModuleDef, TableDef, ViewDef}; +use spacetimedb_schema::identifier::NamespacePath; use spacetimedb_schema::reducer_name::ReducerName; use spacetimedb_schema::schema::{ ColumnSchema, ConstraintSchema, IndexSchema, RowLevelSecuritySchema, Schema, SequenceSchema, TableSchema, @@ -1229,6 +1230,16 @@ impl RelationalDB { Ok(tx.create_view(module_def, view_def)?) } + pub fn create_view_with_prefix( + &self, + tx: &mut MutTx, + owning_def: &ModuleDef, + view_def: &ViewDef, + name_prefix: &NamespacePath, + ) -> Result<(ViewId, TableId), DBError> { + Ok(tx.create_view_with_prefix(owning_def, view_def, name_prefix)?) + } + pub fn drop_view(&self, tx: &mut MutTx, view_id: ViewId) -> Result<(), DBError> { Ok(tx.drop_view(view_id)?) } diff --git a/crates/engine/src/update.rs b/crates/engine/src/update.rs index 66b981f6a2b..e09ae1c076e 100644 --- a/crates/engine/src/update.rs +++ b/crates/engine/src/update.rs @@ -6,14 +6,25 @@ use spacetimedb_datastore::locking_tx_datastore::state_view::StateView; use spacetimedb_datastore::locking_tx_datastore::{MutTxId, TxId}; use spacetimedb_datastore::system_tables::{StViewFields, StViewRow, ST_VIEW_ID}; use spacetimedb_lib::db::auth::StTableType; +use spacetimedb_lib::db::raw_def::v9::{RawRowLevelSecurityDefV9, TableAccess}; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::AlgebraicValue; -use spacetimedb_primitives::{ColSet, TableId}; +use spacetimedb_primitives::{ColSet, ConstraintId, TableId}; use spacetimedb_schema::auto_migrate::{AutoMigratePlan, AutoMigrateStep, ManualMigratePlan, MigratePlan}; -use spacetimedb_schema::def::{ModuleDef, ModuleDefLookup, TableDef, ViewDef}; +use spacetimedb_schema::def::{ConstraintDef, IndexDef, ModuleDef, ModuleDefLookup, SequenceDef, TableDef, ViewDef}; +use spacetimedb_schema::identifier::{Identifier, NamespacePath, NamespacedIdentifier}; use spacetimedb_schema::schema::{ column_schemas_from_defs, ConstraintSchema, IndexSchema, Schema, SequenceSchema, TableSchema, }; +use spacetimedb_schema::table_name::TableName; + +/// The joined, qualified name a `(namespace, name)` key refers to. +/// +/// The database stores names joined, so migration steps build the joined form here rather +/// than every def carrying a pre-joined copy. These are per-step, not per-row. +fn joined(namespace: &NamespacePath, name: &Identifier) -> NamespacedIdentifier { + namespace.join(name.clone()) +} /// The logger used for by [`update_database`] and friends. pub trait UpdateLogger { @@ -55,8 +66,12 @@ fn stale_view_backing_table_recreate_steps<'def>( stdb.with_read_only(Workload::Internal, |tx| -> anyhow::Result<_> { let mut steps = Vec::new(); - for view in module_def.views() { - if view_backing_table_needs_recreate(stdb, tx, module_def, view)? { + // Submodule views are registered in `st_view` under their full namespaced name, + // so they must be walked with their prefix and checked against the typespace of + // the submodule that owns them. + for (_, owning_def, view) in module_def.all_views_with_prefix() { + let full_name = joined(&view.namespace, &view.name); + if view_backing_table_needs_recreate(stdb, tx, owning_def, view, &full_name)? { steps.extend([ AutoMigrateStep::RemoveView(view.key()), AutoMigrateStep::AddView(view.key()), @@ -76,21 +91,22 @@ fn stale_view_backing_table_recreate_steps<'def>( fn view_backing_table_needs_recreate( stdb: &RelationalDB, tx: &mut TxId, - module_def: &ModuleDef, + owning_def: &ModuleDef, view: &ViewDef, + full_name: &NamespacedIdentifier, ) -> anyhow::Result { - let Some(table_id) = view_backing_table_id(tx, view)? else { + let Some(table_id) = view_backing_table_id(tx, full_name)? else { return Ok(false); }; let actual = stdb.schema_for_table(tx, table_id)?; - let expected = TableSchema::from_view_def_for_datastore(module_def, view); + let expected = TableSchema::from_view_def_for_datastore(owning_def, view); Ok(view_backing_row_layout_changed(&actual, &expected)) } -fn view_backing_table_id(tx: &mut TxId, view: &ViewDef) -> anyhow::Result> { - let view_name = AlgebraicValue::from(>::from(&*view.name)); +fn view_backing_table_id(tx: &mut TxId, full_name: &NamespacedIdentifier) -> anyhow::Result> { + let view_name = AlgebraicValue::from(>::from(&**full_name)); let Some(row) = tx .iter_by_col_eq(ST_VIEW_ID, StViewFields::ViewName, &view_name)? .next() @@ -135,15 +151,28 @@ pub fn update_database( // TODO: consider using `ErrorStream` here. let old_module_def = plan.old_def(); + + // Build a map from full-name (namespaced) -> (owning_def, table_def) covering root and all + // submodule tables. Submodule tables are stored in the DB with prefixed names like + // "lib.library_procedure_timer", but `ModuleDef::table()` only has the current level. + // `all_tables_with_prefix()` returns the owning submodule alongside each def, which is also + // needed so that `check_compatible` resolves column type refs against the correct + // (sub)module typespace. + let old_tables_by_name: std::collections::HashMap = old_module_def + .all_tables_with_prefix() + .into_iter() + .map(|(prefix, owning_def, table_def)| (format!("{}{}", prefix, &table_def.name[..]), (owning_def, table_def))) + .collect(); + for table in existing_tables .iter() .filter(|table| table.table_type != StTableType::System && !table.is_view()) { - let old_def = old_module_def - .table(&table.table_name[..]) + let (owning_def, old_def) = old_tables_by_name + .get(table.table_name.as_ref()) .ok_or_else(|| anyhow::anyhow!("table {} not found in old_module_def", table.table_name))?; - table.check_compatible(old_module_def, old_def)?; + table.check_compatible(owning_def, old_def)?; } match plan { @@ -185,10 +214,16 @@ fn auto_migrate_database( for precheck in plan.prechecks { match precheck { - spacetimedb_schema::auto_migrate::AutoMigratePrecheck::CheckAddSequenceRangeValid(sequence_name) => { - let table_def = plan.new.stored_in_table_def(sequence_name).unwrap(); - let sequence_def = &table_def.sequences[sequence_name]; - let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); + spacetimedb_schema::auto_migrate::AutoMigratePrecheck::CheckAddSequenceRangeValid(key) => { + let (namespace, sequence_name) = key; + let (_, table_def) = plan.new.find_storing_table(namespace, sequence_name).ok_or_else(|| { + anyhow::anyhow!("Precheck: sequence `{sequence_name}` not found in new module def") + })?; + let sequence_def: &SequenceDef = plan.new.lookup(key).ok_or_else(|| { + anyhow::anyhow!("Precheck: sequence `{sequence_name}` not found in new module def") + })?; + let table_full_name = joined(namespace, &table_def.name); + let table_id = stdb.table_id_from_name_mut(tx, &table_full_name)?.unwrap(); let ty = table_def .get_column(sequence_def.column) @@ -230,8 +265,10 @@ fn auto_migrate_database( for step in plan.steps { match step { - spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveTable(table_name) => { - let table_id = stdb.table_id_from_name_mut(tx, table_name)?.unwrap(); + spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveTable(table_name_key) => { + let (namespace, local) = table_name_key; + let table_name = joined(namespace, local); + let table_id = stdb.table_id_from_name_mut(tx, &table_name)?.unwrap(); if stdb.table_row_count_mut(tx, table_id).unwrap_or(0) > 0 { anyhow::bail!( @@ -243,23 +280,31 @@ fn auto_migrate_database( log!(logger, "Dropping table `{table_name}`"); stdb.drop_table(tx, table_id)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::AddTable(table_name) => { - let table_def: &TableDef = plan.new.expect_lookup(table_name); - - // Recursively sets IDs to 0. - // They will be initialized by the database when the table is created. - let table_schema = TableSchema::from_module_def(plan.new, table_def, (), TableId::SENTINEL); - + spacetimedb_schema::auto_migrate::AutoMigrateStep::AddTable(table_name_key) => { + let (namespace, local) = table_name_key; + let table_name = joined(namespace, local); + let (owning_def, table_def) = plan + .new + .find_table(table_name_key) + .ok_or_else(|| anyhow::anyhow!("AddTable: table `{table_name}` not found in new module def"))?; log!(logger, "Creating table `{table_name}`"); - - stdb.create_table(tx, table_schema)?; + create_table_from_def_with_prefix(stdb, tx, owning_def, table_def, namespace)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::AddView(view_name) => { - let view_def: &ViewDef = plan.new.expect_lookup(view_name); - stdb.create_view(tx, plan.new, view_def)?; + spacetimedb_schema::auto_migrate::AutoMigrateStep::AddView(view_name_key) => { + let (namespace, local) = view_name_key; + let view_name = joined(namespace, local); + let (owning_def, view_def) = plan + .new + .find_view(view_name_key) + .ok_or_else(|| anyhow::anyhow!("AddView: view `{view_name}` not found in new module def"))?; + create_table_from_view_def_with_prefix(stdb, tx, owning_def, view_def, namespace)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveView(view_name) => { - let view_id = stdb.view_id_from_name_mut(tx, view_name)?.unwrap(); + spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveView(view_name_key) => { + let (namespace, local) = view_name_key; + let view_name = joined(namespace, local); + let view_id = stdb + .view_id_from_name_mut(tx, &view_name)? + .ok_or_else(|| anyhow::anyhow!("RemoveView: view `{view_name}` not found in database"))?; stdb.drop_view(tx, view_id)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::UpdateView(_) => { @@ -269,10 +314,18 @@ fn auto_migrate_database( res = UpdateResult::EvaluateSubscribedViews; } } - spacetimedb_schema::auto_migrate::AutoMigrateStep::AddIndex(index_name) => { - let table_def = plan.new.stored_in_table_def(index_name).unwrap(); - let index_def = table_def.indexes.get(index_name).unwrap(); - let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); + spacetimedb_schema::auto_migrate::AutoMigrateStep::AddIndex(key) => { + let (namespace, index_name) = key; + let (owning_def, table_def) = plan + .new + .find_storing_table(namespace, index_name) + .ok_or_else(|| anyhow::anyhow!("AddIndex: `{index_name}` not found in new module def"))?; + let table_full_name = joined(namespace, &table_def.name); + let index_def: &IndexDef = plan + .new + .lookup(key) + .ok_or_else(|| anyhow::anyhow!("AddIndex: index `{index_name}` not found in new module def"))?; + let table_id = stdb.table_id_from_name_mut(tx, &table_full_name)?.unwrap(); let index_cols = ColSet::from(index_def.algorithm.columns()); @@ -282,131 +335,177 @@ fn auto_migrate_database( .filter_map(|(_, c)| c.data.unique_columns()) .any(|unique_cols| unique_cols == &index_cols); - log!(logger, "Creating index `{}` on table `{}`", index_name, table_def.name); + log!(logger, "Creating index `{index_name}` on table `{table_full_name}`"); - let index_schema = IndexSchema::from_module_def(plan.new, index_def, table_id, 0.into()); + let mut index_schema = IndexSchema::from_module_def(owning_def, index_def, table_id, 0.into()); + + // Apply namespace prefix for submodule indexes + index_schema.index_name = namespace.join_raw(&index_schema.index_name); + index_schema.alias = index_schema.alias.as_ref().map(|alias| namespace.join_raw(alias)); stdb.create_index(tx, index_schema, is_unique)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveIndex(index_name) => { - let table_def = plan.old.stored_in_table_def(index_name).unwrap(); - - let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); + spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveIndex(key) => { + let (namespace, index_name) = key; + let (_owning_def, table_def) = plan + .old + .find_storing_table(namespace, index_name) + .ok_or_else(|| anyhow::anyhow!("RemoveIndex: `{index_name}` not found in old module def"))?; + let table_full_name = joined(namespace, &table_def.name); + let stored_name = namespace.join_raw(&index_name.clone().into()); + let table_id = stdb.table_id_from_name_mut(tx, &table_full_name)?.unwrap(); let table_schema = stdb.schema_for_table_mut(tx, table_id)?; let index_schema = table_schema .indexes .iter() - .find(|index| index.index_name[..] == index_name[..]) + .find(|index| index.index_name == stored_name) .unwrap(); - log!(logger, "Dropping index `{}` on table `{}`", index_name, table_def.name); + log!(logger, "Dropping index `{index_name}` on table `{table_full_name}`"); stdb.drop_index(tx, index_schema.index_id)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveConstraint(constraint_name) => { - let table_def = plan.old.stored_in_table_def(constraint_name).unwrap(); - - let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); + spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveConstraint(key) => { + let (namespace, constraint_name) = key; + let (_owning_def, table_def) = + plan.old.find_storing_table(namespace, constraint_name).ok_or_else(|| { + anyhow::anyhow!("RemoveConstraint: `{constraint_name}` not found in old module def") + })?; + let table_full_name = joined(namespace, &table_def.name); + let stored_name = namespace.join_raw(&constraint_name.clone().into()); + let table_id = stdb.table_id_from_name_mut(tx, &table_full_name)?.unwrap(); let table_schema = stdb.schema_for_table_mut(tx, table_id)?; + let constraint_schema = table_schema .constraints .iter() - .find(|constraint| constraint.constraint_name[..] == constraint_name[..]) + .find(|constraint| constraint.constraint_name == stored_name) .unwrap(); log!( logger, - "Dropping constraint `{}` on table `{}`", - constraint_name, - table_def.name + "Dropping constraint `{constraint_name}` on table `{table_full_name}`" ); stdb.drop_constraint(tx, constraint_schema.constraint_id)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::AddConstraint(constraint_name) => { - let table_def = plan + spacetimedb_schema::auto_migrate::AutoMigrateStep::AddConstraint(key) => { + let (namespace, constraint_name) = key; + let (owning_def, table_def) = plan .new - .stored_in_table_def(constraint_name) - .expect("AddConstraint references a table that should exist in the new module def"); - let constraint_def = &table_def.constraints[constraint_name]; + .find_storing_table(namespace, constraint_name) + .ok_or_else(|| anyhow::anyhow!("AddConstraint: `{constraint_name}` not found in new module def"))?; + let table_full_name = joined(namespace, &table_def.name); + let constraint_def: &ConstraintDef = plan.new.lookup(key).ok_or_else(|| { + anyhow::anyhow!("AddConstraint: constraint `{constraint_name}` not found in new module def") + })?; let table_id = stdb - .table_id_from_name_mut(tx, &table_def.name)? + .table_id_from_name_mut(tx, &table_full_name)? .expect("table should exist in the database for AddConstraint"); - let constraint_schema = ConstraintSchema::from_module_def( - plan.new, - constraint_def, - table_id, - spacetimedb_primitives::ConstraintId::SENTINEL, - ); + let mut constraint_schema = + ConstraintSchema::from_module_def(owning_def, constraint_def, table_id, ConstraintId::SENTINEL); + + // Apply namespace prefix for submodule constraints + constraint_schema.constraint_name = namespace.join_raw(&constraint_schema.constraint_name); log!( logger, - "Adding constraint `{}` on table `{}`", - constraint_name, - table_def.name + "Adding constraint `{constraint_name}` on table `{table_full_name}`" ); stdb.create_constraint(tx, constraint_schema)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::AddSequence(sequence_name) => { - let table_def = plan.new.stored_in_table_def(sequence_name).unwrap(); - let sequence_def = table_def.sequences.get(sequence_name).unwrap(); - - let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); + spacetimedb_schema::auto_migrate::AutoMigrateStep::AddSequence(key) => { + let (namespace, sequence_name) = key; + let (owning_def, table_def) = plan + .new + .find_storing_table(namespace, sequence_name) + .ok_or_else(|| anyhow::anyhow!("AddSequence: `{sequence_name}` not found in new module def"))?; + let table_full_name = joined(namespace, &table_def.name); + let sequence_def: &SequenceDef = plan.new.lookup(key).ok_or_else(|| { + anyhow::anyhow!("AddSequence: sequence `{sequence_name}` not found in new module def") + })?; + let table_id = stdb.table_id_from_name_mut(tx, &table_full_name)?.unwrap(); let table_schema = stdb.schema_for_table_mut(tx, table_id)?; - log!( - logger, - "Adding sequence `{}` to table `{}`", - sequence_name, - table_def.name - ); - let sequence_schema = - SequenceSchema::from_module_def(plan.new, sequence_def, table_schema.table_id, 0.into()); + log!(logger, "Adding sequence `{sequence_name}` to table `{table_full_name}`"); + let mut sequence_schema = + SequenceSchema::from_module_def(owning_def, sequence_def, table_schema.table_id, 0.into()); + + // Apply namespace prefix for submodule sequences + sequence_schema.sequence_name = namespace.join_raw(&sequence_schema.sequence_name); stdb.create_sequence(tx, sequence_schema)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveSequence(sequence_name) => { - let table_def = plan.old.stored_in_table_def(sequence_name).unwrap(); - - let table_id = stdb.table_id_from_name_mut(tx, &table_def.name)?.unwrap(); + spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveSequence(key) => { + let (namespace, sequence_name) = key; + let (_owning_def, table_def) = plan + .old + .find_storing_table(namespace, sequence_name) + .ok_or_else(|| anyhow::anyhow!("RemoveSequence: `{sequence_name}` not found in old module def"))?; + let table_full_name = joined(namespace, &table_def.name); + let stored_name = namespace.join_raw(&sequence_name.clone().into()); + let table_id = stdb.table_id_from_name_mut(tx, &table_full_name)?.unwrap(); let table_schema = stdb.schema_for_table_mut(tx, table_id)?; let sequence_schema = table_schema .sequences .iter() - .find(|sequence| sequence.sequence_name[..] == sequence_name[..]) + .find(|sequence| sequence.sequence_name == stored_name) .unwrap(); log!( logger, - "Dropping sequence `{}` from table `{}`", - sequence_name, - table_def.name + "Dropping sequence `{sequence_name}` from table `{table_full_name}`" ); stdb.drop_sequence(tx, sequence_schema.sequence_id)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::ChangeColumns(table_name) => { - let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap(); - let table_id = stdb.table_id_from_name_mut(tx, table_name).unwrap().unwrap(); - let column_schemas = column_schemas_from_defs(plan.new, &table_def.columns, table_id); + spacetimedb_schema::auto_migrate::AutoMigrateStep::ChangeColumns(table_name_key) => { + let (namespace, local) = table_name_key; + let table_name = joined(namespace, local); + let (owning_def, table_def) = plan.new.find_table(table_name_key).ok_or_else(|| { + anyhow::anyhow!("ChangeColumns: table `{table_name}` not found in new module def") + })?; + let table_id = stdb.table_id_from_name_mut(tx, &table_name).unwrap().unwrap(); + let column_schemas = column_schemas_from_defs(owning_def, &table_def.columns, table_id); - log!(logger, "Changing columns of table `{}`", table_name); + log!(logger, "Changing columns of table `{table_name}`"); stdb.alter_table_row_type(tx, table_id, column_schemas)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::ReschemaEventTable(table_name) => { - let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap(); - let table_id = stdb.table_id_from_name_mut(tx, table_name).unwrap().unwrap(); - let column_schemas = column_schemas_from_defs(plan.new, &table_def.columns, table_id); + spacetimedb_schema::auto_migrate::AutoMigrateStep::ReschemaEventTable(table_name_key) => { + let (namespace, local) = table_name_key; + let table_name = joined(namespace, local); + let (owning_def, table_def) = plan.new.find_table(table_name_key).ok_or_else(|| { + anyhow::anyhow!("ReschemaEventTable: table `{table_name}` not found in new module def") + })?; + let table_id = stdb.table_id_from_name_mut(tx, &table_name).unwrap().unwrap(); + let column_schemas = column_schemas_from_defs(owning_def, &table_def.columns, table_id); log!(logger, "Changing schema of event table `{}`", table_name); stdb.alter_event_table_row_type(tx, table_id, column_schemas)?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::ChangeAccess(table_name) => { - let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap(); - stdb.alter_table_access(tx, table_name, table_def.table_access.into())?; + spacetimedb_schema::auto_migrate::AutoMigrateStep::ChangeAccess(table_name_key) => { + let (namespace, local) = table_name_key; + let table_name = joined(namespace, local); + let access = if let Some((_owning_def, table_def)) = plan.new.find_table(table_name_key) { + table_def.table_access + } else { + let (_owning_def, view_def) = plan.new.find_view(table_name_key).ok_or_else(|| { + anyhow::anyhow!("ChangeAccess: `{table_name}` not found as a table or view in new module def") + })?; + if view_def.is_public { + TableAccess::Public + } else { + TableAccess::Private + } + }; + stdb.alter_table_access(tx, &table_name, access.into())?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::ChangePrimaryKey(table_name) => { - let table_def = plan.new.stored_in_table_def(&table_name.clone().into()).unwrap(); + spacetimedb_schema::auto_migrate::AutoMigrateStep::ChangePrimaryKey(table_name_key) => { + let (namespace, local) = table_name_key; + let table_name = joined(namespace, local); + let (_owning_def, table_def) = plan.new.find_table(table_name_key).ok_or_else(|| { + anyhow::anyhow!("ChangePrimaryKey: table `{table_name}` not found in new module def") + })?; log!(logger, "Changing primary key for table `{table_name}`"); - stdb.alter_table_primary_key(tx, table_name, table_def.primary_key)?; + stdb.alter_table_primary_key(tx, &table_name, table_def.primary_key)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::AddSchedule(_) => { anyhow::bail!("Adding schedules is not yet implemented"); @@ -416,22 +515,26 @@ fn auto_migrate_database( } spacetimedb_schema::auto_migrate::AutoMigrateStep::AddRowLevelSecurity(sql_rls) => { log!(logger, "Adding row-level security `{sql_rls}`"); - let rls = plan.new.lookup_expect(sql_rls); + let rls = plan.new.lookup::(sql_rls).ok_or_else(|| { + anyhow::anyhow!("AddRowLevelSecurity: RLS `{sql_rls}` not found in new module def") + })?; let rls = RowLevelExpr::build_row_level_expr(tx, &auth_ctx, rls)?; stdb.create_row_level_security(tx, rls.def)?; } spacetimedb_schema::auto_migrate::AutoMigrateStep::RemoveRowLevelSecurity(sql_rls) => { - log!(logger, "Removing-row level security `{sql_rls}`"); + log!(logger, "Removing row-level security `{sql_rls}`"); stdb.drop_row_level_security(tx, sql_rls.clone())?; } - spacetimedb_schema::auto_migrate::AutoMigrateStep::AddColumns(table_name) => { - let table_def = plan + spacetimedb_schema::auto_migrate::AutoMigrateStep::AddColumns(table_name_key) => { + let (namespace, local) = table_name_key; + let table_name = joined(namespace, local); + let (owning_def, table_def) = plan .new - .stored_in_table_def(&table_name.clone().into()) - .expect("table must exist"); - let table_id = stdb.table_id_from_name_mut(tx, table_name).unwrap().unwrap(); - let column_schemas = column_schemas_from_defs(plan.new, &table_def.columns, table_id); + .find_table(table_name_key) + .ok_or_else(|| anyhow::anyhow!("AddColumns: table `{table_name}` not found in new module def"))?; + let table_id = stdb.table_id_from_name_mut(tx, &table_name).unwrap().unwrap(); + let column_schemas = column_schemas_from_defs(owning_def, &table_def.columns, table_id); let default_values: Vec = table_def .columns @@ -460,9 +563,80 @@ pub fn create_table_from_def( module_def: &ModuleDef, table_def: &TableDef, ) -> anyhow::Result<()> { - let schema = TableSchema::from_module_def(module_def, table_def, (), TableId::SENTINEL); + create_table_from_def_with_prefix(stdb, tx, module_def, table_def, &NamespacePath::root()) +} + +/// Creates a submodule table in `stdb`, applying the namespace to its canonical name. +/// `name_prefix` is the dot-terminated namespace string (e.g. `"alias."`). +pub fn create_table_from_def_with_prefix( + stdb: &RelationalDB, + tx: &mut MutTxId, + owning_def: &ModuleDef, + table_def: &TableDef, + name_prefix: &NamespacePath, +) -> anyhow::Result<()> { + let mut schema = TableSchema::from_module_def(owning_def, table_def, (), TableId::SENTINEL); + if !name_prefix.is_empty() { + // Store submodule tables under their namespaced *canonical* name, exactly as root + // tables are stored. The accessor name is a codegen concern and lives on as a + // namespaced alias in `st_table_accessor`; it must not become the host's identity + // for the table, and it must not be dropped (namespacing it keeps it unique across + // mounts that share a local name). + schema.table_name = TableName::from(name_prefix.join(table_def.name.clone())); + schema.alias = Some(name_prefix.join(table_def.accessor_name.clone())); + + // Apply the namespace to the scheduled reducer/procedure name so the scheduler can + // resolve it via the namespaced reducer_by_name / procedure_by_name. + if let Some(schedule) = &mut schema.schedule { + // `ScheduleSchema::from_module_def` builds this from a `ScheduleDef`'s single + // `Identifier`, so it has exactly one segment here; take the last rather than + // asserting, so a nested mount can never panic mid-migration. + let local_fn = schedule.function_name.local_name().clone(); + schedule.function_name = name_prefix.join(local_fn); + } + + // Apply the namespace to index canonical names and aliases for global uniqueness. + for index in &mut schema.indexes { + index.index_name = name_prefix.join_raw(&index.index_name); + index.alias = index.alias.as_ref().map(|alias| name_prefix.join_raw(alias)); + } + + // Same for constraint and sequence names. + for constraint in &mut schema.constraints { + constraint.constraint_name = name_prefix.join_raw(&constraint.constraint_name); + } + for sequence in &mut schema.sequences { + sequence.sequence_name = name_prefix.join_raw(&sequence.sequence_name); + } + } stdb.create_table(tx, schema) - .with_context(|| format!("failed to create table {}", &table_def.name))?; + .with_context(|| format!("failed to create table {}", name_prefix.join(table_def.name.clone())))?; + Ok(()) +} + +/// Creates the table for `view_def` in `stdb`. +pub fn create_table_from_view_def( + stdb: &RelationalDB, + tx: &mut MutTxId, + module_def: &ModuleDef, + view_def: &ViewDef, +) -> anyhow::Result<()> { + stdb.create_view(tx, module_def, view_def) + .with_context(|| format!("failed to create table for view {}", &view_def.name))?; + Ok(()) +} + +/// Creates the table for a submodule `view_def` in `stdb`, applying the namespace prefix. +/// `name_prefix` is the dot-terminated namespace string (e.g. `"lib."`). +pub fn create_table_from_view_def_with_prefix( + stdb: &RelationalDB, + tx: &mut MutTxId, + owning_def: &ModuleDef, + view_def: &ViewDef, + name_prefix: &NamespacePath, +) -> anyhow::Result<()> { + stdb.create_view_with_prefix(tx, owning_def, view_def, name_prefix) + .with_context(|| format!("failed to create table for view {}{}", name_prefix, &view_def.name))?; Ok(()) } @@ -477,7 +651,7 @@ mod test { use spacetimedb_datastore::system_tables::ST_EVENT_TABLE_ID; use spacetimedb_lib::{ db::raw_def::{ - v10::RawModuleDefV10Builder, + v10::{RawModuleDefV10Builder, RawModuleDefV10Section, RawSubmoduleV10}, v9::{btree, RawIndexAlgorithm, RawModuleDefV9Builder, TableAccess}, }, Identity, @@ -668,6 +842,143 @@ mod test { .expect("should be a valid module definition") } + fn submodule_table_module(with_sub_objects: bool) -> ModuleDef { + let mut sub = RawModuleDefV10Builder::new(); + let table = sub.build_table_with_new_type("sessions", ProductType::from([("id", U64)]), true); + if with_sub_objects { + table + .with_index(btree(0), "sessions_id_idx", "sessions_id_idx") + .with_unique_constraint(0) + .with_column_sequence(0) + .finish(); + } else { + table.finish(); + } + let mut root = RawModuleDefV10Builder::new().finish(); + root.sections + .push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "lib".to_string(), + module: sub.finish(), + }])); + root.try_into().expect("should be a valid module definition") + } + + /// A submodule table whose source name differs from its canonical name. + /// + /// Under the V10 snake_case policy, source `"FruitBasket"` canonicalizes to + /// `fruit_basket` while `accessor_name` keeps the raw `FruitBasket`. + fn submodule_case_converted_module() -> ModuleDef { + let mut sub = RawModuleDefV10Builder::new(); + sub.build_table_with_new_type("FruitBasket", ProductType::from([("id", U64)]), true) + .finish(); + let mut root = RawModuleDefV10Builder::new().finish(); + root.sections + .push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "lib".to_string(), + module: sub.finish(), + }])); + root.try_into().expect("should be a valid module definition") + } + + /// Submodule tables must be stored under their namespaced *canonical* name, with the + /// accessor name kept as a namespaced alias -- exactly as root tables are. + /// + /// Storing the accessor name as the canonical name makes the host resolve tables by a + /// name that only exists for client codegen, and discarding the alias loses the mapping + /// codegen needs. Neither shows up when accessor == canonical, which is why this uses a + /// case-converted name. + #[test] + fn submodule_table_is_stored_under_canonical_name() -> anyhow::Result<()> { + let stdb = TestDB::durable()?; + let module_def = submodule_case_converted_module(); + + // Precondition: the two names really do differ, or this test proves nothing. + let (prefix, owning_def, table_def) = module_def + .all_tables_with_prefix() + .into_iter() + .next() + .expect("submodule table should exist"); + assert_eq!(&*table_def.name, "fruit_basket", "canonical name"); + assert_eq!(&*table_def.accessor_name, "FruitBasket", "accessor name"); + + let mut tx = begin_mut_tx(&stdb); + create_table_from_def_with_prefix(&stdb, &mut tx, owning_def, table_def, &prefix)?; + + let table_id = stdb + .table_id_from_name_mut(&tx, "lib.fruit_basket")? + .expect("submodule table should be stored under its namespaced canonical name"); + + // And the accessor name survives as a namespaced alias, so codegen can still resolve it. + let schema = stdb.schema_for_table_mut(&tx, table_id)?; + assert_eq!( + schema.alias.as_deref(), + Some("lib.FruitBasket"), + "accessor name should be kept as a namespaced alias, not discarded" + ); + + stdb.commit_tx(tx)?; + Ok(()) + } + + #[test] + fn submodule_constraint_and_sequence_migration() -> anyhow::Result<()> { + let auth_ctx = AuthCtx::for_testing(); + let stdb = TestDB::durable()?; + + let old = submodule_table_module(true); + let new = submodule_table_module(false); + + let mut tx = begin_mut_tx(&stdb); + for (prefix, owning_def, def) in old.all_tables_with_prefix() { + create_table_from_def_with_prefix(&stdb, &mut tx, owning_def, def, &prefix)?; + } + stdb.commit_tx(tx)?; + + let mut tx = begin_mut_tx(&stdb); + let table_id = stdb + .table_id_from_name_mut(&tx, "lib.sessions")? + .expect("submodule table should exist"); + let schema = stdb.schema_for_table_mut(&tx, table_id)?; + assert!(!schema.constraints.is_empty()); + assert!(!schema.sequences.is_empty()); + assert!( + schema.constraints.iter().all(|c| c.constraint_name.starts_with("lib.")), + "constraint names should be namespace-prefixed: {:?}", + schema.constraints + ); + assert!( + schema.sequences.iter().all(|s| s.sequence_name.starts_with("lib.")), + "sequence names should be namespace-prefixed: {:?}", + schema.sequences + ); + + // Removing the constraint and sequence must resolve them by their prefixed names. + let plan = ponder_migrate(&old, &new)?; + let _ = update_database(&stdb, &mut tx, auth_ctx.clone(), plan, &TestLogger)?; + + let schema = stdb.schema_for_table_mut(&tx, table_id)?; + assert!(schema.constraints.is_empty(), "{:?}", schema.constraints); + assert!(schema.sequences.is_empty(), "{:?}", schema.sequences); + + // And adding them back must store prefixed names again. + let plan = ponder_migrate(&new, &old)?; + let _ = update_database(&stdb, &mut tx, auth_ctx, plan, &TestLogger)?; + + let schema = stdb.schema_for_table_mut(&tx, table_id)?; + assert!( + !schema.constraints.is_empty() && schema.constraints.iter().all(|c| c.constraint_name.starts_with("lib.")), + "{:?}", + schema.constraints + ); + assert!( + !schema.sequences.is_empty() && schema.sequences.iter().all(|s| s.sequence_name.starts_with("lib.")), + "{:?}", + schema.sequences + ); + + Ok(()) + } + fn view_module() -> ModuleDef { let mut builder = RawModuleDefV10Builder::new(); let return_type_ref = builder.add_algebraic_type( @@ -724,7 +1035,7 @@ mod test { let MigratePlan::Auto(plan) = &plan else { panic!("expected auto migration"); }; - let my_view = module_def.view("my_view").unwrap().key(); + let my_view = module_def.view("my_view").expect("view should exist").key(); assert!(plan.steps.contains(&AutoMigrateStep::RemoveView(my_view))); assert!(plan.steps.contains(&AutoMigrateStep::AddView(my_view))); assert!(plan.steps.contains(&AutoMigrateStep::DisconnectAllUsers)); diff --git a/crates/expr/src/check.rs b/crates/expr/src/check.rs index ca3bb0304f2..c0af396496f 100644 --- a/crates/expr/src/check.rs +++ b/crates/expr/src/check.rs @@ -4,7 +4,7 @@ use spacetimedb_data_structures::map::HashMap; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::AlgebraicType; use spacetimedb_primitives::TableId; -use spacetimedb_sats::raw_identifier::RawIdentifier; +use spacetimedb_sats::raw_identifier::RawNamespacedIdentifier; use spacetimedb_schema::schema::TableOrViewSchema; use spacetimedb_sql_parser::ast::BinOp; use spacetimedb_sql_parser::{ @@ -35,10 +35,10 @@ pub trait SchemaView { } #[derive(Default)] -pub struct Relvars(HashMap>); +pub struct Relvars(HashMap>); impl Deref for Relvars { - type Target = HashMap>; + type Target = HashMap>; fn deref(&self) -> &Self::Target { &self.0 } diff --git a/crates/expr/src/errors.rs b/crates/expr/src/errors.rs index 489882ce8c8..1e7315cb4eb 100644 --- a/crates/expr/src/errors.rs +++ b/crates/expr/src/errors.rs @@ -1,7 +1,7 @@ use super::statement::InvalidVar; use spacetimedb_lib::AlgebraicType; use spacetimedb_sats::algebraic_type::fmt::fmt_algebraic_type; -use spacetimedb_sats::raw_identifier::RawIdentifier; +use spacetimedb_sats::raw_identifier::RawNamespacedIdentifier; use spacetimedb_schema::table_name::TableName; use spacetimedb_sql_parser::ast::BinOp; use spacetimedb_sql_parser::parser::errors::SqlParseError; @@ -118,7 +118,7 @@ impl UnexpectedType { #[derive(Debug, Error)] #[error("Duplicate name `{0}`")] -pub struct DuplicateName(pub RawIdentifier); +pub struct DuplicateName(pub RawNamespacedIdentifier); #[derive(Debug, Error)] #[error("`filter!` does not support column projections; Must return table rows")] diff --git a/crates/expr/src/expr.rs b/crates/expr/src/expr.rs index d0534fd9475..24c6e8b81a6 100644 --- a/crates/expr/src/expr.rs +++ b/crates/expr/src/expr.rs @@ -1,7 +1,7 @@ use spacetimedb_data_structures::map::HashSet; use spacetimedb_lib::{query::Delta, AlgebraicType, AlgebraicValue}; use spacetimedb_primitives::{TableId, ViewId}; -use spacetimedb_sats::raw_identifier::RawIdentifier; +use spacetimedb_sats::raw_identifier::{RawIdentifier, RawNamespacedIdentifier}; use spacetimedb_schema::{identifier::Identifier, schema::TableOrViewSchema}; use spacetimedb_sql_parser::ast::{BinOp, LogOp, Parameter}; use std::sync::Arc; @@ -41,7 +41,7 @@ impl CollectViews for Vec { #[derive(Debug, PartialEq, Eq)] pub enum ProjectName { None(RelExpr), - Some(RelExpr, RawIdentifier), + Some(RelExpr, RawNamespacedIdentifier), } impl CollectViews for ProjectName { @@ -62,7 +62,7 @@ impl ProjectName { /// What is the name of the return table? /// This is either the table name itself or its alias. - pub fn return_name(&self) -> Option<&RawIdentifier> { + pub fn return_name(&self) -> Option<&RawNamespacedIdentifier> { match self { Self::None(input) => input.return_name(), Self::Some(_, name) => Some(name), @@ -256,7 +256,7 @@ pub struct Relvar { /// The table schema of this relvar pub schema: Arc, /// The name of this relvar - pub alias: RawIdentifier, + pub alias: RawNamespacedIdentifier, /// Does this relvar represent a delta table? pub delta: Option, } @@ -356,7 +356,7 @@ impl RelExpr { /// Does this expression return a single relvar? /// If so, return its name or equivalently its alias. - pub fn return_name(&self) -> Option<&RawIdentifier> { + pub fn return_name(&self) -> Option<&RawNamespacedIdentifier> { match self { Self::RelVar(Relvar { alias, .. }) => Some(alias), Self::Select(input, _) => input.return_name(), @@ -436,7 +436,7 @@ impl Expr { /// A typed qualified field projection #[derive(Debug, Clone, PartialEq, Eq)] pub struct FieldProject { - pub table: RawIdentifier, + pub table: RawNamespacedIdentifier, pub field: usize, pub ty: AlgebraicType, } diff --git a/crates/expr/src/lib.rs b/crates/expr/src/lib.rs index e27a3b735e3..fc0074d5525 100644 --- a/crates/expr/src/lib.rs +++ b/crates/expr/src/lib.rs @@ -17,6 +17,7 @@ use spacetimedb_lib::Timestamp; use spacetimedb_lib::{from_hex_pad, AlgebraicType, AlgebraicValue, ConnectionId, Identity}; use spacetimedb_sats::algebraic_type::fmt::fmt_algebraic_type; use spacetimedb_sats::algebraic_value::ser::ValueSerializer; +use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_sats::uuid::Uuid; use spacetimedb_schema::schema::ColumnSchema; use spacetimedb_sql_parser::ast::{self, BinOp, Parameter, ProjectElem, SqlExpr, SqlIdent, SqlLiteral}; @@ -59,9 +60,13 @@ pub(crate) fn type_proj(input: RelExpr, proj: ast::Project, vars: &Relvars) -> T Ok(ProjectList::Name(vec![ProjectName::Some(input, var)])) } ast::Project::Star(Some(SqlIdent(var))) => Err(Unresolved::var(&var).into()), - ast::Project::Count(SqlIdent(alias)) => { - Ok(ProjectList::Agg(vec![input], AggType::Count, alias, AlgebraicType::U64)) - } + ast::Project::Count(SqlIdent(alias)) => Ok(ProjectList::Agg( + vec![input], + AggType::Count, + // A column alias is always a single SQL identifier, never a qualified name. + RawIdentifier::new(&*alias), + AlgebraicType::U64, + )), ast::Project::Exprs(elems) => { let mut projections = vec![]; let mut names = HashSet::new(); @@ -72,7 +77,8 @@ pub(crate) fn type_proj(input: RelExpr, proj: ast::Project, vars: &Relvars) -> T } if let Expr::Field(p) = type_expr(vars, expr.into(), None)? { - projections.push((alias, p)); + // A column alias is always a single SQL identifier, never a qualified name. + projections.push((RawIdentifier::new(&*alias), p)); } } diff --git a/crates/expr/src/rls.rs b/crates/expr/src/rls.rs index e2bca3b7bd2..f9d6efc0eb8 100644 --- a/crates/expr/src/rls.rs +++ b/crates/expr/src/rls.rs @@ -2,7 +2,7 @@ use std::rc::Rc; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_primitives::TableId; -use spacetimedb_sats::raw_identifier::RawIdentifier; +use spacetimedb_sats::raw_identifier::RawNamespacedIdentifier; use spacetimedb_sql_parser::ast::BinOp; use crate::{ @@ -290,7 +290,7 @@ fn resolve_views_for_expr( /// this function handles the actual replacement of the view with its definition. fn expand_views( expr: RelExpr, - view_def_fragments: &[(TableId, RawIdentifier, Vec)], + view_def_fragments: &[(TableId, RawNamespacedIdentifier, Vec)], out: &mut Vec, ) { match view_def_fragments { @@ -337,19 +337,19 @@ fn resolve_views_for_expr( /// JOIN t AS t ON t.id = v.id WHERE v.x = 0 /// ``` fn alpha_rename_fragments( - return_name: &RawIdentifier, - outer_alias: &RawIdentifier, + return_name: &RawNamespacedIdentifier, + outer_alias: &RawNamespacedIdentifier, inputs: Vec, output: &mut Vec, suffix: &mut usize, ) { for mut fragment in inputs { *suffix += 1; - alpha_rename(&mut fragment, &mut |name: &RawIdentifier| { + alpha_rename(&mut fragment, &mut |name: &RawNamespacedIdentifier| { if name == return_name { return outer_alias.clone(); } - RawIdentifier::new(name.to_string() + "_" + &suffix.to_string()) + RawNamespacedIdentifier::new(name.to_string() + "_" + &suffix.to_string()) }); output.push(fragment); } @@ -357,13 +357,13 @@ fn alpha_rename_fragments( /// When expanding a view, we must do an alpha conversion on the view definition. /// This involves renaming the table aliases before replacing the view reference. -fn alpha_rename(expr: &mut RelExpr, f: &mut impl FnMut(&RawIdentifier) -> RawIdentifier) { +fn alpha_rename(expr: &mut RelExpr, f: &mut impl FnMut(&RawNamespacedIdentifier) -> RawNamespacedIdentifier) { /// Helper for renaming a relvar - fn rename(relvar: &mut Relvar, f: &mut impl FnMut(&RawIdentifier) -> RawIdentifier) { + fn rename(relvar: &mut Relvar, f: &mut impl FnMut(&RawNamespacedIdentifier) -> RawNamespacedIdentifier) { relvar.alias = f(&relvar.alias); } /// Helper for renaming a field reference - fn rename_field(field: &mut FieldProject, f: &mut impl FnMut(&RawIdentifier) -> RawIdentifier) { + fn rename_field(field: &mut FieldProject, f: &mut impl FnMut(&RawNamespacedIdentifier) -> RawNamespacedIdentifier) { field.table = f(&field.table); } expr.visit_mut(&mut |expr| match expr { diff --git a/crates/expr/src/statement.rs b/crates/expr/src/statement.rs index ed45aad73bb..9fbdb869a20 100644 --- a/crates/expr/src/statement.rs +++ b/crates/expr/src/statement.rs @@ -2,7 +2,7 @@ use std::sync::Arc; use spacetimedb_lib::{identity::AuthCtx, st_var::StVarValue, AlgebraicType, AlgebraicValue, ProductValue}; use spacetimedb_primitives::{ColId, TableId}; -use spacetimedb_sats::raw_identifier::RawIdentifier; +use spacetimedb_sats::raw_identifier::RawNamespacedIdentifier; use spacetimedb_schema::schema::{ColumnSchema, TableOrViewSchema}; use spacetimedb_schema::table_name::TableName; use spacetimedb_sql_parser::{ @@ -67,13 +67,13 @@ pub struct TableInsert { pub struct TableDelete { pub table: Arc, - pub alias: RawIdentifier, + pub alias: RawNamespacedIdentifier, pub filter: Option, } pub struct TableUpdate { pub table: Arc, - pub alias: RawIdentifier, + pub alias: RawNamespacedIdentifier, pub columns: Box<[(ColId, AlgebraicValue)]>, pub filter: Option, } @@ -255,7 +255,7 @@ pub fn type_update(update: SqlUpdate, tx: &impl SchemaView) -> TypingResult), + + /// Submodules, keyed by the namespace they are registered under. + Submodules(Vec), } #[derive(Debug, Clone, SpacetimeType)] @@ -124,6 +127,14 @@ pub enum MethodOrAny { Method(crate::http::Method), } +#[derive(Debug, Clone, SpacetimeType)] +#[sats(crate = crate)] +#[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] +pub struct RawSubmoduleV10 { + pub namespace: String, + pub module: RawModuleDefV10, +} + #[derive(Debug, Clone, Copy, Default, SpacetimeType)] #[cfg_attr(feature = "test", derive(PartialEq, Eq, PartialOrd, Ord))] #[sats(crate = crate)] @@ -558,6 +569,14 @@ pub struct RawViewPrimaryKeyDefV10 { } impl RawModuleDefV10 { + /// Get the submodules for this module definition. + pub fn submodules(&self) -> Option<&Vec> { + self.sections.iter().find_map(|s| match s { + RawModuleDefV10Section::Submodules(submodules) => Some(submodules), + _ => None, + }) + } + /// Get the types section, if present. pub fn types(&self) -> Option<&Vec> { self.sections.iter().find_map(|s| match s { diff --git a/crates/sats/src/raw_identifier.rs b/crates/sats/src/raw_identifier.rs index dc5da05af97..1a2e9da2b03 100644 --- a/crates/sats/src/raw_identifier.rs +++ b/crates/sats/src/raw_identifier.rs @@ -66,3 +66,102 @@ impl From for RawIdentifier { RawIdentifier(s.to_lean_string()) } } + +/// A not-yet-validated, dot-separated name, e.g. `"lib.library_table"`. +/// +/// This is the raw counterpart of `spacetimedb_schema`'s `NamespacedIdentifier`, +/// in the same way that [`RawIdentifier`] is the raw counterpart of `Identifier`. +/// +/// The distinction matters: a [`RawIdentifier`] can be validated into a single +/// `Identifier`, but a name containing `.` never can be, since `.` is not a legal +/// identifier character. Names that may carry a namespace therefore use this type +/// rather than [`RawIdentifier`], so that the two cannot be confused. +#[derive(Default, PartialEq, Eq, PartialOrd, Ord, Hash, Clone)] +pub struct RawNamespacedIdentifier(LeanString); + +impl_st!([] RawNamespacedIdentifier, _ts => AlgebraicType::String); +impl_serialize!([] RawNamespacedIdentifier, (self, ser) => ser.serialize_str(&self.0)); +impl_deserialize!([] RawNamespacedIdentifier, de => LeanString::deserialize(de).map(Self)); + +impl RawNamespacedIdentifier { + /// Creates a new `RawNamespacedIdentifier` from a string. + pub fn new(name: impl Into) -> Self { + Self(name.into()) + } + + /// The dot-separated segments of this name, in order. + /// + /// Always yields at least one item; an empty name yields one empty segment. + pub fn segments(&self) -> impl Iterator + Clone { + self.0.split('.') + } + + /// The final segment, i.e. the name with any namespace prefix stripped. + /// + /// `"lib.sessions_id_idx"` yields `"sessions_id_idx"`; an un-namespaced name + /// yields itself. + pub fn local_name(&self) -> &str { + // `rsplit` on a non-empty pattern always yields at least one item. + self.0.rsplit('.').next().unwrap_or(&self.0) + } + + /// Whether this name carries a namespace prefix. + pub fn is_namespaced(&self) -> bool { + self.0.contains('.') + } + + pub fn into_inner(self) -> LeanString { + self.0 + } +} + +impl Deref for RawNamespacedIdentifier { + type Target = str; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl AsRef for RawNamespacedIdentifier { + fn as_ref(&self) -> &str { + &self.0 + } +} + +impl Borrow for RawNamespacedIdentifier { + fn borrow(&self) -> &str { + &self.0 + } +} + +impl fmt::Debug for RawNamespacedIdentifier { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.0, f) + } +} + +impl fmt::Display for RawNamespacedIdentifier { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Display::fmt(&self.0, f) + } +} + +impl From for RawNamespacedIdentifier { + /// Every single identifier is a one-segment namespaced name. + fn from(id: RawIdentifier) -> Self { + Self(id.0) + } +} + +impl From<&'static str> for RawNamespacedIdentifier { + fn from(s: &'static str) -> Self { + Self(LeanString::from_static_str(s)) + } +} + +impl From for RawNamespacedIdentifier { + fn from(s: String) -> Self { + Self(s.to_lean_string()) + } +} diff --git a/crates/schema/src/auto_migrate.rs b/crates/schema/src/auto_migrate.rs index 63239de7ecc..a516815fea1 100644 --- a/crates/schema/src/auto_migrate.rs +++ b/crates/schema/src/auto_migrate.rs @@ -1,7 +1,12 @@ use core::{cmp::Ordering, ops::BitOr}; -use crate::{def::*, error::PrettyAlgebraicType, identifier::Identifier}; +use crate::{ + def::*, + error::PrettyAlgebraicType, + identifier::{Identifier, NamespacePath}, +}; use formatter::format_plan; +use spacetimedb_data_structures::map::HashMap; use spacetimedb_data_structures::{ error_stream::{CollectAllErrors, CombineErrors, ErrorStream}, map::{HashCollectionExt as _, HashSet}, @@ -215,7 +220,7 @@ pub struct AutoMigratePlan<'def> { } impl AutoMigratePlan<'_> { - fn any_step(&self, f: impl Fn(&AutoMigrateStep) -> bool) -> bool { + fn any_step(&self, f: impl Fn(&AutoMigrateStep<'_>) -> bool) -> bool { self.steps.iter().any(f) } @@ -242,6 +247,10 @@ pub enum AutoMigratePrecheck<'def> { } /// A step in an automatic migration. +/// +/// Payloads are [`ModuleDefLookup`] keys: a `(namespace, name)` pair, where the namespace is +/// the path the owning module is mounted under and is empty for root-level items. This lets +/// submodule and root-level items be handled uniformly. #[derive(PartialEq, Eq, Debug, PartialOrd, Ord)] pub enum AutoMigrateStep<'def> { // It is important FOR CORRECTNESS that `Remove` variants are declared before `Add` variants in this enum! @@ -270,17 +279,16 @@ pub enum AutoMigrateStep<'def> { RemoveSequence(::Key<'def>), /// Remove a schedule annotation from a table. RemoveSchedule(::Key<'def>), - /// Remove a view and corresponding view table + /// Remove a view and corresponding view table. RemoveView(::Key<'def>), - /// Remove a row-level security query. + /// Remove a row-level security query. Payload is the SQL text. RemoveRowLevelSecurity(::Key<'def>), + /// Remove an empty table and all its sub-objects (indexes, constraints, sequences). /// Validated at execution time: fails if the table contains data. RemoveTable(::Key<'def>), /// Change the column types of a table, in a layout compatible way. - /// - /// This should be done before any new indices are added. ChangeColumns(::Key<'def>), /// Change the column types of an event table, in a way that may not be layout-compatible. @@ -289,13 +297,9 @@ pub enum AutoMigrateStep<'def> { /// Add columns to a table, in a layout-INCOMPATIBLE way. /// /// This is a destructive operation that requires first running a `DisconnectAllUsers`. - /// - /// The added columns are guaranteed to be contiguous - /// and at the end of the table. + /// The added columns are guaranteed to be contiguous and at the end of the table. /// They are also guaranteed to have default values set. - /// - /// When this step is present, - /// no `ChangeColumns` steps will be, for the same table. + /// When this step is present, no `ChangeColumns` steps will be, for the same table. AddColumns(::Key<'def>), /// Add a table, including all indexes, constraints, and sequences. @@ -309,24 +313,22 @@ pub enum AutoMigrateStep<'def> { AddSequence(::Key<'def>), /// Add a schedule annotation to a table. AddSchedule(::Key<'def>), - /// Add a view and corresponding view table + /// Add a view and corresponding view table. AddView(::Key<'def>), - /// Add a row-level security query. + /// Add a row-level security query. Payload is the SQL text. AddRowLevelSecurity(::Key<'def>), - /// Change the access of a table. + /// Change the access of a table or view. ChangeAccess(::Key<'def>), /// Change the primary key of a table. /// - /// This updates the `table_primary_key` field in `st_table` - /// to match the new module definition. - /// Without this step, a stale primary key in the stored schema - /// causes `check_compatible` to fail on the next publish. - /// See: + /// This updates the `table_primary_key` field in `st_table` to match the new module definition. + /// Without this step, a stale primary key in the stored schema causes `check_compatible` to + /// fail on the next publish. See: ChangePrimaryKey(::Key<'def>), - /// Recompute a view, update its backing table, and push updates to clients + /// Recompute a view, update its backing table, and push updates to clients. UpdateView(::Key<'def>), /// Disconnect all users connected to the module. @@ -486,26 +488,19 @@ pub fn ponder_auto_migrate<'def>(old: &'def ModuleDef, new: &'def ModuleDef) -> let views_ok = auto_migrate_views(&mut plan); let tables_ok = auto_migrate_tables(&mut plan); - // Filter out sub-objects of added/removed tables — they're handled by `AddTable`/`RemoveTable`. - let (new_tables, removed_tables): (HashSet<&Identifier>, HashSet<&Identifier>) = - diff(plan.old, plan.new, ModuleDef::tables).fold( - (HashSet::new(), HashSet::new()), - |(mut added, mut removed), diff| { - match diff { - Diff::Add { new } => { - added.insert(&new.name); - } - Diff::Remove { old } => { - removed.insert(&old.name); - } - Diff::MaybeChange { .. } => {} - } - (added, removed) - }, - ); - let indexes_ok = auto_migrate_indexes(&mut plan, &new_tables, &removed_tables); - let sequences_ok = auto_migrate_sequences(&mut plan, &new_tables, &removed_tables); - let constraints_ok = auto_migrate_constraints(&mut plan, &new_tables, &removed_tables); + // Sub-objects of added/removed tables are handled by AddTable/RemoveTable, not individually. + let old_module = plan.old; + let new_module = plan.new; + let old_table_names: HashSet> = + old_module.all_tables_with_prefix().into_iter().map(key_of).collect(); + let new_table_names: HashSet> = + new_module.all_tables_with_prefix().into_iter().map(key_of).collect(); + let added_tables: HashSet> = new_table_names.difference(&old_table_names).copied().collect(); + let removed_tables: HashSet> = old_table_names.difference(&new_table_names).copied().collect(); + + let indexes_ok = auto_migrate_indexes(&mut plan, &added_tables, &removed_tables); + let sequences_ok = auto_migrate_sequences(&mut plan, &added_tables, &removed_tables); + let constraints_ok = auto_migrate_constraints(&mut plan, &added_tables, &removed_tables); // IMPORTANT: RLS auto-migrate steps must come last, // since they assume that any schema changes, like adding or dropping tables, // have already been reflected in the database state. @@ -520,170 +515,228 @@ pub fn ponder_auto_migrate<'def>(old: &'def ModuleDef, new: &'def ModuleDef) -> Ok(plan) } -/// A diff between two items. -/// `Add` means the item is present in the new `ModuleDef` but not the old. -/// `Remove` means the item is present in the old `ModuleDef` but not the new. -/// `MaybeChange` indicates the item is present in both. -#[derive(Debug)] -enum Diff<'def, T> { - Add { new: &'def T }, - Remove { old: &'def T }, - MaybeChange { old: &'def T, new: &'def T }, -} +/// Shorthands for the namespace-qualified [`ModuleDefLookup`] keys used throughout planning. +type TableKey<'def> = ::Key<'def>; +type ViewKey<'def> = ::Key<'def>; +type IndexKey<'def> = ::Key<'def>; +type SequenceKey<'def> = ::Key<'def>; +type ConstraintKey<'def> = ::Key<'def>; -/// Diff a collection of items, looking them up in both the old and new `ModuleDef` by their `ModuleDefLookup::Key`. -/// Keys are required to be stable across migrations, which makes this possible. -fn diff<'def, T: ModuleDefLookup, I: Iterator>( - old: &'def ModuleDef, - new: &'def ModuleDef, - iter: impl Fn(&'def ModuleDef) -> I, -) -> impl Iterator> { - iter(old) - .map(move |old_item| match T::lookup(new, old_item.key()) { - Some(new_item) => Diff::MaybeChange { - old: old_item, - new: new_item, - }, - None => Diff::Remove { old: old_item }, - }) - .chain(iter(new).filter_map(move |new_item| { - if T::lookup(old, new_item.key()).is_none() { - Some(Diff::Add { new: new_item }) - } else { - None - } - })) +fn key_of<'def>((_, _, table): (NamespacePath, &'def ModuleDef, &'def TableDef)) -> TableKey<'def> { + table.key() } -fn auto_migrate_views(plan: &mut AutoMigratePlan<'_>) -> Result<()> { - diff(plan.old, plan.new, ModuleDef::views) - .map(|table_diff| -> Result<()> { - match table_diff { - Diff::Add { new } => { - plan.steps.push(AutoMigrateStep::AddView(new.key())); - Ok(()) - } +fn auto_migrate_views<'def>(plan: &mut AutoMigratePlan<'def>) -> Result<()> { + let old_module = plan.old; + let new_module = plan.new; + + // Views across all submodules, keyed by their namespace-qualified key. + let old_views: HashMap, (&'def ModuleDef, &'def ViewDef)> = old_module + .all_views_with_prefix() + .into_iter() + .map(|(_, owning, view)| (view.key(), (owning, view))) + .collect(); + let new_views: HashMap, (&'def ModuleDef, &'def ViewDef)> = new_module + .all_views_with_prefix() + .into_iter() + .map(|(_, owning, view)| (view.key(), (owning, view))) + .collect(); + + type ViewEntry<'a> = (ViewKey<'a>, (&'a ModuleDef, &'a ViewDef), (&'a ModuleDef, &'a ViewDef)); + let mut maybe_change: Vec> = vec![]; + for (key, old_entry) in &old_views { + match new_views.get(key) { + Some(new_entry) => maybe_change.push((*key, *old_entry, *new_entry)), + None => { // From the user's perspective, views do not have persistent state. // Hence removal does not require a manual migration - just disconnecting clients. - Diff::Remove { old } => { - plan.steps.push(AutoMigrateStep::RemoveView(old.key())); - plan.ensure_disconnect_all_users(); - Ok(()) - } - Diff::MaybeChange { old, new } => auto_migrate_view(plan, old, new), + plan.steps.push(AutoMigrateStep::RemoveView(*key)); + plan.ensure_disconnect_all_users(); } + } + } + for key in new_views.keys() { + if !old_views.contains_key(key) { + plan.steps.push(AutoMigrateStep::AddView(*key)); + } + } + + let results: Vec> = maybe_change + .into_iter() + .map(|(key, (old_owning, old_view), (new_owning, new_view))| { + auto_migrate_view(plan, key, old_owning, old_view, new_owning, new_view) }) - .collect_all_errors() + .collect(); + results.into_iter().collect_all_errors::>().map(|_| ()) } -fn auto_migrate_view<'def>(plan: &mut AutoMigratePlan<'def>, old: &'def ViewDef, new: &'def ViewDef) -> Result<()> { - let key = old.key(); - - if old.is_public != new.is_public { - plan.steps.push(AutoMigrateStep::ChangeAccess(key)); - } - +fn auto_migrate_view<'def>( + plan: &mut AutoMigratePlan<'def>, + key: ViewKey<'def>, + old_owning: &ModuleDef, + old: &ViewDef, + new_owning: &ModuleDef, + new: &ViewDef, +) -> Result<()> { // We can always auto-migrate a view because we can always re-compute it. // However certain things require us to disconnect clients: // 1. If we add or remove a column or parameter // 2. If we change the order of the columns or parameters // 3. If we change the types of the columns or parameters // 4. If we change the context parameter - let Any(incompatible_return_type) = diff(plan.old, plan.new, |def| { - def.lookup_expect::(key).return_columns.iter() - }) - .map(|col_diff| { - match col_diff { - // We must disconnect clients if we add or remove a parameter or column - Diff::Add { .. } | Diff::Remove { .. } => Any(true), - Diff::MaybeChange { old, new } => { - if old.col_id != new.col_id { + let old_return_cols: HashMap<&Identifier, &ViewColumnDef> = + old.return_columns.iter().map(|c| (&c.name, c)).collect(); + let new_return_cols: HashMap<&Identifier, &ViewColumnDef> = + new.return_columns.iter().map(|c| (&c.name, c)).collect(); + + let Any(incompatible_return_type) = old + .return_columns + .iter() + .map(|old_col| match new_return_cols.get(&old_col.name) { + None => Any(true), + Some(new_col) => { + if old_col.col_id != new_col.col_id { return Any(true); - }; - + } ensure_old_ty_upgradable_to_new( false, - &|| old.view_name.clone(), - &|| old.name.clone(), - &WithTypespace::new(plan.old.typespace(), &old.ty) + &|| old_col.view_name.clone(), + &|| old_col.name.clone(), + &WithTypespace::new(old_owning.typespace(), &old_col.ty) .resolve_refs() .expect("valid ViewDefs must have valid type refs"), - &WithTypespace::new(plan.new.typespace(), &new.ty) + &WithTypespace::new(new_owning.typespace(), &new_col.ty) .resolve_refs() .expect("valid ViewDefs must have valid type refs"), ) .unwrap_or(Any(true)) } - } - }) - .collect(); - - let Any(incompatible_param_types) = diff(plan.old, plan.new, |def| { - def.lookup_expect::(key).param_columns.iter() - }) - .map(|col_diff| { - match col_diff { - // We must disconnect clients if we add or remove a parameter or column - Diff::Add { .. } | Diff::Remove { .. } => Any(true), - Diff::MaybeChange { old, new } => { - if old.col_id != new.col_id { + }) + .chain(new.return_columns.iter().map(|new_col| { + if old_return_cols.contains_key(&new_col.name) { + Any(false) + } else { + Any(true) // added column → incompatible + } + })) + .collect(); + + let old_param_cols: HashMap<&Identifier, &ViewParamDef> = old.param_columns.iter().map(|c| (&c.name, c)).collect(); + let new_param_cols: HashMap<&Identifier, &ViewParamDef> = new.param_columns.iter().map(|c| (&c.name, c)).collect(); + + let Any(incompatible_param_types) = old + .param_columns + .iter() + .map(|old_col| match new_param_cols.get(&old_col.name) { + None => Any(true), + Some(new_col) => { + if old_col.col_id != new_col.col_id { return Any(true); - }; - + } ensure_old_ty_upgradable_to_new( false, - &|| old.view_name.clone(), - &|| old.name.clone(), - &WithTypespace::new(plan.old.typespace(), &old.ty) + &|| old_col.view_name.clone(), + &|| old_col.name.clone(), + &WithTypespace::new(old_owning.typespace(), &old_col.ty) .resolve_refs() .expect("valid ViewDefs must have valid type refs"), - &WithTypespace::new(plan.new.typespace(), &new.ty) + &WithTypespace::new(new_owning.typespace(), &new_col.ty) .resolve_refs() .expect("valid ViewDefs must have valid type refs"), ) .unwrap_or(Any(true)) } - } - }) - .collect(); + }) + .chain(new.param_columns.iter().map(|new_col| { + if old_param_cols.contains_key(&new_col.name) { + Any(false) + } else { + Any(true) + } + })) + .collect(); + + if old.is_public != new.is_public { + plan.steps.push(AutoMigrateStep::ChangeAccess(key)); + } if old.is_anonymous != new.is_anonymous || old.primary_key != new.primary_key || incompatible_return_type || incompatible_param_types { - plan.steps.push(AutoMigrateStep::AddView(new.key())); - plan.steps.push(AutoMigrateStep::RemoveView(old.key())); - + plan.steps.push(AutoMigrateStep::AddView(key)); + plan.steps.push(AutoMigrateStep::RemoveView(key)); plan.ensure_disconnect_all_users(); } else { - plan.steps.push(AutoMigrateStep::UpdateView(new.key())); + plan.steps.push(AutoMigrateStep::UpdateView(key)); } Ok(()) } -fn auto_migrate_tables(plan: &mut AutoMigratePlan<'_>) -> Result<()> { - diff(plan.old, plan.new, ModuleDef::tables) - .map(|table_diff| -> Result<()> { - match table_diff { - Diff::Add { new } => { - plan.steps.push(AutoMigrateStep::AddTable(new.key())); - Ok(()) - } - Diff::Remove { old } => { - plan.steps.push(AutoMigrateStep::RemoveTable(old.key())); - plan.ensure_disconnect_all_users(); - Ok(()) - } - Diff::MaybeChange { old, new } => auto_migrate_table(plan, old, new), - } +fn auto_migrate_tables<'def>(plan: &mut AutoMigratePlan<'def>) -> Result<()> { + let old_module = plan.old; + let new_module = plan.new; + + // Tables across all submodules, keyed by their namespace-qualified key. The key is the + // canonical name, which is what the database stores; the accessor name is only an alias. + // Matching by canonical name means a table whose accessor casing changed but whose + // canonical name did not is still the same logical table, so no spurious Remove+Add. + let old_tables: HashMap, (&'def ModuleDef, &'def TableDef)> = old_module + .all_tables_with_prefix() + .into_iter() + .map(|(_, owning, table)| (table.key(), (owning, table))) + .collect(); + let new_tables: HashMap, (&'def ModuleDef, &'def TableDef)> = new_module + .all_tables_with_prefix() + .into_iter() + .map(|(_, owning, table)| (table.key(), (owning, table))) + .collect(); + + for key in old_tables.keys() { + if !new_tables.contains_key(key) { + plan.steps.push(AutoMigrateStep::RemoveTable(*key)); + plan.ensure_disconnect_all_users(); + } + } + for key in new_tables.keys() { + if !old_tables.contains_key(key) { + plan.steps.push(AutoMigrateStep::AddTable(*key)); + } + } + + type TableEntry<'a> = ( + TableKey<'a>, + (&'a ModuleDef, &'a TableDef), + (&'a ModuleDef, &'a TableDef), + ); + let maybe_change: Vec> = old_tables + .iter() + .filter_map(|(key, (old_owning, old_table))| { + new_tables + .get(key) + .map(|(new_owning, new_table)| (*key, (*old_owning, *old_table), (*new_owning, *new_table))) + }) + .collect(); + + let results: Vec> = maybe_change + .into_iter() + .map(|(key, (old_owning, old_table), (new_owning, new_table))| { + auto_migrate_table(plan, key, old_owning, old_table, new_owning, new_table) }) - .collect_all_errors() + .collect(); + results.into_iter().collect_all_errors::>().map(|_| ()) } -fn auto_migrate_table<'def>(plan: &mut AutoMigratePlan<'def>, old: &'def TableDef, new: &'def TableDef) -> Result<()> { - let key = old.key(); +fn auto_migrate_table<'def>( + plan: &mut AutoMigratePlan<'def>, + key: TableKey<'def>, + old_owning: &ModuleDef, + old: &'def TableDef, + new_owning: &ModuleDef, + new: &'def TableDef, +) -> Result<()> { let type_ok: Result<()> = if old.table_type == new.table_type { Ok(()) } else { @@ -713,114 +766,104 @@ fn auto_migrate_table<'def>(plan: &mut AutoMigratePlan<'def>, old: &'def TableDe plan.steps.push(AutoMigrateStep::ChangePrimaryKey(key)); } if old.schedule != new.schedule { - // Note: this handles the case where there's an altered ScheduleDef for some reason. - if let Some(old_schedule) = old.schedule.as_ref() { + if let Some(old_schedule) = &old.schedule { plan.steps.push(AutoMigrateStep::RemoveSchedule(old_schedule.key())); } - if let Some(new_schedule) = new.schedule.as_ref() { + if let Some(new_schedule) = &new.schedule { plan.steps.push(AutoMigrateStep::AddSchedule(new_schedule.key())); } } - let columns_ok = diff(plan.old, plan.new, |def| { - def.lookup_expect::(key).columns.iter() - }) - .map(|col_diff| -> Result<_> { - match col_diff { - Diff::Add { new } => { - if is_event { - // Event tables never have any resident rows. - // As such, this is not a data migration; the table doesn't have any data in it to migrate. - // However, changing the schema of an event table will break clients. - - // `row_type_changed`, `columns_added`, `event_schema_changed` - Ok(ArrayMonoid([Any(false), Any(false), Any(true)])) - } else if new.default_value.is_some() { - // `row_type_changed`, `columns_added`, `event_schema_changed` - Ok(ArrayMonoid([Any(false), Any(true), Any(false)])) - } else { - Err(AutoMigrateError::AddColumn { - table: new.table_name.clone(), - column: new.name.clone(), - } - .into()) - } - } - Diff::Remove { old } => { - if is_event { - // Event tables never have any resident rows. - // As such, this is not a data migration; the table doesn't have any data in it to migrate. - // However, changing the schema of an event table will break clients. - - // `row_type_changed`, `columns_added`, `event_schema_changed` - Ok(ArrayMonoid([Any(false), Any(false), Any(true)])) - } else { - Err(AutoMigrateError::RemoveColumn { - table: old.table_name.clone(), - column: old.name.clone(), - } - .into()) - } - } - Diff::MaybeChange { old, new } => { - // Check column type upgradability. - let old_ty = WithTypespace::new(plan.old.typespace(), &old.ty) - .resolve_refs() - .expect("valid TableDef must have valid type refs"); - let new_ty = WithTypespace::new(plan.new.typespace(), &new.ty) - .resolve_refs() - .expect("valid TableDef must have valid type refs"); - let types_ok = ensure_old_ty_upgradable_to_new( - false, - &|| old.table_name.clone(), - &|| old.name.clone(), - &old_ty, - &new_ty, - ) - .or_else(|err| { + // Diff columns directly using the table defs (avoids root-only ModuleDefLookup). + let new_col_by_name: HashMap<&Identifier, &ColumnDef> = new.columns.iter().map(|c| (&c.name, c)).collect(); + let old_col_by_name: HashMap<&Identifier, &ColumnDef> = old.columns.iter().map(|c| (&c.name, c)).collect(); + + let columns_ok = old + .columns + .iter() + .map(|old_col| -> Result> { + match new_col_by_name.get(&old_col.name) { + None => { if is_event { - // If this is an event table, it's fine to layout-incompatibly non-upgradably change the layout, - // 'cause there can't be any rows to break. - Ok(Any(true)) + // Event tables never have any resident rows, so removing a column is not a + // data migration. However, changing the schema will break clients. + // `row_type_changed`, `columns_added`, `event_schema_changed` + Ok(ArrayMonoid([Any(false), Any(false), Any(true)])) } else { - Err(err) - } - }); - - // Note that the diff algorithm relies on `ModuleDefLookup` for `ColumnDef`, - // which looks up columns by NAME, NOT position: precisely to allow this step to work! - - // Note: We reject changes to positions. This means that, if a column was present in the old version of the table, - // it must be in the same place in the new version of the table. - // This guarantees that any added columns live at the end of the table. - let positions_ok = if old.col_id == new.col_id { - Ok(Any(false)) - } else if is_event { - Ok(Any(true)) - } else { - Err(AutoMigrateError::ReorderTable { - table: old.table_name.clone(), + Err(AutoMigrateError::RemoveColumn { + table: old_col.table_name.clone(), + column: old_col.name.clone(), + } + .into()) } - .into()) - }; - - (types_ok, positions_ok) - .combine_errors() - // `row_type_changed`, `column_added`, `event_schema_changed` - .map(|(types_changed, positions_changed)| { + } + Some(new_col) => { + let old_ty = WithTypespace::new(old_owning.typespace(), &old_col.ty) + .resolve_refs() + .expect("valid TableDef must have valid type refs"); + let new_ty = WithTypespace::new(new_owning.typespace(), &new_col.ty) + .resolve_refs() + .expect("valid TableDef must have valid type refs"); + let types_ok = ensure_old_ty_upgradable_to_new( + false, + &|| old_col.table_name.clone(), + &|| old_col.name.clone(), + &old_ty, + &new_ty, + ) + .or_else(|err| { if is_event { - // Event tables get a different auto-migrate step when their schema changes, - // as they don't have any rows to rewrite. So we track a different element in the array of change types. - ArrayMonoid([Any(false), Any(false), types_changed | positions_changed]) + // Event tables have no rows, so layout-incompatible type changes are fine. + Ok(Any(true)) } else { - assert!(!positions_changed.0); - ArrayMonoid([types_changed, Any(false), Any(false)]) + Err(err) + } + }); + // Reject reordering of existing columns (unless it's an event table). + let positions_ok = if old_col.col_id == new_col.col_id { + Ok(Any(false)) + } else if is_event { + Ok(Any(true)) + } else { + Err(AutoMigrateError::ReorderTable { + table: old_col.table_name.clone(), } - }) + .into()) + }; + (types_ok, positions_ok) + .combine_errors() + // `row_type_changed`, `columns_added`, `event_schema_changed` + .map(|(types_changed, positions_changed)| { + if is_event { + ArrayMonoid([Any(false), Any(false), types_changed | positions_changed]) + } else { + assert!(!positions_changed.0); + ArrayMonoid([types_changed, Any(false), Any(false)]) + } + }) + } } - } - }) - .collect_all_errors::>(); + }) + .chain(new.columns.iter().map(|new_col| -> Result> { + if old_col_by_name.contains_key(&new_col.name) { + Ok(ArrayMonoid([Any(false), Any(false), Any(false)])) + } else if is_event { + // Event tables never have any resident rows, so adding a column is not a data + // migration. However, changing the schema will break clients. + // `row_type_changed`, `columns_added`, `event_schema_changed` + Ok(ArrayMonoid([Any(false), Any(false), Any(true)])) + } else if new_col.default_value.is_some() { + // `row_type_changed`, `columns_added`, `event_schema_changed` + Ok(ArrayMonoid([Any(false), Any(true), Any(false)])) + } else { + Err(AutoMigrateError::AddColumn { + table: new_col.table_name.clone(), + column: new_col.name.clone(), + } + .into()) + } + })) + .collect_all_errors::>(); let ((), (), ArrayMonoid([Any(row_type_changed), Any(columns_added), Any(event_schema_changed)])) = (type_ok, event_ok, columns_ok).combine_errors()?; @@ -1029,141 +1072,195 @@ fn ensure_old_ty_upgradable_to_new( } } -fn auto_migrate_indexes( - plan: &mut AutoMigratePlan<'_>, - new_tables: &HashSet<&Identifier>, - removed_tables: &HashSet<&Identifier>, +fn auto_migrate_indexes<'def>( + plan: &mut AutoMigratePlan<'def>, + new_tables: &HashSet>, + removed_tables: &HashSet>, ) -> Result<()> { - diff(plan.old, plan.new, ModuleDef::indexes) - .map(|index_diff| -> Result<()> { - match index_diff { - Diff::Add { new } => { - if !new_tables.contains(&plan.new.stored_in_table_def(&new.name).unwrap().name) { - plan.steps.push(AutoMigrateStep::AddIndex(new.key())); - } - Ok(()) - } - Diff::Remove { old } => { - if !removed_tables.contains(&plan.old.stored_in_table_def(&old.name).unwrap().name) { - plan.steps.push(AutoMigrateStep::RemoveIndex(old.key())); - } - Ok(()) + let old_module = plan.old; + let new_module = plan.new; + + // Keyed by the index's namespace-qualified key; the value carries its table's key so + // sub-objects of added/removed tables can be skipped. + let index_map = |module: &'def ModuleDef| { + let mut map: HashMap, (TableKey<'def>, &'def IndexDef)> = HashMap::new(); + for (_, _, table) in module.all_tables_with_prefix() { + for index in table.indexes.values() { + map.insert(index.key(), (table.key(), index)); + } + } + map + }; + let old_indexes = index_map(old_module); + let new_indexes = index_map(new_module); + + // Removed indexes: in old but not in new, and not part of a removed table. + for (index_key, (table_key, _)) in &old_indexes { + if !new_indexes.contains_key(index_key) && !removed_tables.contains(table_key) { + plan.steps.push(AutoMigrateStep::RemoveIndex(*index_key)); + } + } + + // Added indexes: in new but not in old, and not part of a newly added table. + for (index_key, (table_key, _)) in &new_indexes { + if !old_indexes.contains_key(index_key) && !new_tables.contains(table_key) { + plan.steps.push(AutoMigrateStep::AddIndex(*index_key)); + } + } + + // Changed indexes: same name in both. + let change_results: Vec> = old_indexes + .iter() + .filter_map(|(index_key, (_, old_idx))| { + new_indexes + .get(index_key) + .map(|(_, new_idx)| (*index_key, old_idx, new_idx)) + }) + .map(|(index_key, old_idx, new_idx)| { + if old_idx.accessor_name != new_idx.accessor_name { + Err(AutoMigrateError::ChangeIndexAccessor { + index: old_idx.name.clone(), + old_accessor: old_idx.accessor_name.clone(), + new_accessor: new_idx.accessor_name.clone(), } - Diff::MaybeChange { old, new } => { - if old.accessor_name != new.accessor_name { - Err(AutoMigrateError::ChangeIndexAccessor { - index: old.name.clone(), - old_accessor: old.accessor_name.clone(), - new_accessor: new.accessor_name.clone(), - } - .into()) - } else { - if old.algorithm != new.algorithm { - plan.steps.push(AutoMigrateStep::RemoveIndex(old.key())); - plan.steps.push(AutoMigrateStep::AddIndex(old.key())); - } - Ok(()) - } + .into()) + } else { + if old_idx.algorithm != new_idx.algorithm { + plan.steps.push(AutoMigrateStep::RemoveIndex(index_key)); + plan.steps.push(AutoMigrateStep::AddIndex(index_key)); } + Ok(()) } }) - .collect_all_errors() + .collect(); + change_results.into_iter().collect_all_errors::>().map(|_| ()) } -fn auto_migrate_sequences( - plan: &mut AutoMigratePlan, - new_tables: &HashSet<&Identifier>, - removed_tables: &HashSet<&Identifier>, +fn auto_migrate_sequences<'def>( + plan: &mut AutoMigratePlan<'def>, + new_tables: &HashSet>, + removed_tables: &HashSet>, ) -> Result<()> { - diff(plan.old, plan.new, ModuleDef::sequences) - .map(|sequence_diff| -> Result<()> { - match sequence_diff { - Diff::Add { new } => { - if !new_tables.contains(&plan.new.stored_in_table_def(&new.name).unwrap().name) { - plan.prechecks - .push(AutoMigratePrecheck::CheckAddSequenceRangeValid(new.key())); - plan.steps.push(AutoMigrateStep::AddSequence(new.key())); - } - Ok(()) - } - Diff::Remove { old } => { - if !removed_tables.contains(&plan.old.stored_in_table_def(&old.name).unwrap().name) { - plan.steps.push(AutoMigrateStep::RemoveSequence(old.key())); - } - Ok(()) - } - Diff::MaybeChange { old, new } => { - // we do not need to check column ids, since in an automigrate, column ids are not changed. - if old != new { - plan.prechecks - .push(AutoMigratePrecheck::CheckAddSequenceRangeValid(new.key())); - plan.steps.push(AutoMigrateStep::RemoveSequence(old.key())); - plan.steps.push(AutoMigrateStep::AddSequence(new.key())); - } - Ok(()) - } + let old_module = plan.old; + let new_module = plan.new; + + // Keyed by the sequence's namespace-qualified key; the value carries its table's key so + // sub-objects of added/removed tables can be skipped. + let sequence_map = |module: &'def ModuleDef| { + let mut map: HashMap, (TableKey<'def>, &'def SequenceDef)> = HashMap::new(); + for (_, _, table) in module.all_tables_with_prefix() { + for sequence in table.sequences.values() { + map.insert(sequence.key(), (table.key(), sequence)); } - }) - .collect_all_errors() + } + map + }; + let old_seqs = sequence_map(old_module); + let new_seqs = sequence_map(new_module); + + // Removed sequences: in old but not in new, and not part of a removed table. + for (sequence_key, (table_key, _)) in &old_seqs { + if !new_seqs.contains_key(sequence_key) && !removed_tables.contains(table_key) { + plan.steps.push(AutoMigrateStep::RemoveSequence(*sequence_key)); + } + } + + // Added or changed sequences. + for (sequence_key, (table_key, new_seq)) in &new_seqs { + if let Some((_, old_seq)) = old_seqs.get(sequence_key) { + // we do not need to check column ids, since in an automigrate, column ids are not changed. + if *old_seq != *new_seq { + plan.prechecks + .push(AutoMigratePrecheck::CheckAddSequenceRangeValid(*sequence_key)); + plan.steps.push(AutoMigrateStep::RemoveSequence(*sequence_key)); + plan.steps.push(AutoMigrateStep::AddSequence(*sequence_key)); + } + } else if !new_tables.contains(table_key) { + plan.prechecks + .push(AutoMigratePrecheck::CheckAddSequenceRangeValid(*sequence_key)); + plan.steps.push(AutoMigrateStep::AddSequence(*sequence_key)); + } + } + + Ok(()) } -fn auto_migrate_constraints( - plan: &mut AutoMigratePlan, - new_tables: &HashSet<&Identifier>, - removed_tables: &HashSet<&Identifier>, +fn auto_migrate_constraints<'def>( + plan: &mut AutoMigratePlan<'def>, + new_tables: &HashSet>, + removed_tables: &HashSet>, ) -> Result<()> { - diff(plan.old, plan.new, ModuleDef::constraints) - .map(|constraint_diff| -> Result<()> { - match constraint_diff { - Diff::Add { new } => { - if new_tables.contains(&plan.new.stored_in_table_def(&new.name).unwrap().name) { - // it's okay to add a constraint in a new table. - Ok(()) - } else { - // existing table — duplicate detection happens inside create_constraint - plan.steps.push(AutoMigrateStep::AddConstraint(new.key())); - Ok(()) - } - } - Diff::Remove { old } => { - if !removed_tables.contains(&plan.old.stored_in_table_def(&old.name).unwrap().name) { - plan.steps.push(AutoMigrateStep::RemoveConstraint(old.key())); - } - Ok(()) - } - Diff::MaybeChange { old, new } => { - if old == new { - Ok(()) - } else { - Err(AutoMigrateError::ChangeUniqueConstraint { - constraint: old.name.clone(), - } - .into()) - } - } + let old_module = plan.old; + let new_module = plan.new; + + // Keyed by the constraint's namespace-qualified key; the value carries its table's key so + // sub-objects of added/removed tables can be skipped. + let constraint_map = |module: &'def ModuleDef| { + let mut map: HashMap, (TableKey<'def>, &'def ConstraintDef)> = HashMap::new(); + for (_, _, table) in module.all_tables_with_prefix() { + for constraint in table.constraints.values() { + map.insert(constraint.key(), (table.key(), constraint)); } - }) - .collect_all_errors() + } + map + }; + let old_constraints = constraint_map(old_module); + let new_constraints = constraint_map(new_module); + + let mut results: Vec> = vec![]; + + // Added constraints. + for (constraint_key, (table_key, _new_constraint)) in &new_constraints { + if !old_constraints.contains_key(constraint_key) && !new_tables.contains(table_key) { + // existing table — duplicate detection happens inside create_constraint + plan.steps.push(AutoMigrateStep::AddConstraint(*constraint_key)); + // it's okay to add a constraint in a new table — AddTable covers it. + } + } + + // Removed constraints: not part of a removed table. + for (constraint_key, (table_key, _)) in &old_constraints { + if !new_constraints.contains_key(constraint_key) && !removed_tables.contains(table_key) { + plan.steps.push(AutoMigrateStep::RemoveConstraint(*constraint_key)); + } + } + + // Changed constraints. + for (constraint_key, (_, new_constraint)) in &new_constraints { + if let Some((_, old_constraint)) = old_constraints.get(constraint_key) + && *old_constraint != *new_constraint + { + results.push(Err(AutoMigrateError::ChangeUniqueConstraint { + constraint: old_constraint.name.clone(), + } + .into())); + } + } + + results.into_iter().collect_all_errors::>().map(|_| ()) } // Because we can refer to many tables and fields on the row level-security query, we need to remove all of them, // then add the new ones, instead of trying to track the graph of dependencies. -fn auto_migrate_row_level_security(plan: &mut AutoMigratePlan) -> Result<()> { - // Track if any RLS rules were changed. - let mut old_rls = HashSet::new(); - let mut new_rls = HashSet::new(); +fn auto_migrate_row_level_security<'def>(plan: &mut AutoMigratePlan<'def>) -> Result<()> { + let old_sqls: Vec<&'def Box> = plan.old.row_level_security().map(|rls| &rls.sql).collect(); + let new_sqls: Vec<&'def Box> = plan.new.row_level_security().map(|rls| &rls.sql).collect(); + + let changed = { + let old_set: HashSet<&str> = old_sqls.iter().map(|s| &***s).collect(); + let new_set: HashSet<&str> = new_sqls.iter().map(|s| &***s).collect(); + old_set != new_set + }; - for rls in plan.old.row_level_security() { - old_rls.insert(rls.key()); - plan.steps.push(AutoMigrateStep::RemoveRowLevelSecurity(rls.key())); + for sql in old_sqls { + plan.steps.push(AutoMigrateStep::RemoveRowLevelSecurity(sql)); } - for rls in plan.new.row_level_security() { - new_rls.insert(rls.key()); - plan.steps.push(AutoMigrateStep::AddRowLevelSecurity(rls.key())); + for sql in new_sqls { + plan.steps.push(AutoMigrateStep::AddRowLevelSecurity(sql)); } // We can force flush the cache by force disconnecting all clients if an RLS rule has been added, removed, or updated. - if old_rls != new_rls { + if changed { plan.ensure_disconnect_all_users(); } @@ -1179,10 +1276,29 @@ mod tests { AlgebraicType, AlgebraicValue, ProductType, ScheduleAt, }; use spacetimedb_primitives::ColId; - use v10::{ExplicitNames, RawModuleDefV10Builder}; + use v10::{ExplicitNames, RawModuleDefV10Builder, RawModuleDefV10Section, RawSubmoduleV10}; use v9::{RawModuleDefV9Builder, TableAccess}; use validate::tests::expect_identifier; + /// Test helper: a namespace path, leaked so assertions can borrow it inline. + fn ns(path: &str) -> &'static NamespacePath { + let mut p = NamespacePath::root(); + for seg in path.split('.').filter(|s| !s.is_empty()) { + p = p.child(Identifier::for_test(seg)); + } + Box::leak(Box::new(p)) + } + + /// Test helper: a `(namespace, name)` key for a table, view or schedule. + fn key(namespace: &str, name: &str) -> (&'static NamespacePath, &'static Identifier) { + (ns(namespace), Box::leak(Box::new(Identifier::for_test(name)))) + } + + /// Test helper: a `(namespace, name)` key for an index, sequence or constraint. + fn sub_key(namespace: &str, name: &str) -> (&'static NamespacePath, &'static RawIdentifier) { + (ns(namespace), Box::leak(Box::new(RawIdentifier::new(name)))) + } + fn create_module_def(build_module: impl Fn(&mut RawModuleDefV9Builder)) -> ModuleDef { let mut builder = RawModuleDefV9Builder::new(); build_module(&mut builder); @@ -1410,95 +1526,90 @@ mod tests { let new_def = updated_module_def(); let plan = ponder_auto_migrate(&old_def, &new_def).expect("auto migration should succeed"); - let apples = expect_identifier("Apples"); - let bananas = expect_identifier("Bananas"); - let deliveries = expect_identifier("Deliveries"); - let oranges = expect_identifier("Oranges"); - let my_view = expect_identifier("my_view"); - - let bananas_sequence: RawIdentifier = "Bananas_id_seq".into(); - let apples_unique_constraint: RawIdentifier = "Apples_id_key".into(); - let apples_sequence: RawIdentifier = "Apples_id_seq".into(); - let apples_id_name_index: RawIdentifier = "Apples_id_name_idx_btree".into(); - let apples_id_count_index: RawIdentifier = "Apples_id_count_idx_btree".into(); - let deliveries_schedule = expect_identifier("Deliveries_sched"); - let inspections_schedule = expect_identifier("Inspections_sched"); + let apples = key("", "Apples"); + let bananas = key("", "Bananas"); + let deliveries = key("", "Deliveries"); + let oranges = key("", "Oranges"); + let my_view = key("", "my_view"); + + // Sub-objects are keyed by (namespace, local name). + let bananas_sequence = sub_key("", "Bananas_id_seq"); + let apples_unique_constraint = sub_key("", "Apples_id_key"); + let apples_sequence = sub_key("", "Apples_id_seq"); + let apples_id_name_index = sub_key("", "Apples_id_name_idx_btree"); + let apples_id_count_index = sub_key("", "Apples_id_count_idx_btree"); + let deliveries_schedule = key("", "Deliveries_sched"); + let inspections_schedule = key("", "Inspections_sched"); assert!(plan.prechecks.is_sorted()); assert_eq!(plan.prechecks.len(), 1); assert_eq!( plan.prechecks[0], - AutoMigratePrecheck::CheckAddSequenceRangeValid(&bananas_sequence) + AutoMigratePrecheck::CheckAddSequenceRangeValid(bananas_sequence) ); - let sql_old = RawRowLevelSecurityDefV9 { - sql: "SELECT * FROM Apples".into(), - }; - let sql_new = RawRowLevelSecurityDefV9 { - sql: "SELECT * FROM Bananas".into(), - }; + let sql_old: Box = "SELECT * FROM Apples".into(); + let sql_new: Box = "SELECT * FROM Bananas".into(); let steps = &plan.steps[..]; assert!(steps.is_sorted()); assert!( - steps.contains(&AutoMigrateStep::RemoveSequence(&apples_sequence)), + steps.contains(&AutoMigrateStep::RemoveSequence(apples_sequence)), "{steps:?}" ); assert!( - steps.contains(&AutoMigrateStep::RemoveConstraint(&apples_unique_constraint)), + steps.contains(&AutoMigrateStep::RemoveConstraint(apples_unique_constraint)), "{steps:?}" ); assert!( - steps.contains(&AutoMigrateStep::RemoveIndex(&apples_id_name_index)), + steps.contains(&AutoMigrateStep::RemoveIndex(apples_id_name_index)), "{steps:?}" ); assert!( - steps.contains(&AutoMigrateStep::AddIndex(&apples_id_count_index)), + steps.contains(&AutoMigrateStep::AddIndex(apples_id_count_index)), "{steps:?}" ); - assert!(steps.contains(&AutoMigrateStep::ChangeAccess(&bananas)), "{steps:?}"); + assert!(steps.contains(&AutoMigrateStep::ChangeAccess(bananas)), "{steps:?}"); assert!( - steps.contains(&AutoMigrateStep::AddSequence(&bananas_sequence)), + steps.contains(&AutoMigrateStep::AddSequence(bananas_sequence)), "{steps:?}" ); - assert!(steps.contains(&AutoMigrateStep::AddTable(&oranges)), "{steps:?}"); + assert!(steps.contains(&AutoMigrateStep::AddTable(oranges)), "{steps:?}"); + // Schedule steps are keyed by TABLE name (schedules are 1:1 with tables). assert!( - steps.contains(&AutoMigrateStep::RemoveSchedule(&deliveries_schedule)), + steps.contains(&AutoMigrateStep::RemoveSchedule(deliveries_schedule)), "{steps:?}" ); assert!( - steps.contains(&AutoMigrateStep::AddSchedule(&inspections_schedule)), + steps.contains(&AutoMigrateStep::AddSchedule(inspections_schedule)), "{steps:?}" ); assert!( - steps.contains(&AutoMigrateStep::RemoveRowLevelSecurity(&sql_old.sql)), + steps.contains(&AutoMigrateStep::RemoveRowLevelSecurity(&sql_old)), "{steps:?}" ); assert!( - steps.contains(&AutoMigrateStep::AddRowLevelSecurity(&sql_new.sql)), + steps.contains(&AutoMigrateStep::AddRowLevelSecurity(&sql_new)), "{steps:?}" ); - assert!(steps.contains(&AutoMigrateStep::ChangeColumns(&apples)), "{steps:?}"); - assert!( - steps.contains(&AutoMigrateStep::ChangeColumns(&deliveries)), - "{steps:?}" - ); + assert!(steps.contains(&AutoMigrateStep::ChangeColumns(apples)), "{steps:?}"); + assert!(steps.contains(&AutoMigrateStep::ChangeColumns(deliveries)), "{steps:?}"); assert!(steps.contains(&AutoMigrateStep::DisconnectAllUsers), "{steps:?}"); - assert!(steps.contains(&AutoMigrateStep::AddColumns(&bananas)), "{steps:?}"); + assert!(steps.contains(&AutoMigrateStep::AddColumns(bananas)), "{steps:?}"); // Column is changed but it will not reflect in steps due to `AutoMigrateStep::AddColumns` - assert!(!steps.contains(&AutoMigrateStep::ChangeColumns(&bananas)), "{steps:?}"); + assert!(!steps.contains(&AutoMigrateStep::ChangeColumns(bananas)), "{steps:?}"); - assert!(steps.contains(&AutoMigrateStep::RemoveView(&my_view)), "{steps:?}"); - assert!(steps.contains(&AutoMigrateStep::AddView(&my_view)), "{steps:?}"); + assert!(steps.contains(&AutoMigrateStep::RemoveView(my_view)), "{steps:?}"); + assert!(steps.contains(&AutoMigrateStep::AddView(my_view)), "{steps:?}"); } #[test] @@ -1920,14 +2031,52 @@ mod tests { ); }); - let my_view = expect_identifier("my_view"); + let plan = ponder_auto_migrate(&old_def, &new_def).expect("auto migration should succeed"); + let steps = &plan.steps[..]; + + assert!(!plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::AddView(key("", "my_view"))), + "{steps:?}" + ); + assert!( + !steps.contains(&AutoMigrateStep::RemoveView(key("", "my_view"))), + "{steps:?}" + ); + } + + #[test] + fn change_view_visibility_is_a_cheap_access_change() { + let view_module = |is_public: bool| { + create_module_def(move |builder| { + let return_type_ref = builder.add_algebraic_type( + [], + "my_view_return_type", + AlgebraicType::product([("a", AlgebraicType::U64)]), + true, + ); + builder.add_view( + "my_view", + 0, + is_public, + true, + ProductType::from([("x", AlgebraicType::U32)]), + AlgebraicType::array(AlgebraicType::Ref(return_type_ref)), + ); + }) + }; + + let old_def = view_module(false); + let new_def = view_module(true); let plan = ponder_auto_migrate(&old_def, &new_def).expect("auto migration should succeed"); let steps = &plan.steps[..]; + let my_view = key("", "my_view"); assert!(!plan.disconnects_all_users(), "{plan:#?}"); - assert!(steps.contains(&AutoMigrateStep::AddView(&my_view)), "{steps:?}"); - assert!(!steps.contains(&AutoMigrateStep::RemoveView(&my_view)), "{steps:?}"); + assert!(steps.contains(&AutoMigrateStep::ChangeAccess(my_view)), "{steps:?}"); + assert!(!steps.contains(&AutoMigrateStep::AddView(my_view)), "{steps:?}"); + assert!(!steps.contains(&AutoMigrateStep::RemoveView(my_view)), "{steps:?}"); } #[test] @@ -1950,14 +2099,18 @@ mod tests { }); let new_def = create_module_def(|_| {}); - let my_view = expect_identifier("my_view"); - let plan = ponder_auto_migrate(&old_def, &new_def).expect("auto migration should succeed"); let steps = &plan.steps[..]; assert!(plan.disconnects_all_users(), "{plan:#?}"); - assert!(steps.contains(&AutoMigrateStep::RemoveView(&my_view)), "{steps:?}"); - assert!(!steps.contains(&AutoMigrateStep::AddView(&my_view)), "{steps:?}"); + assert!( + steps.contains(&AutoMigrateStep::RemoveView(key("", "my_view"))), + "{steps:?}" + ); + assert!( + !steps.contains(&AutoMigrateStep::AddView(key("", "my_view"))), + "{steps:?}" + ); } #[test] @@ -2044,23 +2197,21 @@ mod tests { }), }, ] { - let my_view = expect_identifier("my_view"); - let plan = ponder_auto_migrate(&old_def, &new_def).expect("auto migration should succeed"); let steps = &plan.steps[..]; assert!(!plan.disconnects_all_users(), "{name}, plan: {plan:#?}"); assert!( - steps.contains(&AutoMigrateStep::UpdateView(&my_view)), + steps.contains(&AutoMigrateStep::UpdateView(key("", "my_view"))), "{name}, steps: {steps:?}" ); assert!( - !steps.contains(&AutoMigrateStep::AddView(&my_view)), + !steps.contains(&AutoMigrateStep::AddView(key("", "my_view"))), "{name}, steps: {steps:?}" ); assert!( - !steps.contains(&AutoMigrateStep::RemoveView(&my_view)), + !steps.contains(&AutoMigrateStep::RemoveView(key("", "my_view"))), "{name}, steps: {steps:?}" ); } @@ -2093,22 +2244,20 @@ mod tests { let old_def = module_def(); let new_def = module_def(); - let level_2_person = expect_identifier("Level2Person"); - let plan = ponder_auto_migrate(&old_def, &new_def).expect("auto migration should succeed"); let steps = &plan.steps[..]; assert!(!plan.disconnects_all_users(), "{plan:#?}"); assert!( - steps.contains(&AutoMigrateStep::UpdateView(&level_2_person)), + steps.contains(&AutoMigrateStep::UpdateView(key("", "Level2Person"))), "steps: {steps:?}" ); assert!( - !steps.contains(&AutoMigrateStep::AddView(&level_2_person)), + !steps.contains(&AutoMigrateStep::AddView(key("", "Level2Person"))), "steps: {steps:?}" ); assert!( - !steps.contains(&AutoMigrateStep::RemoveView(&level_2_person)), + !steps.contains(&AutoMigrateStep::RemoveView(key("", "Level2Person"))), "steps: {steps:?}" ); } @@ -2442,23 +2591,21 @@ mod tests { }), }, ] { - let my_view = expect_identifier("my_view"); - let plan = ponder_auto_migrate(&old_def, &new_def).expect("auto migration should succeed"); let steps = &plan.steps[..]; assert!(plan.disconnects_all_users(), "{name}, plan: {plan:?}"); assert!( - steps.contains(&AutoMigrateStep::AddView(&my_view)), + steps.contains(&AutoMigrateStep::AddView(key("", "my_view"))), "{name}, steps: {steps:?}" ); assert!( - steps.contains(&AutoMigrateStep::RemoveView(&my_view)), + steps.contains(&AutoMigrateStep::RemoveView(key("", "my_view"))), "{name}, steps: {steps:?}" ); assert!( - !steps.contains(&AutoMigrateStep::UpdateView(&my_view)), + !steps.contains(&AutoMigrateStep::UpdateView(key("", "my_view"))), "{name}, steps: {steps:?}" ); } @@ -2588,12 +2735,11 @@ mod tests { .finish(); }); - let drop_table = expect_identifier("Drop"); let plan = ponder_auto_migrate(&old, &new).expect("removing a table should produce a valid plan"); assert_eq!( plan.steps, &[ - AutoMigrateStep::RemoveTable(&drop_table), + AutoMigrateStep::RemoveTable(key("", "Drop")), AutoMigrateStep::DisconnectAllUsers, ], ); @@ -2611,15 +2757,408 @@ mod tests { }); let new = create_module_def(|_builder| {}); - let drop_table = expect_identifier("Drop"); let plan = ponder_auto_migrate(&old, &new).expect("removing a table should produce a valid plan"); assert_eq!( plan.steps, &[ - AutoMigrateStep::RemoveTable(&drop_table), + AutoMigrateStep::RemoveTable(key("", "Drop")), AutoMigrateStep::DisconnectAllUsers, ], "plan should only contain RemoveTable + DisconnectAllUsers, no orphan sub-object steps" ); } + + fn make_submodule(namespace: &str, build: impl Fn(&mut RawModuleDefV10Builder)) -> RawSubmoduleV10 { + let mut builder = RawModuleDefV10Builder::new(); + build(&mut builder); + RawSubmoduleV10 { + namespace: namespace.to_string(), + module: builder.finish(), + } + } + + fn create_module_def_with_submodules( + build_root: impl Fn(&mut RawModuleDefV10Builder), + submodules: Vec, + ) -> ModuleDef { + let mut builder = RawModuleDefV10Builder::new(); + build_root(&mut builder); + let mut raw = builder.finish(); + if !submodules.is_empty() { + raw.sections.push(RawModuleDefV10Section::Submodules(submodules)); + } + raw.try_into().expect("should be a valid module definition") + } + + #[test] + fn submodule_table_unchanged() { + let submodule = || { + make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + }) + }; + let old = create_module_def_with_submodules(|_| {}, vec![submodule()]); + let new = create_module_def_with_submodules(|_| {}, vec![submodule()]); + + let plan = ponder_auto_migrate(&old, &new).expect("no-op migration should succeed"); + let namespaced: Vec<_> = plan + .steps + .iter() + .filter(|s| format!("{s:?}").contains("lib.sessions")) + .collect(); + assert!( + namespaced.is_empty(), + "unchanged submodule should produce no steps for lib.sessions: {plan:#?}" + ); + } + + #[test] + fn submodule_add_table() { + let old = create_module_def_with_submodules( + |_| {}, + vec![make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + })], + ); + let new = create_module_def_with_submodules( + |_| {}, + vec![make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + b.build_table_with_new_type("tokens", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + })], + ); + + let plan = ponder_auto_migrate(&old, &new).expect("adding a submodule table should succeed"); + let steps = &plan.steps[..]; + + assert!(!plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::AddTable(key("lib", "tokens"))), + "{steps:?}" + ); + } + + #[test] + fn submodule_remove_table() { + let old = create_module_def_with_submodules( + |_| {}, + vec![make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + b.build_table_with_new_type("tokens", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + })], + ); + let new = create_module_def_with_submodules( + |_| {}, + vec![make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + })], + ); + + let plan = ponder_auto_migrate(&old, &new).expect("removing a submodule table should succeed"); + let steps = &plan.steps[..]; + + assert!(plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::RemoveTable(key("lib", "tokens"))), + "{steps:?}" + ); + } + + #[test] + fn submodule_add_index() { + let sessions_without_index = || { + make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + }) + }; + let sessions_with_index = || { + make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .with_index(btree(0), "sessions_id_idx", "sessions_id_idx") + .finish(); + }) + }; + + let old = create_module_def_with_submodules(|_| {}, vec![sessions_without_index()]); + let new = create_module_def_with_submodules(|_| {}, vec![sessions_with_index()]); + + let plan = ponder_auto_migrate(&old, &new).expect("adding a submodule index should succeed"); + let steps = &plan.steps[..]; + + assert!( + steps.contains(&AutoMigrateStep::AddIndex(sub_key("lib", "sessions_id_idx_btree"))), + "{steps:?}" + ); + } + + #[test] + fn submodule_remove_index() { + let sessions_without_index = || { + make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + }) + }; + let sessions_with_index = || { + make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .with_index(btree(0), "sessions_id_idx", "sessions_id_idx") + .finish(); + }) + }; + + let old = create_module_def_with_submodules(|_| {}, vec![sessions_with_index()]); + let new = create_module_def_with_submodules(|_| {}, vec![sessions_without_index()]); + + let plan = ponder_auto_migrate(&old, &new).expect("removing a submodule index should succeed"); + let steps = &plan.steps[..]; + + assert!( + steps.contains(&AutoMigrateStep::RemoveIndex(sub_key("lib", "sessions_id_idx_btree"))), + "{steps:?}" + ); + } + + #[test] + fn submodule_add_sequence() { + let without_seq = || { + make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + }) + }; + let with_seq = || { + make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .with_column_sequence(0) + .finish(); + }) + }; + + let old = create_module_def_with_submodules(|_| {}, vec![without_seq()]); + let new = create_module_def_with_submodules(|_| {}, vec![with_seq()]); + + let plan = ponder_auto_migrate(&old, &new).expect("adding a submodule sequence should succeed"); + let steps = &plan.steps[..]; + + assert!( + steps.iter().any(|s| matches!( + s, + AutoMigrateStep::AddSequence((namespace, _)) if *namespace == ns("lib") + )), + "expected AddSequence for lib.sessions_*: {steps:?}" + ); + assert!( + plan.prechecks.iter().any(|p| matches!( + p, + AutoMigratePrecheck::CheckAddSequenceRangeValid((namespace, _)) if *namespace == ns("lib") + )), + "expected CheckAddSequenceRangeValid precheck: {:?}", + plan.prechecks + ); + } + + #[test] + fn submodule_add_view() { + let without_view = || make_submodule("lib", |_| {}); + let with_view = || { + make_submodule("lib", |b| { + let ret_ref = b.add_algebraic_type( + [], + "lib_view_return", + AlgebraicType::product([("a", AlgebraicType::U64)]), + true, + ); + b.add_view( + "lib_view", + 0, + true, + true, + ProductType::from([("x", AlgebraicType::U32)]), + AlgebraicType::array(AlgebraicType::Ref(ret_ref)), + ); + }) + }; + + let old = create_module_def_with_submodules(|_| {}, vec![without_view()]); + let new = create_module_def_with_submodules(|_| {}, vec![with_view()]); + + let plan = ponder_auto_migrate(&old, &new).expect("adding a submodule view should succeed"); + let steps = &plan.steps[..]; + + assert!(!plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::AddView(key("lib", "lib_view"))), + "{steps:?}" + ); + } + + #[test] + fn submodule_remove_view() { + let without_view = || make_submodule("lib", |_| {}); + let with_view = || { + make_submodule("lib", |b| { + let ret_ref = b.add_algebraic_type( + [], + "lib_view_return", + AlgebraicType::product([("a", AlgebraicType::U64)]), + true, + ); + b.add_view( + "lib_view", + 0, + true, + true, + ProductType::from([("x", AlgebraicType::U32)]), + AlgebraicType::array(AlgebraicType::Ref(ret_ref)), + ); + }) + }; + + let old = create_module_def_with_submodules(|_| {}, vec![with_view()]); + let new = create_module_def_with_submodules(|_| {}, vec![without_view()]); + + let plan = ponder_auto_migrate(&old, &new).expect("removing a submodule view should succeed"); + let steps = &plan.steps[..]; + + assert!(plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::RemoveView(key("lib", "lib_view"))), + "{steps:?}" + ); + } + + #[test] + fn add_whole_submodule() { + let old = create_module_def_with_submodules(|_| {}, vec![]); + let new = create_module_def_with_submodules( + |_| {}, + vec![make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + b.build_table_with_new_type("tokens", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + })], + ); + + let plan = ponder_auto_migrate(&old, &new).expect("adding a whole submodule should succeed"); + let steps = &plan.steps[..]; + + assert!(!plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::AddTable(key("lib", "sessions"))), + "{steps:?}" + ); + assert!( + steps.contains(&AutoMigrateStep::AddTable(key("lib", "tokens"))), + "{steps:?}" + ); + } + + #[test] + fn remove_whole_submodule() { + let old = create_module_def_with_submodules( + |_| {}, + vec![make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + b.build_table_with_new_type("tokens", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + })], + ); + let new = create_module_def_with_submodules(|_| {}, vec![]); + + let plan = ponder_auto_migrate(&old, &new).expect("removing a whole submodule should succeed"); + let steps = &plan.steps[..]; + + assert!(plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::RemoveTable(key("lib", "sessions"))), + "{steps:?}" + ); + assert!( + steps.contains(&AutoMigrateStep::RemoveTable(key("lib", "tokens"))), + "{steps:?}" + ); + } + + #[test] + fn nested_submodule_add_table() { + let make_nested_def = |add_baz_items: bool| { + let baz_submodule = make_submodule("baz", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + if add_baz_items { + b.build_table_with_new_type("baz_items", ProductType::from([("id", AlgebraicType::U32)]), true) + .finish(); + } + }); + + let mut auth_builder = RawModuleDefV10Builder::new(); + auth_builder + .build_table_with_new_type("auth_users", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + let mut auth_raw = auth_builder.finish(); + auth_raw + .sections + .push(RawModuleDefV10Section::Submodules(vec![baz_submodule])); + + let auth_submodule = RawSubmoduleV10 { + namespace: "auth".to_string(), + module: auth_raw, + }; + + let root_builder = RawModuleDefV10Builder::new(); + let mut root_raw = root_builder.finish(); + root_raw + .sections + .push(RawModuleDefV10Section::Submodules(vec![auth_submodule])); + root_raw.try_into().expect("should be a valid module definition") + }; + + let old: ModuleDef = make_nested_def(false); + let new: ModuleDef = make_nested_def(true); + + let plan = ponder_auto_migrate(&old, &new).expect("adding a deeply nested table should succeed"); + let steps = &plan.steps[..]; + + assert!(!plan.disconnects_all_users(), "{plan:#?}"); + assert!( + steps.contains(&AutoMigrateStep::AddTable(key("auth.baz", "baz_items"))), + "{steps:?}" + ); + } + + #[test] + fn submodule_remove_table_no_orphan_sub_objects() { + let old = create_module_def_with_submodules( + |_| {}, + vec![make_submodule("lib", |b| { + b.build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .with_primary_key(0) + .with_unique_constraint(0) + .with_index(btree(0), "sessions_id_idx", "sessions_id_idx") + .finish(); + })], + ); + let new = create_module_def_with_submodules(|_| {}, vec![make_submodule("lib", |_| {})]); + + let plan = ponder_auto_migrate(&old, &new).expect("removing a submodule table with sub-objects should succeed"); + assert_eq!( + plan.steps, + &[ + AutoMigrateStep::RemoveTable(key("lib", "sessions")), + AutoMigrateStep::DisconnectAllUsers, + ], + "should only contain RemoveTable + DisconnectAllUsers, no orphan sub-object steps" + ); + } } diff --git a/crates/schema/src/auto_migrate/formatter.rs b/crates/schema/src/auto_migrate/formatter.rs index cd6c1042077..ede9d9e44fa 100644 --- a/crates/schema/src/auto_migrate/formatter.rs +++ b/crates/schema/src/auto_migrate/formatter.rs @@ -2,11 +2,14 @@ use std::io; -use super::{AutoMigratePlan, ModuleDefLookup, TableDef}; +use super::AutoMigratePlan; use crate::{ auto_migrate::AutoMigrateStep, - def::{ConstraintData, FunctionKind, ModuleDef, ScheduleDef, ViewDef}, - identifier::Identifier, + def::{ + ConstraintData, ConstraintDef, FunctionKind, IndexDef, ModuleDef, ModuleDefLookup, ScheduleDef, SequenceDef, + TableDef, ViewDef, + }, + identifier::{Identifier, NamespacePath, NamespacedIdentifier}, }; use itertools::Itertools; use spacetimedb_lib::db::raw_def::v9::{RawRowLevelSecurityDefV9, TableAccess, TableType}; @@ -25,6 +28,13 @@ pub fn format_plan(f: &mut F, plan: &AutoMigratePlan) -> Ok(()) } +/// The joined, qualified form of a `(namespace, name)` key. +/// +/// Built on demand: this is display output, so the allocation is incidental. +fn joined>((namespace, name): (&NamespacePath, &N)) -> NamespacedIdentifier { + namespace.join(name.clone().into()) +} + fn format_step( f: &mut F, step: &AutoMigrateStep, @@ -43,10 +53,7 @@ fn format_step( // So we must recompute it and send any updates to clients. // No need to include this step in the formatted plan. AutoMigrateStep::UpdateView(_) => Ok(()), - AutoMigrateStep::RemoveTable(t) => { - let table_def: &TableDef = plan.old.expect_lookup(*t); - f.format_remove_table(&table_def.name) - } + AutoMigrateStep::RemoveTable(t) => f.format_remove_table(&joined(*t)), AutoMigrateStep::AddTable(t) => { let table_info = extract_table_info(*t, plan)?; f.format_add_table(&table_info) @@ -80,9 +87,11 @@ fn format_step( f.format_change_access(&access_info) } AutoMigrateStep::ChangePrimaryKey(table) => { - let old_table = plan.old.lookup_expect::(*table); - let new_table = plan.new.lookup_expect::(*table); - f.format_change_primary_key(table, old_table.primary_key, new_table.primary_key) + let name = joined(*table); + let not_found = || FormattingErrors::TableNotFound { table: (&*name).into() }; + let (_, old_table) = plan.old.find_table(*table).ok_or_else(not_found)?; + let (_, new_table) = plan.new.find_table(*table).ok_or_else(not_found)?; + f.format_change_primary_key(&name, old_table.primary_key, new_table.primary_key) } AutoMigrateStep::AddSchedule(schedule) => { let schedule_info = extract_schedule_info(*schedule, plan.new)?; @@ -93,13 +102,13 @@ fn format_step( f.format_schedule(&schedule_info, Action::Removed) } AutoMigrateStep::AddRowLevelSecurity(rls) => { - if let Some(rls_info) = extract_rls_info(*rls, plan)? { + if let Some(rls_info) = extract_rls_info(rls, plan)? { f.format_rls(&rls_info, Action::Created)?; } Ok(()) } AutoMigrateStep::RemoveRowLevelSecurity(rls) => { - if let Some(rls_info) = extract_rls_info(*rls, plan)? { + if let Some(rls_info) = extract_rls_info(rls, plan)? { f.format_rls(&rls_info, Action::Removed)?; } Ok(()) @@ -119,7 +128,7 @@ fn format_step( // TODO(format-event-table-reschema): I (pgoldman 2026-06-10) didn't have time to meaningfully format event table reschemas, // so for now we're just printing the table name. - AutoMigrateStep::ReschemaEventTable(table) => f.format_event_table_reschema(table), + AutoMigrateStep::ReschemaEventTable(table) => f.format_event_table_reschema(&joined(*table)), }?; Ok(()) @@ -161,7 +170,7 @@ pub enum Action { pub trait MigrationFormatter { fn format_header(&mut self) -> io::Result<()>; fn format_add_table(&mut self, table_info: &TableInfo) -> io::Result<()>; - fn format_remove_table(&mut self, table_name: &Identifier) -> io::Result<()>; + fn format_remove_table(&mut self, table_name: &NamespacedIdentifier) -> io::Result<()>; fn format_view(&mut self, view_info: &ViewInfo, action: Action) -> io::Result<()>; fn format_index(&mut self, index_info: &IndexInfo, action: Action) -> io::Result<()>; fn format_constraint(&mut self, constraint_info: &ConstraintInfo, action: Action) -> io::Result<()>; @@ -169,7 +178,7 @@ pub trait MigrationFormatter { fn format_change_access(&mut self, access_info: &AccessChangeInfo) -> io::Result<()>; fn format_change_primary_key( &mut self, - table_name: &str, + table_name: &NamespacedIdentifier, old_pk: Option, new_pk: Option, ) -> io::Result<()>; @@ -180,12 +189,12 @@ pub trait MigrationFormatter { fn format_disconnect_warning(&mut self) -> io::Result<()>; // TODO(format-event-table-reschema): I (pgoldman 2026-06-10) didn't have time to meaningfully format event table reschemas, // so for now we're just printing the table name. - fn format_event_table_reschema(&mut self, table_name: &Identifier) -> io::Result<()>; + fn format_event_table_reschema(&mut self, table_name: &NamespacedIdentifier) -> io::Result<()>; } #[derive(Debug, Clone, PartialEq)] pub struct TableInfo { - pub name: RawIdentifier, + pub name: NamespacedIdentifier, pub is_system: bool, pub access: TableAccess, pub columns: Vec, @@ -197,7 +206,7 @@ pub struct TableInfo { #[derive(Debug, Clone, PartialEq)] pub struct ViewInfo { - pub name: RawIdentifier, + pub name: NamespacedIdentifier, pub params: Vec, pub columns: Vec, pub is_anonymous: bool, @@ -224,34 +233,40 @@ pub struct ColumnInfo { #[derive(Debug, Clone, PartialEq)] pub struct ConstraintInfo { + /// The constraint's own name. Table-relative (not namespaced) when reported as + /// part of a [`TableInfo`]; the full namespaced name otherwise. pub name: RawIdentifier, pub columns: Vec, - pub table_name: Identifier, + pub table_name: NamespacedIdentifier, } #[derive(Debug, Clone, PartialEq)] pub struct IndexInfo { + /// The index's own name. Table-relative (not namespaced) when reported as + /// part of a [`TableInfo`]; the full namespaced name otherwise. pub name: RawIdentifier, pub columns: Vec, - pub table_name: RawIdentifier, + pub table_name: NamespacedIdentifier, } #[derive(Debug, Clone, PartialEq)] pub struct SequenceInfo { + /// The sequence's own name. Table-relative (not namespaced) when reported as + /// part of a [`TableInfo`]; the full namespaced name otherwise. pub name: RawIdentifier, pub column_name: Identifier, - pub table_name: Identifier, + pub table_name: NamespacedIdentifier, } #[derive(Debug, Clone, PartialEq)] pub struct AccessChangeInfo { - pub table_name: Identifier, + pub table_name: NamespacedIdentifier, pub new_access: TableAccess, } #[derive(Debug, Clone, PartialEq)] pub struct ScheduleInfo { - pub table_name: Identifier, + pub table_name: NamespacedIdentifier, pub function_name: Identifier, pub function_kind: FunctionKind, } @@ -263,7 +278,7 @@ pub struct RlsInfo { #[derive(Debug, Clone, PartialEq)] pub struct ColumnChanges { - pub table_name: Identifier, + pub table_name: NamespacedIdentifier, pub changes: Vec, } @@ -282,24 +297,26 @@ pub enum ColumnChange { #[derive(Debug, Clone, PartialEq)] pub struct NewColumns { - pub table_name: Identifier, + pub table_name: NamespacedIdentifier, pub columns: Vec, } -// Data extraction functions (these replace the original print functions' data gathering logic) fn extract_table_info( - table: ::Key<'_>, + key: ::Key<'_>, plan: &super::AutoMigratePlan, ) -> Result { - let table_def = plan.new.table(table).ok_or_else(|| FormattingErrors::TableNotFound { - table: table.to_string().into(), - })?; + let (owning_def, table_def) = plan + .new + .find_table(key) + .ok_or_else(|| FormattingErrors::TableNotFound { + table: (&*joined(key)).into(), + })?; let columns = table_def .columns .iter() .map(|column| { - let type_name = WithTypespace::new(plan.new.typespace(), &column.ty) + let type_name = WithTypespace::new(owning_def.typespace(), &column.ty) .resolve_refs() .map_err(|_| FormattingErrors::TypeResolution)?; Ok(ColumnInfo { @@ -326,7 +343,7 @@ fn extract_table_info( Ok(column.name.clone()) }) .collect::, FormattingErrors>>()?, - table_name: table_def.name.clone(), + table_name: joined(key), }) }) .collect::, FormattingErrors>>()?; @@ -349,7 +366,7 @@ fn extract_table_info( Ok(IndexInfo { name: index.name.clone(), columns, - table_name: table_def.name.clone().into(), + table_name: joined(key), }) }) .collect::, FormattingErrors>>()?; @@ -365,19 +382,19 @@ fn extract_table_info( Ok(SequenceInfo { name: sequence.name.clone(), column_name: column.name.clone(), - table_name: table_def.name.clone(), + table_name: joined(key), }) }) .collect::, FormattingErrors>>()?; let schedule = table_def.schedule.as_ref().map(|schedule| ScheduleInfo { - table_name: table_def.name.clone(), + table_name: joined(key), function_name: schedule.function_name.clone(), function_kind: schedule.function_kind, }); Ok(TableInfo { - name: table_def.name.clone().into(), + name: joined(key), is_system: table_def.table_type == TableType::System, access: table_def.table_access, columns, @@ -389,21 +406,23 @@ fn extract_table_info( } fn extract_view_info( - view: ::Key<'_>, + key: ::Key<'_>, module_def: &ModuleDef, ) -> Result { - let view_def = module_def.view(view).ok_or_else(|| FormattingErrors::ViewNotFound { - view: view.to_string().into(), - })?; + let (owning_def, view_def) = module_def + .find_view(key) + .ok_or_else(|| FormattingErrors::ViewNotFound { + view: (&*joined(key)).into(), + })?; - let name = view_def.name.clone().into(); + let name = joined(key); let is_anonymous = view_def.is_anonymous; let params = view_def .param_columns .iter() .map(|column| { - let type_name = WithTypespace::new(module_def.typespace(), &column.ty) + let type_name = WithTypespace::new(owning_def.typespace(), &column.ty) .resolve_refs() .map_err(|_| FormattingErrors::TypeResolution)?; Ok(ViewParamInfo { @@ -417,7 +436,7 @@ fn extract_view_info( .return_columns .iter() .map(|column| { - let type_name = WithTypespace::new(module_def.typespace(), &column.ty) + let type_name = WithTypespace::new(owning_def.typespace(), &column.ty) .resolve_refs() .map_err(|_| FormattingErrors::TypeResolution)?; Ok(ViewColumnInfo { @@ -436,13 +455,14 @@ fn extract_view_info( } fn extract_index_info( - index: ::Key<'_>, + key: ::Key<'_>, module_def: &ModuleDef, ) -> Result { - let table_def = module_def - .stored_in_table_def(index) + let (namespace, name) = key; + let (_, table_def) = module_def + .find_storing_table(namespace, name) .ok_or(FormattingErrors::IndexNotFound)?; - let index_def = table_def.indexes.get(index).ok_or(FormattingErrors::IndexNotFound)?; + let index_def: &IndexDef = module_def.lookup(key).ok_or(FormattingErrors::IndexNotFound)?; let columns = index_def .algorithm @@ -457,21 +477,19 @@ fn extract_index_info( Ok(IndexInfo { name: index_def.name.clone(), columns, - table_name: table_def.name.clone().into(), + table_name: namespace.join(table_def.name.clone()), }) } fn extract_constraint_info( - constraint: ::Key<'_>, + key: ::Key<'_>, module_def: &ModuleDef, ) -> Result { - let table_def = module_def - .stored_in_table_def(constraint) - .ok_or(FormattingErrors::ConstraintNotFound)?; - let constraint_def = table_def - .constraints - .get(constraint) + let (namespace, name) = key; + let (_, table_def) = module_def + .find_storing_table(namespace, name) .ok_or(FormattingErrors::ConstraintNotFound)?; + let constraint_def: &ConstraintDef = module_def.lookup(key).ok_or(FormattingErrors::ConstraintNotFound)?; let ConstraintData::Unique(unique_constraint_data) = &constraint_def.data; let columns = unique_constraint_data @@ -486,21 +504,19 @@ fn extract_constraint_info( Ok(ConstraintInfo { name: constraint_def.name.clone(), columns, - table_name: table_def.name.clone(), + table_name: namespace.join(table_def.name.clone()), }) } fn extract_sequence_info( - sequence: ::Key<'_>, + key: ::Key<'_>, module_def: &ModuleDef, ) -> Result { - let table_def = module_def - .stored_in_table_def(sequence) - .ok_or(FormattingErrors::SequenceNotFound)?; - let sequence_def = table_def - .sequences - .get(sequence) + let (namespace, name) = key; + let (_, table_def) = module_def + .find_storing_table(namespace, name) .ok_or(FormattingErrors::SequenceNotFound)?; + let sequence_def: &SequenceDef = module_def.lookup(key).ok_or(FormattingErrors::SequenceNotFound)?; let column = table_def .get_column(sequence_def.column) @@ -509,34 +525,42 @@ fn extract_sequence_info( Ok(SequenceInfo { name: sequence_def.name.clone(), column_name: column.name.clone(), - table_name: table_def.name.clone(), + table_name: namespace.join(table_def.name.clone()), }) } fn extract_access_change_info( - table: ::Key<'_>, + key: ::Key<'_>, plan: &super::AutoMigratePlan, ) -> Result { - let table_def = plan.new.table(table).ok_or_else(|| FormattingErrors::TableNotFound { - table: table.to_string().into(), - })?; + let (_, table_def) = plan + .new + .find_table(key) + .ok_or_else(|| FormattingErrors::TableNotFound { + table: (&*joined(key)).into(), + })?; Ok(AccessChangeInfo { - table_name: table_def.name.clone(), + table_name: joined(key), new_access: table_def.table_access, }) } fn extract_schedule_info( - schedule_table: ::Key<'_>, + schedule_name: ::Key<'_>, module_def: &ModuleDef, ) -> Result { + // Schedules are keyed by their own name; the table they annotate is the one that stores them. + let (namespace, name) = schedule_name; let schedule_def: &ScheduleDef = module_def - .lookup(schedule_table) + .lookup(schedule_name) + .ok_or(FormattingErrors::ScheduleNotFound)?; + let (_, table_def) = module_def + .find_storing_table(namespace, name.as_raw()) .ok_or(FormattingErrors::ScheduleNotFound)?; Ok(ScheduleInfo { - table_name: schedule_def.name.clone(), + table_name: namespace.join(table_def.name.clone()), function_name: schedule_def.function_name.clone(), function_kind: schedule_def.function_kind, }) @@ -546,8 +570,10 @@ fn extract_rls_info( rls: ::Key<'_>, plan: &super::AutoMigratePlan, ) -> Result, FormattingErrors> { - // Skip if policy unchanged (implementation detail workaround) - if plan.old.lookup::(rls) == plan.new.lookup::(rls) { + let in_old = plan.old.row_level_security().any(|r| r.sql == *rls); + let in_new = plan.new.row_level_security().any(|r| r.sql == *rls); + // Skip if policy is unchanged (present in both old and new). + if in_old == in_new { return Ok(None); } @@ -557,15 +583,21 @@ fn extract_rls_info( } fn extract_column_changes( - table: ::Key<'_>, + key: ::Key<'_>, plan: &super::AutoMigratePlan, ) -> Result { - let old_table = plan.old.table(table).ok_or_else(|| FormattingErrors::TableNotFound { - table: table.to_string().into(), - })?; - let new_table = plan.new.table(table).ok_or_else(|| FormattingErrors::TableNotFound { - table: table.to_string().into(), - })?; + let (old_owning, old_table) = plan + .old + .find_table(key) + .ok_or_else(|| FormattingErrors::TableNotFound { + table: (&*joined(key)).into(), + })?; + let (new_owning, new_table) = plan + .new + .find_table(key) + .ok_or_else(|| FormattingErrors::TableNotFound { + table: (&*joined(key)).into(), + })?; let mut changes = Vec::new(); @@ -579,10 +611,10 @@ fn extract_column_changes( }); } if old_col.ty != new_col.ty { - let old_type = WithTypespace::new(plan.old.typespace(), &old_col.ty) + let old_type = WithTypespace::new(old_owning.typespace(), &old_col.ty) .resolve_refs() .map_err(|_| FormattingErrors::TypeResolution)?; - let new_type = WithTypespace::new(plan.new.typespace(), &new_col.ty) + let new_type = WithTypespace::new(new_owning.typespace(), &new_col.ty) .resolve_refs() .map_err(|_| FormattingErrors::TypeResolution)?; changes.push(ColumnChange::TypeChanged { @@ -595,26 +627,32 @@ fn extract_column_changes( } Ok(ColumnChanges { - table_name: new_table.name.clone(), + table_name: joined(key), changes, }) } fn extract_new_columns( - table: ::Key<'_>, + key: ::Key<'_>, plan: &super::AutoMigratePlan, ) -> Result { - let table_def = plan.new.table(table).ok_or_else(|| FormattingErrors::TableNotFound { - table: table.to_string().into(), - })?; - let old_table_def = plan.old.table(table).ok_or_else(|| FormattingErrors::TableNotFound { - table: table.to_string().into(), - })?; + let (new_owning, table_def) = plan + .new + .find_table(key) + .ok_or_else(|| FormattingErrors::TableNotFound { + table: (&*joined(key)).into(), + })?; + let (_, old_table_def) = plan + .old + .find_table(key) + .ok_or_else(|| FormattingErrors::TableNotFound { + table: (&*joined(key)).into(), + })?; let mut new_columns = Vec::new(); for column in &table_def.columns { if !old_table_def.columns.iter().any(|c| c.col_id == column.col_id) { - let type_name = WithTypespace::new(plan.new.typespace(), &column.ty) + let type_name = WithTypespace::new(new_owning.typespace(), &column.ty) .resolve_refs() .map_err(|_| FormattingErrors::TypeResolution)?; new_columns.push(ColumnInfo { @@ -626,7 +664,7 @@ fn extract_new_columns( } Ok(NewColumns { - table_name: table_def.name.clone(), + table_name: joined(key), columns: new_columns, }) } diff --git a/crates/schema/src/auto_migrate/termcolor_formatter.rs b/crates/schema/src/auto_migrate/termcolor_formatter.rs index 3f9764cfabf..f19d19ec7ad 100644 --- a/crates/schema/src/auto_migrate/termcolor_formatter.rs +++ b/crates/schema/src/auto_migrate/termcolor_formatter.rs @@ -7,12 +7,12 @@ use spacetimedb_sats::algebraic_type::fmt::fmt_algebraic_type; use termcolor::{Buffer, Color, ColorChoice, ColorSpec, WriteColor}; use crate::auto_migrate::formatter::ViewInfo; -use crate::identifier::Identifier; use super::formatter::{ AccessChangeInfo, Action, ColumnChange, ColumnChanges, ConstraintInfo, IndexInfo, MigrationFormatter, NewColumns, RlsInfo, ScheduleInfo, SequenceInfo, TableInfo, }; +use crate::identifier::NamespacedIdentifier; /// Color scheme for consistent formatting #[derive(Debug, Clone)] @@ -231,7 +231,7 @@ impl MigrationFormatter for TermColorFormatter { self.write_line("") } - fn format_remove_table(&mut self, table_name: &Identifier) -> io::Result<()> { + fn format_remove_table(&mut self, table_name: &NamespacedIdentifier) -> io::Result<()> { self.write_action_prefix(&Action::Removed)?; self.buffer.write_all(b" table: ")?; self.write_colored(table_name, Some(self.colors.table_name), true)?; @@ -327,7 +327,7 @@ impl MigrationFormatter for TermColorFormatter { fn format_change_primary_key( &mut self, - table_name: &str, + table_name: &NamespacedIdentifier, old_pk: Option, new_pk: Option, ) -> io::Result<()> { @@ -427,7 +427,7 @@ impl MigrationFormatter for TermColorFormatter { self.buffer.write_all(b"\n") } - fn format_event_table_reschema(&mut self, table_name: &Identifier) -> io::Result<()> { + fn format_event_table_reschema(&mut self, table_name: &NamespacedIdentifier) -> io::Result<()> { // TODO(format-event-table-reschema): I (pgoldman 2026-06-10) didn't have time to meaningfully format event table reschemas, // so for now we're just printing the table name. diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index 4ce29c0b98b..91e3299d7c9 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -20,7 +20,7 @@ use std::fmt::{self, Debug, Write}; use std::hash::Hash; use crate::error::{IdentifierError, ValidationErrors}; -use crate::identifier::Identifier; +use crate::identifier::{Identifier, NamespacePath, NamespacedIdentifier}; use crate::reducer_name::ReducerName; use crate::schema::{Schema, TableSchema}; use crate::type_for_generate::{AlgebraicTypeUse, ProductTypeDef, TypespaceForGenerate}; @@ -35,7 +35,7 @@ use spacetimedb_lib::db::raw_def::v10::{ ExplicitNames, MethodOrAny, RawColumnDefaultValueV10, RawConstraintDefV10, RawHttpHandlerDefV10, RawHttpRouteDefV10, RawIndexDefV10, RawLifeCycleReducerDefV10, RawModuleDefV10, RawModuleDefV10Section, RawProcedureDefV10, RawReducerDefV10, RawRowLevelSecurityDefV10, RawScheduleDefV10, RawScopedTypeNameV10, - RawSequenceDefV10, RawTableDefV10, RawTypeDefV10, RawViewDefV10, RawViewPrimaryKeyDefV10, + RawSequenceDefV10, RawSubmoduleV10, RawTableDefV10, RawTypeDefV10, RawViewDefV10, RawViewPrimaryKeyDefV10, }; use spacetimedb_lib::db::raw_def::v9::{ Lifecycle, RawColumnDefaultValueV9, RawConstraintDataV9, RawConstraintDefV9, RawIndexAlgorithm, RawIndexDefV9, @@ -76,7 +76,7 @@ pub type StrMap = HashMap; /// use spacetimedb_lib::RawModuleDef; /// use spacetimedb_sats::raw_identifier::RawIdentifier; /// use spacetimedb_schema::def::{ModuleDef, TableDef, IndexDef, TypeDef, ModuleDefLookup, ScopedTypeName}; -/// use spacetimedb_schema::identifier::Identifier; +/// use spacetimedb_schema::identifier::{Identifier, NamespacePath}; /// /// fn read_raw_module_def_from_file() -> RawModuleDef { /// // ... @@ -90,8 +90,13 @@ pub type StrMap = HashMap; /// let index_name: RawIdentifier = "my_table_my_column_idx_btree".into(); /// let scoped_type_name = ScopedTypeName::try_new([], "MyType").expect("valid scoped type name"); /// -/// let table: Option<&TableDef> = module_def.lookup(&table_name); -/// let index: Option<&IndexDef> = module_def.lookup(&index_name); +/// // Items owned by a module are keyed by `(namespace, name)`, where the namespace is the +/// // path the owning module is mounted under -- empty for the root module. +/// let root = NamespacePath::root(); +/// +/// let table: Option<&TableDef> = module_def.lookup((&root, &table_name)); +/// let index: Option<&IndexDef> = module_def.lookup((&root, &index_name)); +/// // Types are keyed by their scoped name, which is already unique across the module. /// let type_def: Option<&TypeDef> = module_def.lookup(&scoped_type_name); /// // etc. /// ``` @@ -105,6 +110,13 @@ pub type StrMap = HashMap; #[derive(Debug, Clone)] #[non_exhaustive] pub struct ModuleDef { + /// The namespace path this module is mounted under, empty for the root. + /// + /// Maintained by [`ModuleDef::apply_namespace`] alongside the `namespace` of every def, + /// and used to resolve namespace-qualified keys relative to whichever module they are + /// looked up in. + path: NamespacePath, + /// The tables of the module definition. tables: IdentifierMap, @@ -164,6 +176,9 @@ pub struct ModuleDef { /// was authored under. #[allow(unused)] raw_module_def_version: RawModuleDefVersion, + + /// Submodules, keyed by the namespace they are registered under. + submodules: IndexMap, } #[derive(Debug, Clone, Copy, Eq, PartialEq)] @@ -180,6 +195,11 @@ impl ModuleDef { self.raw_module_def_version } + /// The submodules of the module definition. + pub fn submodules(&self) -> &IndexMap { + &self.submodules + } + /// The tables of the module definition. pub fn tables(&self) -> impl Iterator { self.tables.values() @@ -205,14 +225,214 @@ impl ModuleDef { self.tables().filter_map(|table| table.schedule.as_ref()) } + /// All tables across this module and all submodules, in depth-first order. + /// + /// Each item is `(path, owning_def, table_def)` where `path` is the namespace path + /// to be prepended to the table's name for database storage (empty for the root). + pub fn all_tables_with_prefix(&self) -> Vec<(NamespacePath, &ModuleDef, &TableDef)> { + let mut out = Vec::new(); + self.collect_tables_with_prefix(&NamespacePath::root(), &mut out); + out + } + + fn collect_tables_with_prefix<'a>( + &'a self, + prefix: &NamespacePath, + out: &mut Vec<(NamespacePath, &'a ModuleDef, &'a TableDef)>, + ) { + for table in self.tables.values() { + out.push((prefix.clone(), self, table)); + } + for (ns, submodule) in &self.submodules { + submodule.collect_tables_with_prefix(&prefix.child(ns.clone()), out); + } + } + + /// All views across this module and all submodules, in depth-first order. + /// + /// Each item is `(path, owning_def, view_def)` where `path` is the namespace path + /// to be prepended to the view's name (empty for the root). + pub fn all_views_with_prefix(&self) -> Vec<(NamespacePath, &ModuleDef, &ViewDef)> { + let mut out = Vec::new(); + self.collect_views_with_prefix(&NamespacePath::root(), &mut out); + out + } + + fn collect_views_with_prefix<'a>( + &'a self, + prefix: &NamespacePath, + out: &mut Vec<(NamespacePath, &'a ModuleDef, &'a ViewDef)>, + ) { + for view in self.views.values() { + out.push((prefix.clone(), self, view)); + } + for (ns, submodule) in &self.submodules { + submodule.collect_views_with_prefix(&prefix.child(ns.clone()), out); + } + } + + /// Record the namespace path of this module on every def it owns, recursively. + /// + /// A def's [`ModuleDefLookup`] key is `(namespace, name)`, and a def cannot otherwise + /// know which namespace it is mounted under: submodules are validated in isolation, so + /// this runs once the module tree is assembled. + fn apply_namespace(&mut self, path: &NamespacePath) { + self.path = path.clone(); + for table in self.tables.values_mut() { + table.namespace = path.clone(); + for index in table.indexes.values_mut() { + index.namespace = path.clone(); + } + for sequence in table.sequences.values_mut() { + sequence.namespace = path.clone(); + } + for constraint in table.constraints.values_mut() { + constraint.namespace = path.clone(); + } + if let Some(schedule) = &mut table.schedule { + schedule.namespace = path.clone(); + } + } + for view in self.views.values_mut() { + view.namespace = path.clone(); + } + for reducer in self.reducers.values_mut() { + reducer.name = ReducerName::new(path.join(reducer.name.local().clone())); + } + for (namespace, submodule) in &mut self.submodules { + submodule.apply_namespace(&path.child(namespace.clone())); + } + } + + /// Debug-only check that every def's recorded namespace matches its module. + #[cfg(debug_assertions)] + fn assert_namespaces_applied(&self, path: &NamespacePath) { + for table in self.tables.values() { + debug_assert_eq!(&table.namespace, path, "table namespace stale"); + for index in table.indexes.values() { + debug_assert_eq!(&index.namespace, path, "index namespace stale"); + } + } + for view in self.views.values() { + debug_assert_eq!(&view.namespace, path, "view namespace stale"); + } + for (namespace, submodule) in &self.submodules { + submodule.assert_namespaces_applied(&path.child(namespace.clone())); + } + } + + /// Resolve the submodule mounted at `path`, relative to this module. + fn resolve_path(&self, path: &NamespacePath) -> Option<&ModuleDef> { + // Keys are qualified from the root, so drop the part of the path we already are. + let rest = path.segments().strip_prefix(self.path.segments())?; + let mut owning = self; + for namespace in rest { + owning = owning.submodules.get(namespace)?; + } + Some(owning) + } + + /// Look up a table by its [`ModuleDefLookup`] key, also returning the module that owns it. + /// + /// The owning module is needed to resolve the table's type refs, which point into that + /// module's typespace rather than the root's. + pub fn find_table(&self, key: ::Key<'_>) -> Option<(&ModuleDef, &TableDef)> { + let owning = self.resolve_path(key.0)?; + Some((owning, owning.tables.get(key.1)?)) + } + + /// Look up a view by its [`ModuleDefLookup`] key, also returning the module that owns it. + pub fn find_view(&self, key: ::Key<'_>) -> Option<(&ModuleDef, &ViewDef)> { + let owning = self.resolve_path(key.0)?; + Some((owning, owning.views.get(key.1)?)) + } + + /// The table that stores the sub-object (index, sequence, constraint or schedule) + /// named `name` in the module mounted at `namespace`. + pub fn find_storing_table( + &self, + namespace: &NamespacePath, + name: &RawIdentifier, + ) -> Option<(&ModuleDef, &TableDef)> { + let owning = self.resolve_path(namespace)?; + Some((owning, owning.stored_in_table_def(name)?)) + } + /// The reducers of the module definition. pub fn reducers(&self) -> impl Iterator { self.reducers.values() } - /// Returns an iterator over all reducer ids and definitions. - pub fn reducer_ids_and_defs(&self) -> impl ExactSizeIterator { - self.reducers.values().enumerate().map(|(idx, def)| (idx.into(), def)) + /// Returns all reducer ids and definitions in depth-first submodule order. + /// + /// IDs are assigned as follows: consumer's own reducers first (0..N), then each + /// submodule's reducers in the order they appear in `submodules`, recursively. + pub fn reducer_ids_and_defs(&self) -> Vec<(ReducerId, &ReducerDef)> { + let mut out = Vec::with_capacity(self.reducer_count()); + self.collect_reducers(0, &mut out); + out + } + + /// Total reducer count including all submodules (depth-first sum). + pub fn reducer_count(&self) -> usize { + self.reducers.len() + self.submodules.values().map(|m| m.reducer_count()).sum::() + } + + fn collect_reducers<'a>(&'a self, offset: usize, out: &mut Vec<(ReducerId, &'a ReducerDef)>) { + for (i, def) in self.reducers.values().enumerate() { + out.push(((offset + i).into(), def)); + } + let mut child_offset = offset + self.reducers.len(); + for submodule in self.submodules.values() { + submodule.collect_reducers(child_offset, out); + child_offset += submodule.reducer_count(); + } + } + + /// All reducers across this module and all submodules, in depth-first order. + /// + /// Each item is `(path, owning_def, reducer_def)` where `path` is the namespace path + /// to be prepended to the reducer's name as its wire name (empty for the root). + pub fn all_reducers_with_prefix(&self) -> Vec<(NamespacePath, &ModuleDef, &ReducerDef)> { + let mut out = Vec::new(); + self.collect_reducers_with_prefix(&NamespacePath::root(), &mut out); + out + } + + fn collect_reducers_with_prefix<'a>( + &'a self, + prefix: &NamespacePath, + out: &mut Vec<(NamespacePath, &'a ModuleDef, &'a ReducerDef)>, + ) { + for reducer in self.reducers.values() { + out.push((prefix.clone(), self, reducer)); + } + for (ns, submodule) in &self.submodules { + submodule.collect_reducers_with_prefix(&prefix.child(ns.clone()), out); + } + } + + /// All procedures across this module and all submodules, in depth-first order. + /// + /// Each item is `(path, owning_def, procedure_def)` where `path` is the namespace path + /// to be prepended to the procedure's name as its wire name (empty for the root). + pub fn all_procedures_with_prefix(&self) -> Vec<(NamespacePath, &ModuleDef, &ProcedureDef)> { + let mut out = Vec::new(); + self.collect_procedures_with_prefix(&NamespacePath::root(), &mut out); + out + } + + fn collect_procedures_with_prefix<'a>( + &'a self, + prefix: &NamespacePath, + out: &mut Vec<(NamespacePath, &'a ModuleDef, &'a ProcedureDef)>, + ) { + for procedure in self.procedures.values() { + out.push((prefix.clone(), self, procedure)); + } + for (ns, submodule) in &self.submodules { + submodule.collect_procedures_with_prefix(&prefix.child(ns.clone()), out); + } } /// The procedures of the module definition. @@ -230,6 +450,26 @@ impl ModuleDef { &self.http_routes } + /// Returns warnings about this module's definition that will not prevent publishing. + /// + /// - Warns when a submodule registers HTTP routes via a top-level router export + /// because those routes are ignored by the host. + pub fn collect_warnings(&self) -> Vec { + let mut warnings = Vec::new(); + for (namespace, submodule_def) in self.submodules() { + if !submodule_def.http_routes().is_empty() { + warnings.push(format!( + "The submodule under namespace '{namespace}' registers HTTP routes via a router. \ + Route registrations in submodules are ignored. Only the root module's routes \ + are served. Define routes in the root module and call the submodule's HTTP handler \ + functions via `ctx.as.{namespace}`." + )); + } + warnings.extend(submodule_def.collect_warnings()); + } + warnings + } + /// Returns an iterator over all HTTP handler ids and definitions. pub fn http_handler_ids_and_defs(&self) -> impl ExactSizeIterator { self.http_handlers @@ -364,14 +604,59 @@ impl ModuleDef { self.reducers.get_full(name).map(|(idx, _, def)| (idx.into(), def)) } - /// Look up a reducer by its id. + /// Look up a reducer by its wire name, resolving qualified names like `"myauth.verify_token"`. + /// + /// A plain name searches the consumer's own reducers. A dot-qualified name routes to + /// the matching submodule and recurses. Nesting is supported: `"auth.baz.cleanup"`. + /// Returns the depth-first `ReducerId` and the `ReducerDef`. + pub fn reducer_by_name(&self, name: &str) -> Option<(ReducerId, &ReducerDef)> { + self.reducer_by_name_with_module(name).map(|(id, def, _)| (id, def)) + } + + /// Like `reducer_by_name` but also returns the `ModuleDef` that owns the reducer. + /// Use the returned `ModuleDef` (not `self`) when calling `arg_seed_for`, so that + /// type-index references in the `ReducerDef` are resolved against the correct typespace. + pub fn reducer_by_name_with_module<'a>(&'a self, name: &str) -> Option<(ReducerId, &'a ReducerDef, &'a ModuleDef)> { + match name.split_once('.') { + None => self + .reducers + .get_full(name) + .map(|(idx, _, def)| (idx.into(), def, self)), + Some((namespace, rest)) => { + let mut offset = self.reducers.len(); + for (ns, submodule) in &self.submodules { + if ns == namespace { + let (inner_id, def, owning) = submodule.reducer_by_name_with_module(rest)?; + return Some(((offset + inner_id.idx()).into(), def, owning)); + } + offset += submodule.reducer_count(); + } + None + } + } + } + + /// Look up a reducer by its depth-first id. pub fn reducer_by_id(&self, id: ReducerId) -> &ReducerDef { - &self.reducers[id.idx()] + self.get_reducer_by_id(id) + .unwrap_or_else(|| panic!("reducer id {id:?} out of range")) } - /// Look up a reducer by its id. + /// Look up a reducer by its depth-first id, returning `None` if it doesn't exist. pub fn get_reducer_by_id(&self, id: ReducerId) -> Option<&ReducerDef> { - self.reducers.get_index(id.idx()).map(|(_, def)| def) + let idx = id.idx(); + if idx < self.reducers.len() { + return self.reducers.get_index(idx).map(|(_, def)| def); + } + let mut offset = self.reducers.len(); + for submodule in self.submodules.values() { + let count = submodule.reducer_count(); + if idx < offset + count { + return submodule.get_reducer_by_id(ReducerId::from(idx - offset)); + } + offset += count; + } + None } /// Look up a view by its id, and whether it is anonymous. @@ -382,6 +667,139 @@ impl ModuleDef { .map(|(_, def)| def) } + /// Look up a view by its globally-unique fn_ptr (the offset-adjusted id used by the WASM dispatch layer). + /// Returns the `ViewDef` and the owning `ModuleDef`. + pub fn get_view_by_global_id_with_module( + &self, + global_id: ViewFnPtr, + is_anonymous: bool, + ) -> Option<(&ViewDef, &ModuleDef)> { + self.get_view_by_global_id_inner(global_id.0, is_anonymous, 0, 0) + } + + fn get_view_by_global_id_inner( + &self, + global_id: u32, + is_anonymous: bool, + anon_offset: u32, + non_anon_offset: u32, + ) -> Option<(&ViewDef, &ModuleDef)> { + let local_count = if is_anonymous { + self.anon_view_count() as u32 + } else { + self.non_anon_view_count() as u32 + }; + let offset = if is_anonymous { anon_offset } else { non_anon_offset }; + if global_id < offset + local_count { + return self + .views + .values() + .find(|def| def.fn_ptr.0 + offset == global_id && def.is_anonymous == is_anonymous) + .map(|def| (def, self)); + } + let mut anon_off = anon_offset + self.anon_view_count() as u32; + let mut non_anon_off = non_anon_offset + self.non_anon_view_count() as u32; + for submodule in self.submodules.values() { + let submodule_anon = submodule.total_anon_view_count() as u32; + let submodule_non_anon = submodule.total_non_anon_view_count() as u32; + let submodule_count = if is_anonymous { + submodule_anon + } else { + submodule_non_anon + }; + let submodule_off = if is_anonymous { anon_off } else { non_anon_off }; + if global_id < submodule_off + submodule_count { + return submodule.get_view_by_global_id_inner(global_id, is_anonymous, anon_off, non_anon_off); + } + anon_off += submodule_anon; + non_anon_off += submodule_non_anon; + } + None + } + + /// Look up a view by its wire name, resolving dot-qualified names like `"lib.library_view"`. + /// + /// A plain name searches this module's own views. A dot-qualified name routes to + /// the matching submodule and recurses. Returns the `ViewDef` and the owning `ModuleDef`. + pub fn view_by_name_with_module<'a>(&'a self, name: &str) -> Option<(&'a ViewDef, &'a ModuleDef)> { + match name.split_once('.') { + None => self.views.get(name).map(|def| (def, self)), + Some((namespace, rest)) => { + let submodule = self.submodules.get(namespace)?; + submodule.view_by_name_with_module(rest) + } + } + } + + /// Like [`view_by_name_with_module`] but also returns the globally-unique `ViewFnPtr` + /// that the WASM dispatch layer expects (offset by all anon/non-anon views that precede + /// this one in depth-first module order). + pub fn view_by_name_with_global_fn_ptr<'a>( + &'a self, + name: &NamespacedIdentifier, + ) -> Option<(ViewFnPtr, &'a ViewDef, &'a ModuleDef)> { + let anon_offset = 0u32; + let non_anon_offset = 0u32; + self.view_by_name_with_global_fn_ptr_inner(name, anon_offset, non_anon_offset) + } + + fn view_by_name_with_global_fn_ptr_inner<'a>( + &'a self, + name: &str, + anon_offset: u32, + non_anon_offset: u32, + ) -> Option<(ViewFnPtr, &'a ViewDef, &'a ModuleDef)> { + match name.split_once('.') { + None => { + let def = self.views.get(name)?; + let offset = if def.is_anonymous { anon_offset } else { non_anon_offset }; + Some((ViewFnPtr(def.fn_ptr.0 + offset), def, self)) + } + Some((namespace, rest)) => { + let mut anon_off = anon_offset + self.anon_view_count() as u32; + let mut non_anon_off = non_anon_offset + self.non_anon_view_count() as u32; + for (ns, submodule) in &self.submodules { + if ns == namespace { + return submodule.view_by_name_with_global_fn_ptr_inner(rest, anon_off, non_anon_off); + } + anon_off += submodule.total_anon_view_count() as u32; + non_anon_off += submodule.total_non_anon_view_count() as u32; + } + None + } + } + } + + /// Count of anonymous views in this module (not including submodules). + pub fn anon_view_count(&self) -> usize { + self.views.values().filter(|v| v.is_anonymous).count() + } + + /// Count of non-anonymous views in this module (not including submodules). + pub fn non_anon_view_count(&self) -> usize { + self.views.values().filter(|v| !v.is_anonymous).count() + } + + /// Total anonymous view count including all submodules (depth-first sum). + pub fn total_anon_view_count(&self) -> usize { + self.anon_view_count() + + self + .submodules + .values() + .map(|m| m.total_anon_view_count()) + .sum::() + } + + /// Total non-anonymous view count including all submodules (depth-first sum). + pub fn total_non_anon_view_count(&self) -> usize { + self.non_anon_view_count() + + self + .submodules + .values() + .map(|m| m.total_non_anon_view_count()) + .sum::() + } + /// Convenience method to look up a procedure, possibly by a string. pub fn procedure>(&self, name: &K) -> Option<&ProcedureDef> { // If the string IS a valid identifier, we can just look it up. @@ -399,12 +817,63 @@ impl ModuleDef { /// Look up a procuedure by its id, panicking if it doesn't exist. pub fn procedure_by_id(&self, id: ProcedureId) -> &ProcedureDef { - &self.procedures[id.idx()] + self.get_procedure_by_id(id) + .unwrap_or_else(|| panic!("procedure id {id:?} out of range")) } /// Look up a procuedure by its id, returning `None` if it doesn't exist. pub fn get_procedure_by_id(&self, id: ProcedureId) -> Option<&ProcedureDef> { - self.procedures.get_index(id.idx()).map(|(_, def)| def) + let idx = id.idx(); + if idx < self.procedures.len() { + return self.procedures.get_index(idx).map(|(_, def)| def); + } + let mut offset = self.procedures.len(); + for submodule in self.submodules.values() { + let count = submodule.procedure_count(); + if idx < offset + count { + return submodule.get_procedure_by_id(ProcedureId::from(idx - offset)); + } + offset += count; + } + None + } + + /// Total procedure count including all submodules (depth-first sum). + pub fn procedure_count(&self) -> usize { + self.procedures.len() + self.submodules.values().map(|m| m.procedure_count()).sum::() + } + + /// Look up a procedure by its wire name, resolving qualified names like `"mylib.proc_name"`. + /// + /// A plain name searches the module's own procedures. A dot-qualified name routes to + /// the matching submodule and recurses. Returns the depth-first `ProcedureId` and the `ProcedureDef`. + pub fn procedure_by_name(&self, name: &str) -> Option<(ProcedureId, &ProcedureDef)> { + self.procedure_by_name_with_module(name).map(|(id, def, _)| (id, def)) + } + + /// Like `procedure_by_name` but also returns the `ModuleDef` that owns the procedure. + /// Use the returned `ModuleDef` (not `self`) when calling `arg_seed_for`. + pub fn procedure_by_name_with_module<'a>( + &'a self, + name: &str, + ) -> Option<(ProcedureId, &'a ProcedureDef, &'a ModuleDef)> { + match name.split_once('.') { + None => self + .procedures + .get_full(name) + .map(|(idx, _, def)| (idx.into(), def, self)), + Some((namespace, rest)) => { + let mut offset = self.procedures.len(); + for (ns, submodule) in &self.submodules { + if ns == namespace { + let (inner_id, def, owning) = submodule.procedure_by_name_with_module(rest)?; + return Some(((offset + inner_id.idx()).into(), def, owning)); + } + offset += submodule.procedure_count(); + } + None + } + } } /// Looks up a lifecycle reducer defined in the module. @@ -412,6 +881,11 @@ impl ModuleDef { self.lifecycle_reducers[lifecycle].map(|i| (i, &self.reducers[i.idx()])) } + /// All lifecycle reducer assignments for this module (does not include submodules). + pub fn lifecycle_reducers_map(&self) -> &EnumMap> { + &self.lifecycle_reducers + } + /// Returns a `DeserializeSeed` that can pull data from a `Deserializer` for `def`. pub fn arg_seed_for<'a, T>(&'a self, def: &'a T) -> ArgsSeed<'a, T> { ArgsSeed(self.typespace.with_type(def)) @@ -490,9 +964,13 @@ impl TryFrom for ModuleDef { validate::v9::validate(v9_mod) } } +/// Note: this conversion is lossy for modules with submodules. `RawModuleDefV9` has no +/// submodule representation, so submodules (and everything defined in them) are dropped. +/// Callers serving V9 to old clients should be aware those clients see a partial module. impl From for RawModuleDefV9 { fn from(val: ModuleDef) -> Self { let ModuleDef { + path: _, tables, views, reducers, @@ -507,6 +985,7 @@ impl From for RawModuleDefV9 { http_handlers: _, http_routes: _, raw_module_def_version: _, + submodules: _, } = val; // Extract column defaults from tables before consuming tables @@ -551,6 +1030,7 @@ impl TryFrom for ModuleDef { impl From for RawModuleDefV10 { fn from(val: ModuleDef) -> Self { let ModuleDef { + path: _, tables, views, reducers, @@ -565,6 +1045,7 @@ impl From for RawModuleDefV10 { http_handlers, http_routes, raw_module_def_version: _, + submodules, } = val; let mut sections = Vec::new(); @@ -723,6 +1204,17 @@ impl From for RawModuleDefV10 { // Always emit ExplicitNames so canonical names survive the round-trip. sections.push(RawModuleDefV10Section::ExplicitNames(explicit_names)); + let submodules: Vec<_> = submodules + .into_iter() + .map(|(namespace, module)| RawSubmoduleV10 { + namespace: namespace.to_string(), + module: module.into(), + }) + .collect(); + if !submodules.is_empty() { + sections.push(RawModuleDefV10Section::Submodules(submodules)); + } + RawModuleDefV10 { sections } } } @@ -761,6 +1253,14 @@ pub struct TableDef { /// Unique within a module, acts as the table's identifier. /// Must be a valid [crate::db::identifier::Identifier]. pub name: Identifier, + + /// The namespace path this table's module is mounted under, empty for the root. + /// + /// Together with `name` this forms the table's [`ModuleDefLookup`] key. Storing the + /// path rather than a pre-joined qualified name keeps the local name un-duplicated; + /// the joined form is built at the few places that need it (see + /// [`NamespacePath::join`]). Maintained by [`ModuleDef::apply_namespace`]. + pub namespace: NamespacePath, /// The table identifier as defined in the module source. /// /// All the codegens should use `accessor_name` to derive the table identifier for clients. @@ -858,6 +1358,7 @@ impl From for RawTableDefV10 { fn from(val: TableDef) -> Self { let TableDef { name: _, + namespace: _, product_type_ref, primary_key, columns, // names/types are reconstructed from the product type; only defaults are read here. @@ -908,6 +1409,7 @@ impl From for TableDef { .. } = def; Self { + namespace: NamespacePath::root(), name, product_type_ref, primary_key, @@ -930,6 +1432,12 @@ pub struct SequenceDef { /// The name of the sequence. Must be unique within the containing `ModuleDef`. pub name: RawIdentifier, + /// The namespace path this sequence's module is mounted under. + /// + /// Together with `name` this forms the sequence's [`ModuleDefLookup`] key. A pair is used + /// rather than a dotted name because `name` is not guaranteed to be dot-free. + pub namespace: NamespacePath, + /// The position of the column associated with this sequence. /// This refers to a column in the same `RawTableDef` that contains this `RawSequenceDef`. /// The column must have integral type. @@ -993,6 +1501,12 @@ pub struct IndexDef { /// generated by the system using the same algorithm as V9 and earlier. pub name: RawIdentifier, + /// The namespace path this index's module is mounted under. + /// + /// Together with `name` this forms the index's [`ModuleDefLookup`] key. A pair is used + /// rather than a dotted name because `name` is not guaranteed to be dot-free. + pub namespace: NamespacePath, + /// This will be the same as `name` for [`RawModuleDefV9`]. /// For [`RawModuleDefV10`], this is an auto-generated globally unique name /// derived from the module source. @@ -1302,6 +1816,12 @@ pub struct ConstraintDef { /// The name of the constraint. Unique within the containing `ModuleDef`. pub name: RawIdentifier, + /// The namespace path this constraint's module is mounted under. + /// + /// Together with `name` this forms the constraint's [`ModuleDefLookup`] key. A pair is used + /// rather than a dotted name because `name` is not guaranteed to be dot-free. + pub namespace: NamespacePath, + /// The data for the constraint. pub data: ConstraintData, } @@ -1429,6 +1949,10 @@ pub struct ScheduleDef { /// The name of the schedule. Must be unique within the containing `ModuleDef`. pub name: Identifier, + /// The namespace path this schedule's module is mounted under. + /// See [`TableDef::namespace`]. + pub namespace: NamespacePath, + /// The name of the column that stores the desired invocation time. /// /// Must be named `scheduled_at` and be of type `ScheduleAt`. @@ -1603,6 +2127,10 @@ pub struct ViewDef { /// The name of the view. This must be unique within the module. pub name: Identifier, + /// The namespace path this view's module is mounted under. + /// See [`TableDef::namespace`]. + pub namespace: NamespacePath, + /// The view identifier as defined in the module source. /// /// Similar to `[TableDef.accessor_name]`. @@ -1781,7 +2309,8 @@ impl From for RawFunctionVisibility { #[derive(Debug, Clone, Eq, PartialEq)] #[non_exhaustive] pub struct ReducerDef { - /// The name of the reducer. This must be unique within the module's set of reducers and procedures. + /// The name of the reducer. This must be unique within the module's set of reducers and + /// procedures. Qualified by the namespace the module is mounted under; see [`ReducerName`]. pub name: ReducerName, /// The reducer name as defined in the module source. @@ -1921,38 +2450,43 @@ impl From for RawMiscModuleExportV9 { } impl ModuleDefLookup for TableDef { - type Key<'a> = &'a Identifier; + // (namespace, name). See [`TableDef::namespace`]. + type Key<'a> = (&'a NamespacePath, &'a Identifier); fn key(&self) -> Self::Key<'_> { - &self.name + (&self.namespace, &self.name) } - fn lookup<'a>(module_def: &'a ModuleDef, key: Self::Key<'_>) -> Option<&'a Self> { - module_def.tables.get(key) + fn lookup<'a>(module_def: &'a ModuleDef, (namespace, name): Self::Key<'_>) -> Option<&'a Self> { + module_def.resolve_path(namespace)?.tables.get(name) } } impl ModuleDefLookup for SequenceDef { - type Key<'a> = &'a RawIdentifier; + // (namespace, name). See [`SequenceDef::namespace`]. + type Key<'a> = (&'a NamespacePath, &'a RawIdentifier); fn key(&self) -> Self::Key<'_> { - &self.name + (&self.namespace, &self.name) } - fn lookup<'a>(module_def: &'a ModuleDef, key: Self::Key<'_>) -> Option<&'a Self> { - module_def.stored_in_table_def(key)?.sequences.get(key) + fn lookup<'a>(module_def: &'a ModuleDef, (namespace, name): Self::Key<'_>) -> Option<&'a Self> { + let owning = module_def.resolve_path(namespace)?; + owning.stored_in_table_def(name)?.sequences.get(name) } } impl ModuleDefLookup for IndexDef { - type Key<'a> = &'a RawIdentifier; + // (namespace, name). See [`IndexDef::namespace`]. + type Key<'a> = (&'a NamespacePath, &'a RawIdentifier); fn key(&self) -> Self::Key<'_> { - &self.name + (&self.namespace, &self.name) } - fn lookup<'a>(module_def: &'a ModuleDef, key: Self::Key<'_>) -> Option<&'a Self> { - module_def.stored_in_table_def(key)?.indexes.get(key) + fn lookup<'a>(module_def: &'a ModuleDef, (namespace, name): Self::Key<'_>) -> Option<&'a Self> { + let owning = module_def.resolve_path(namespace)?; + owning.stored_in_table_def(name)?.indexes.get(name) } } @@ -2009,14 +2543,16 @@ impl ModuleDefLookup for ViewParamDef { } impl ModuleDefLookup for ConstraintDef { - type Key<'a> = &'a RawIdentifier; + // (namespace, name). See [`ConstraintDef::namespace`]. + type Key<'a> = (&'a NamespacePath, &'a RawIdentifier); fn key(&self) -> Self::Key<'_> { - &self.name + (&self.namespace, &self.name) } - fn lookup<'a>(module_def: &'a ModuleDef, key: Self::Key<'_>) -> Option<&'a Self> { - module_def.stored_in_table_def(key)?.constraints.get(key) + fn lookup<'a>(module_def: &'a ModuleDef, (namespace, name): Self::Key<'_>) -> Option<&'a Self> { + let owning = module_def.resolve_path(namespace)?; + owning.stored_in_table_def(name)?.constraints.get(name) } } @@ -2033,19 +2569,17 @@ impl ModuleDefLookup for RawRowLevelSecurityDefV9 { } impl ModuleDefLookup for ScheduleDef { - type Key<'a> = &'a Identifier; + // (namespace, name). See [`ScheduleDef::namespace`]. + type Key<'a> = (&'a NamespacePath, &'a Identifier); fn key(&self) -> Self::Key<'_> { - &self.name + (&self.namespace, &self.name) } - fn lookup<'a>(module_def: &'a ModuleDef, key: Self::Key<'_>) -> Option<&'a Self> { - let schedule = module_def.stored_in_table_def(key.as_raw())?.schedule.as_ref()?; - if &schedule.name == key { - Some(schedule) - } else { - None - } + fn lookup<'a>(module_def: &'a ModuleDef, (namespace, name): Self::Key<'_>) -> Option<&'a Self> { + let owning = module_def.resolve_path(namespace)?; + let schedule = owning.stored_in_table_def(name.as_raw())?.schedule.as_ref()?; + (&schedule.name == name).then_some(schedule) } } @@ -2088,14 +2622,15 @@ impl ModuleDefLookup for ProcedureDef { } impl ModuleDefLookup for ViewDef { - type Key<'a> = &'a Identifier; + // (namespace, name). See [`TableDef::namespace`]. + type Key<'a> = (&'a NamespacePath, &'a Identifier); fn key(&self) -> Self::Key<'_> { - &self.name + (&self.namespace, &self.name) } - fn lookup<'a>(view_def: &'a ModuleDef, key: Self::Key<'_>) -> Option<&'a Self> { - view_def.views.get(key) + fn lookup<'a>(module_def: &'a ModuleDef, (namespace, name): Self::Key<'_>) -> Option<&'a Self> { + module_def.resolve_path(namespace)?.views.get(name) } } @@ -2211,4 +2746,85 @@ mod tests { .count() == 2)) } + + #[test] + fn submodule_reducer_ids_are_depth_first() { + use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10Builder, RawModuleDefV10Section, RawSubmoduleV10}; + + // baz library: 1 reducer + let mut baz_builder = RawModuleDefV10Builder::new(); + baz_builder.add_reducer("baz_reduce", ProductType::unit()); + + // auth library: 1 own reducer, uses baz as a submodule + let mut auth_builder = RawModuleDefV10Builder::new(); + auth_builder.add_reducer("auth_verify", ProductType::unit()); + let mut auth_raw = auth_builder.finish(); + auth_raw + .sections + .push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "baz".to_string(), + module: baz_builder.finish(), + }])); + + // consumer: 2 own reducers, uses auth as a submodule + let mut consumer_builder = RawModuleDefV10Builder::new(); + consumer_builder.add_reducer("consumer_a", ProductType::unit()); + consumer_builder.add_reducer("consumer_b", ProductType::unit()); + let mut consumer_raw = consumer_builder.finish(); + consumer_raw + .sections + .push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "auth".to_string(), + module: auth_raw, + }])); + + let def: ModuleDef = consumer_raw.try_into().expect("valid module"); + + // Total count: 2 consumer + 1 auth + 1 baz + assert_eq!(def.reducer_count(), 4); + + // Depth-first order: consumer_a=0, consumer_b=1, auth_verify=2, baz_reduce=3 + let ids_and_defs = def.reducer_ids_and_defs(); + assert_eq!(ids_and_defs.len(), 4); + assert_eq!(ids_and_defs[0].0, ReducerId(0)); + assert_eq!(&*ids_and_defs[0].1.name, "consumer_a"); + assert_eq!(ids_and_defs[1].0, ReducerId(1)); + assert_eq!(&*ids_and_defs[1].1.name, "consumer_b"); + assert_eq!(ids_and_defs[2].0, ReducerId(2)); + // A submodule reducer's `name` is qualified; `local()` is the name within its module. + assert_eq!(&*ids_and_defs[2].1.name, "auth.auth_verify"); + assert_eq!(&**ids_and_defs[2].1.name.local(), "auth_verify"); + assert_eq!(ids_and_defs[3].0, ReducerId(3)); + assert_eq!(&*ids_and_defs[3].1.name, "auth.baz.baz_reduce"); + assert_eq!(&**ids_and_defs[3].1.name.local(), "baz_reduce"); + + // get_reducer_by_id resolves submodule reducer IDs correctly + assert_eq!(&*def.reducer_by_id(ReducerId(2)).name, "auth.auth_verify"); + assert_eq!(&*def.reducer_by_id(ReducerId(3)).name, "auth.baz.baz_reduce"); + assert!(def.get_reducer_by_id(ReducerId(4)).is_none()); + + // reducer_by_name routes plain names to own reducers + let (id, rdef) = def.reducer_by_name("consumer_a").expect("plain name resolves"); + assert_eq!(id, ReducerId(0)); + assert_eq!(&*rdef.name, "consumer_a"); + + // reducer_by_name routes qualified names to submodule reducers + let (id, rdef) = def + .reducer_by_name("auth.auth_verify") + .expect("qualified name resolves"); + assert_eq!(id, ReducerId(2)); + assert_eq!(&*rdef.name, "auth.auth_verify"); + + // reducer_by_name routes deeply nested qualified names + let (id, rdef) = def + .reducer_by_name("auth.baz.baz_reduce") + .expect("nested qualified name resolves"); + assert_eq!(id, ReducerId(3)); + assert_eq!(&*rdef.name, "auth.baz.baz_reduce"); + + // Non-existent names return None + assert!(def.reducer_by_name("auth.nonexistent").is_none()); + assert!(def.reducer_by_name("nonexistent").is_none()); + assert!(def.reducer_by_name("nonamespace.auth_verify").is_none()); + } } diff --git a/crates/schema/src/def/deserialize.rs b/crates/schema/src/def/deserialize.rs index 336ff2a22a3..6e2e0c80af8 100644 --- a/crates/schema/src/def/deserialize.rs +++ b/crates/schema/src/def/deserialize.rs @@ -34,7 +34,7 @@ impl FunctionDef for ReducerDef { &self.params } fn name(&self) -> &Identifier { - self.name.as_identifier() + self.name.local() } } diff --git a/crates/schema/src/def/error.rs b/crates/schema/src/def/error.rs index da3ec2e1ac9..77bb195ebcb 100644 --- a/crates/schema/src/def/error.rs +++ b/crates/schema/src/def/error.rs @@ -5,7 +5,7 @@ use derive_more::Display; use spacetimedb_lib::db::raw_def::IndexType; use spacetimedb_primitives::{ColId, ColList}; use spacetimedb_sats::product_value::InvalidFieldError; -use spacetimedb_sats::raw_identifier::RawIdentifier; +use spacetimedb_sats::raw_identifier::RawNamespacedIdentifier; use spacetimedb_sats::satn::Satn as _; use spacetimedb_sats::{buffer, AlgebraicType, AlgebraicValue}; use std::fmt; @@ -114,7 +114,7 @@ pub enum SchemaError { MultiplePrimaryKeys { table: Box, pks: Vec }, #[error("{ty} {name} columns `{columns:?}` not found in table `{table}`")] ColumnsNotFound { - name: RawIdentifier, + name: RawNamespacedIdentifier, table: TableName, columns: Vec, ty: DefType, @@ -124,7 +124,7 @@ pub enum SchemaError { #[error("table `{table}` have `Constraints::unset()` for columns: {columns:?}")] ConstraintUnset { table: TableName, - name: RawIdentifier, + name: RawNamespacedIdentifier, columns: ColList, }, #[error("Attempt to define a column with more than 1 auto_inc sequence: Table: `{table}`, Field: `{field}`")] @@ -132,7 +132,7 @@ pub enum SchemaError { #[error("Only Btree Indexes are supported: Table: `{table}`, Index: `{index}` is a `{index_type}`")] OnlyBtree { table: TableName, - index: RawIdentifier, + index: RawNamespacedIdentifier, index_type: IndexType, }, } diff --git a/crates/schema/src/def/validate/v10.rs b/crates/schema/src/def/validate/v10.rs index aa398743f97..d6303f3a81c 100644 --- a/crates/schema/src/def/validate/v10.rs +++ b/crates/schema/src/def/validate/v10.rs @@ -2,9 +2,11 @@ use spacetimedb_data_structures::error_stream::ErrorStream; use spacetimedb_data_structures::map::HashMap; use spacetimedb_lib::bsatn::Deserializer; use spacetimedb_lib::db::raw_def::v10::*; +use spacetimedb_lib::db::raw_def::v9::Lifecycle; use spacetimedb_lib::db::view::{extract_view_return_product_type_ref, ViewKind}; use spacetimedb_lib::de::DeserializeSeed as _; use spacetimedb_lib::http::character_is_acceptable_for_route_path; +use spacetimedb_primitives::ReducerId; use spacetimedb_sats::{Typespace, WithTypespace}; use crate::def::validate::v9::{ @@ -13,8 +15,10 @@ use crate::def::validate::v9::{ }; use crate::def::*; use crate::error::ValidationError; +use crate::identifier::NamespacePath; use crate::type_for_generate::ProductTypeDef; use crate::{def::validate::Result, error::TypeLocation}; +use spacetimedb_sats::raw_identifier::RawIdentifier; // Utitility struct to look up canonical names for tables, functions, and indexes based on the // explicit names provided in the `RawModuleDefV10`. #[derive(Default)] @@ -71,7 +75,7 @@ impl From for ValidationCase { } } } -/// Validate a `RawModuleDefV9` and convert it into a `ModuleDef`, +/// Validate a `RawModuleDefV10` and convert it into a `ModuleDef`, /// or return a stream of errors if the definition is invalid. pub fn validate(def: RawModuleDefV10) -> Result { let mut typespace = def.typespace().cloned().unwrap_or_else(|| Typespace::EMPTY.clone()); @@ -83,6 +87,7 @@ pub fn validate(def: RawModuleDefV10) -> Result { .map(ExplicitNamesLookup::new) .unwrap_or_default(); let view_primary_keys = def.view_primary_keys().cloned().unwrap_or_default(); + let submodules = validate_submodules(def.submodules().into_iter().flat_map(|s| s.iter().cloned()).collect()); // Original `typespace` needs to be preserved to be assign `accesor_name`s to columns. let typespace_with_accessor_names = typespace.clone(); @@ -292,18 +297,16 @@ pub fn validate(def: RawModuleDefV10) -> Result { .map(|rls| (rls.sql.clone(), rls.to_owned())) .collect(); - let (tables, types, reducers, procedures, views, http_handlers, http_routes) = - tables_types_reducers_procedures_views - .map( - |(tables, types, reducers, procedures, views, (http_handlers, http_routes))| { - (tables, types, reducers, procedures, views, http_handlers, http_routes) - }, - ) + let ((tables, types, reducers, procedures, views, (http_handlers, http_routes)), submodules) = + (tables_types_reducers_procedures_views, submodules) + .combine_errors() .map_err(|errors: ValidationErrors| errors.sort_deduplicate())?; let typespace_for_generate = typespace_for_generate.finish(); - Ok(ModuleDef { + let mut module_def = ModuleDef { + // Set by `apply_namespace` below. + path: NamespacePath::root(), tables, reducers, views, @@ -318,7 +321,66 @@ pub fn validate(def: RawModuleDefV10) -> Result { http_handlers, http_routes, raw_module_def_version: RawModuleDefVersion::V10, - }) + submodules, + }; + + // Submodules were validated in isolation, so their defs carry root-relative names. + // Now that the tree is assembled, qualify every name by the namespace it is mounted + // under. This recurses, so nesting resolves at whichever level ends up outermost. + module_def.apply_namespace(&NamespacePath::root()); + #[cfg(debug_assertions)] + module_def.assert_namespaces_applied(&NamespacePath::root()); + + Ok(module_def) +} + +/// Validate that each submodule's namespace is a valid identifier of at most 63 characters, +/// that no two submodules share the same namespace, and that no submodule declares lifecycle +/// reducers (lifecycle reducers are only permitted in the root module). +/// This function will inspect each sub-submodule and recursively collect errors. +fn validate_submodules(submodules: Vec) -> Result> { + let mut errors = vec![]; + let mut map = IndexMap::with_capacity(submodules.len()); + + for submodule in submodules { + if submodule.namespace.len() > 63 { + errors.push(ValidationError::NamespaceTooLong { + namespace: submodule.namespace.clone().into(), + len: submodule.namespace.len(), + }); + } + + let namespace = match Identifier::new(RawIdentifier::from(submodule.namespace.clone())) { + Ok(namespace) => namespace, + Err(error) => { + errors.push(ValidationError::IdentifierError { error }); + continue; + } + }; + + if map.contains_key(&namespace) { + errors.push(ValidationError::DuplicateName { + name: submodule.namespace.into(), + }); + } else { + match validate(submodule.module) { + Ok(def) => { + for (lifecycle, opt_id) in def.lifecycle_reducers_map() { + if opt_id.is_some() { + errors.push(ValidationError::LifecycleInSubmodule { + lifecycle, + namespace: submodule.namespace.clone(), + }); + } + } + map.insert(namespace, def); + } + Err(e) => errors.extend(e.into_iter()), + } + } + } + + ValidationErrors::add_extra_errors(Ok(map), errors) } /// Change the visibility of scheduled functions and lifecycle reducers to Internal. @@ -586,8 +648,11 @@ impl<'a> ModuleValidatorV10<'a> { ) .combine_errors()?; + let name = identifier(name)?; Ok(TableDef { - name: identifier(name)?, + // Set by `ModuleDef::apply_namespace` once the module tree is assembled. + namespace: NamespacePath::root(), + name, product_type_ref, primary_key, columns, @@ -890,6 +955,7 @@ impl<'a> ModuleValidatorV10<'a> { (return_type_for_generate, return_columns, param_columns).combine_errors()?; Ok(ViewDef { + namespace: NamespacePath::root(), name, accessor_name: identifier(accessor_name)?, is_anonymous, @@ -1090,16 +1156,21 @@ mod tests { }; use crate::error::*; use crate::identifier::Identifier; + use crate::identifier::NamespacePath; use crate::type_for_generate::ClientCodegenError; use itertools::Itertools; use spacetimedb_data_structures::expect_error_matching; - use spacetimedb_lib::db::raw_def::v10::{CaseConversionPolicy, MethodOrAny, RawModuleDefV10Builder}; + use spacetimedb_lib::db::raw_def::v10::{ + CaseConversionPolicy, MethodOrAny, RawModuleDefV10, RawModuleDefV10Builder, RawModuleDefV10Section, + RawSubmoduleV10, + }; use spacetimedb_lib::db::raw_def::v9::{btree, direct, hash}; use spacetimedb_lib::db::raw_def::*; use spacetimedb_lib::http::Method as HttpMethod; use spacetimedb_lib::ScheduleAt; use spacetimedb_primitives::{ColId, ColList, ColSet}; + use spacetimedb_sats::raw_identifier::RawIdentifier; use spacetimedb_sats::{AlgebraicType, AlgebraicTypeRef, AlgebraicValue, ProductType, SumValue}; use v9::{Lifecycle, TableAccess, TableType}; @@ -1250,6 +1321,8 @@ mod tests { [ &IndexDef { name: "apples_apple_name_count_fresh_idx_btree".into(), + + namespace: NamespacePath::root(), source_name: "apples_id".into(), accessor_name: None, algorithm: BTreeAlgorithm { @@ -1259,12 +1332,16 @@ mod tests { }, &IndexDef { name: "apples_count_fresh_idx_direct".into(), + + namespace: NamespacePath::root(), accessor_name: None, source_name: "Apples_count_direct".into(), algorithm: DirectAlgorithm { column: ColId(2) }.into() }, &IndexDef { name: "apples_type_idx_btree".into(), + + namespace: NamespacePath::root(), source_name: "Apples_type_btree".into(), accessor_name: None, algorithm: BTreeAlgorithm { @@ -1474,6 +1551,118 @@ mod tests { }); } + /// Defs in a submodule must carry namespace-qualified keys, and `lookup` must round-trip + /// through them. Nesting resolves at whichever level ends up outermost. + #[test] + fn submodule_defs_have_namespaced_keys() { + use crate::def::{ModuleDefLookup, TableDef}; + + let mut inner = RawModuleDefV10Builder::new(); + inner + .build_table_with_new_type("sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + + let mut middle = RawModuleDefV10Builder::new().finish(); + middle + .sections + .push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "inner".to_string(), + module: inner.finish(), + }])); + + let mut root = RawModuleDefV10Builder::new().finish(); + root.sections + .push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "outer".to_string(), + module: middle, + }])); + + let def: ModuleDef = root.try_into().expect("should validate"); + + let (_, _, table) = def + .all_tables_with_prefix() + .into_iter() + .next() + .expect("nested submodule table should exist"); + + // The recorded namespace is the full mount path, not just the innermost segment, + // and the local name is untouched. + assert_eq!(table.namespace.to_string(), "outer.inner."); + assert_eq!(&*table.name, "sessions"); + // The joined form is built on demand from the two. + assert_eq!(&*table.namespace.join(table.name.clone()), "outer.inner.sessions"); + + // And the key round-trips through `lookup` from the root. + let found = ::lookup(&def, table.key()).expect("lookup by key"); + assert_eq!(found.key(), table.key()); + + // A namespace that does not resolve to a real mount yields nothing. + let bogus = NamespacePath::root() + .child(Identifier::for_test("outer")) + .child(Identifier::for_test("nope")); + assert!(::lookup(&def, (&bogus, &table.name)).is_none()); + } + + #[test] + fn validates_submodules_recursively() { + let mut submodule_builder = RawModuleDefV10Builder::new(); + submodule_builder + .build_table_with_new_type("Sessions", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + + let raw = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "authlib".to_string(), + module: submodule_builder.finish(), + }])], + }; + + let def: ModuleDef = raw.try_into().expect("submodule should validate"); + let submodules = def.submodules(); + + assert_eq!(submodules.len(), 1); + let submodule = submodules.get("authlib").expect("authlib submodule should exist"); + assert!(submodule.table(&expect_identifier("sessions")).is_some()); + } + + #[test] + fn invalid_submodule_namespace() { + let raw = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "".to_string(), + module: RawModuleDefV10::default(), + }])], + }; + + let result: Result = raw.try_into(); + + expect_error_matching!(result, ValidationError::IdentifierError { error } => { + error == &IdentifierError::Empty {} + }); + } + + #[test] + fn duplicate_submodule_namespace() { + let raw = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![ + RawSubmoduleV10 { + namespace: "authlib".to_string(), + module: RawModuleDefV10::default(), + }, + RawSubmoduleV10 { + namespace: "authlib".to_string(), + module: RawModuleDefV10::default(), + }, + ])], + }; + + let result: Result = raw.try_into(); + + expect_error_matching!(result, ValidationError::DuplicateName { name } => { + name == &RawIdentifier::from("authlib") + }); + } + #[test] fn invalid_unique_constraint_column_ref() { let mut builder = RawModuleDefV10Builder::new(); @@ -2191,7 +2380,7 @@ mod tests { def.reducers.contains_key(&do_delivery), "reducer 'do_delivery' not found" ); - assert_eq!(def.reducers[&do_delivery].name.as_identifier(), &do_delivery); + assert_eq!(def.reducers[&do_delivery].name.local(), &do_delivery); // "ProcessItem" (PascalCase) → "process_item" let process_item = id("process_item"); @@ -2199,7 +2388,7 @@ mod tests { def.reducers.contains_key(&process_item), "reducer 'process_item' not found" ); - assert_eq!(def.reducers[&process_item].name.as_identifier(), &process_item); + assert_eq!(def.reducers[&process_item].name.local(), &process_item); // ═══════════════════════════════════════════════════════════════════════════ // TYPE NAMES — PascalCase; scoped names keep their scope segments unchanged @@ -2460,7 +2649,7 @@ mod tests { !def.reducers.contains_key(&id("do_delivery")), "'do_delivery' must not exist when overridden" ); - assert_eq!(def.reducers[&deliver_ident].name.as_identifier(), &deliver_ident); + assert_eq!(def.reducers[&deliver_ident].name.local(), &deliver_ident); // Non-overridden reducer still follows SnakeCase. assert!(def.reducers.contains_key(&id("process_item"))); @@ -2516,4 +2705,94 @@ mod tests { assert_eq!(view.return_columns[0].view_name, id("Level2Person")); assert_eq!(view.param_columns[0].view_name, id("Level2Person")); } + + #[test] + fn namespace_exactly_63_chars_is_ok() { + let namespace = "a".repeat(63); + let raw = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace, + module: RawModuleDefV10::default(), + }])], + }; + let result: Result = raw.try_into(); + assert!(result.is_ok(), "63-char namespace should be valid"); + } + + #[test] + fn namespace_64_chars_is_rejected() { + let namespace = "a".repeat(64); + let raw = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: namespace.clone(), + module: RawModuleDefV10::default(), + }])], + }; + let expected_ns = RawIdentifier::from(namespace.clone()); + let result: Result = raw.try_into(); + expect_error_matching!(result, ValidationError::NamespaceTooLong { namespace: ns, len } => { + ns == &expected_ns && len == &64usize + }); + } + + fn make_module_with_lifecycle(lifecycle: Lifecycle) -> RawModuleDefV10 { + let mut b = RawModuleDefV10Builder::new(); + b.add_lifecycle_reducer(lifecycle, "lifecycle_fn", ProductType::unit()); + b.finish() + } + + #[test] + fn lifecycle_in_submodule_is_rejected() { + let raw = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "auth".to_string(), + module: make_module_with_lifecycle(Lifecycle::Init), + }])], + }; + + let result: Result = raw.try_into(); + expect_error_matching!(result, ValidationError::LifecycleInSubmodule { lifecycle, namespace } => { + lifecycle == &Lifecycle::Init && namespace == "auth" + }); + } + + #[test] + fn lifecycle_in_root_with_submodule_is_ok() { + // Root declares Init; the submodule has no lifecycle — this is valid. + let consumer_raw = make_module_with_lifecycle(Lifecycle::Init); + let mut sections = consumer_raw.sections; + sections.push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "auth".to_string(), + module: RawModuleDefV10::default(), + }])); + + let result: Result = RawModuleDefV10 { sections }.try_into(); + assert!( + result.is_ok(), + "lifecycle in root with a lifecycle-free submodule should be valid" + ); + } + + #[test] + fn lifecycle_in_nested_submodule_is_rejected() { + // Root uses auth as submodule; auth uses baz as submodule; baz declares a lifecycle. Should be rejected. + let auth = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "baz".to_string(), + module: make_module_with_lifecycle(Lifecycle::Init), + }])], + }; + + let raw = RawModuleDefV10 { + sections: vec![RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "auth".to_string(), + module: auth, + }])], + }; + + let result: Result = raw.try_into(); + expect_error_matching!(result, ValidationError::LifecycleInSubmodule { lifecycle, namespace } => { + lifecycle == &Lifecycle::Init && namespace == "baz" + }); + } } diff --git a/crates/schema/src/def/validate/v9.rs b/crates/schema/src/def/validate/v9.rs index 618f8e3c9c4..8de4927afa2 100644 --- a/crates/schema/src/def/validate/v9.rs +++ b/crates/schema/src/def/validate/v9.rs @@ -153,7 +153,9 @@ pub fn validate(def: RawModuleDefV9) -> Result { let typespace_for_generate = typespace_for_generate.finish(); - Ok(ModuleDef { + let mut module_def = ModuleDef { + // V9 has no submodules, so every def is at the root. + path: NamespacePath::root(), tables, reducers, views, @@ -168,7 +170,14 @@ pub fn validate(def: RawModuleDefV9) -> Result { http_handlers: IndexMap::new(), http_routes: Vec::new(), raw_module_def_version: RawModuleDefVersion::V9OrEarlier, - }) + submodules: IndexMap::new(), + }; + + // Records each def's namespace. V9 has no submodules, so this just resolves everything at + // the root, but the defs still need their `namespace` populated for keys to work. + module_def.apply_namespace(&NamespacePath::root()); + + Ok(module_def) } struct ModuleValidatorV9<'a> { @@ -319,6 +328,8 @@ impl ModuleValidatorV9<'_> { .combine_errors()?; Ok(TableDef { + // Set by `ModuleDef::apply_namespace` once the module tree is assembled. + namespace: NamespacePath::root(), name: name.clone(), product_type_ref, primary_key, @@ -519,6 +530,7 @@ impl ModuleValidatorV9<'_> { (name, return_type_for_generate, return_columns, param_columns).combine_errors()?; Ok(ViewDef { + namespace: NamespacePath::root(), name: name.clone(), is_anonymous, is_public, @@ -914,8 +926,11 @@ impl CoreValidator<'_> { let (_, (at_column, id_column), function_name) = (name_res, at_id, function_name).combine_errors()?; + let name = Identifier::new(name).map_err(|error| ValidationError::IdentifierError { error })?; Ok(ScheduleDef { - name: Identifier::new(name).map_err(|error| ValidationError::IdentifierError { error })?, + // Set by `ModuleDef::apply_namespace` once the module tree is assembled. + namespace: NamespacePath::root(), + name, at_column, id_column, function_name, @@ -1166,6 +1181,8 @@ impl<'a, 'b> TableValidator<'a, 'b> { let (name, column, (min_value, start, max_value)) = (name, column, min_start_max).combine_errors()?; Ok(SequenceDef { + // Set by `ModuleDef::apply_namespace` once the module tree is assembled. + namespace: NamespacePath::root(), name, column, min_value, @@ -1195,6 +1212,8 @@ impl<'a, 'b> TableValidator<'a, 'b> { .transpose()?; Ok(IndexDef { + // Set by `ModuleDef::apply_namespace` once the module tree is assembled. + namespace: NamespacePath::root(), name: name.clone(), accessor_name: codegen_name, source_name: name, @@ -1234,6 +1253,8 @@ impl<'a, 'b> TableValidator<'a, 'b> { let algorithm = self.validate_algorithm(&name, algorithm_raw.clone())?; Ok(IndexDef { + // Set by `ModuleDef::apply_namespace` once the module tree is assembled. + namespace: NamespacePath::root(), name: name.clone(), accessor_name: accessor_name.map(identifier).transpose()?, source_name, @@ -1302,6 +1323,8 @@ impl<'a, 'b> TableValidator<'a, 'b> { let (name, columns) = (name, columns).combine_errors()?; let columns: ColSet = columns.into(); Ok(ConstraintDef { + // Stamped by `ModuleDef::apply_namespace` once the module tree is assembled. + namespace: NamespacePath::root(), name, data: ConstraintData::Unique(UniqueConstraintData { columns }), }) @@ -1514,7 +1537,7 @@ pub(crate) fn check_scheduled_functions_exist( if let Some(schedule) = &mut table.schedule { if let Some(reducer) = reducers.get(&schedule.function_name) { schedule.function_kind = FunctionKind::Reducer; - validate_params(&reducer.params, table.product_type_ref, reducer.name.clone().into()) + validate_params(&reducer.params, table.product_type_ref, reducer.name.local().clone()) .map_err(Into::into) } else if let Some(procedure) = procedures.get(&schedule.function_name) { schedule.function_kind = FunctionKind::Procedure; @@ -1649,6 +1672,7 @@ mod tests { SequenceDef, UniqueConstraintData, }; use crate::error::*; + use crate::identifier::NamespacePath; use crate::type_for_generate::ClientCodegenError; use itertools::Itertools; @@ -1802,18 +1826,24 @@ mod tests { [ &IndexDef { name: "Apples_count_idx_direct".into(), + + namespace: NamespacePath::root(), accessor_name: Some(expect_identifier("Apples_count_direct")), algorithm: DirectAlgorithm { column: 2.into() }.into(), source_name: "Apples_count_idx_direct".into(), }, &IndexDef { name: "Apples_name_count_idx_btree".into(), + + namespace: NamespacePath::root(), accessor_name: Some(expect_identifier("apples_id")), algorithm: BTreeAlgorithm { columns: [1, 2].into() }.into(), source_name: "Apples_name_count_idx_btree".into(), }, &IndexDef { name: "Apples_type_idx_btree".into(), + + namespace: NamespacePath::root(), accessor_name: Some(expect_identifier("Apples_type_btree")), algorithm: BTreeAlgorithm { columns: 3.into() }.into(), source_name: "Apples_type_idx_btree".into(), @@ -2393,9 +2423,25 @@ mod tests { raw_def.tables[0].sequences[0].name = Some("wacky.sequence()".into()); let def: ModuleDef = raw_def.try_into().unwrap(); - assert!(def.lookup::(&"wacky.constraint()".into()).is_some()); - assert!(def.lookup::(&"wacky.index()".into()).is_some()); - assert!(def.lookup::(&"wacky.sequence()".into()).is_some()); + // Sub-object names may themselves contain dots, which is exactly why their keys are + // (namespace, local name) pairs rather than a dot-joined name. + let root = NamespacePath::root(); + assert!(def + .lookup::((&root, &"wacky.constraint()".into())) + .is_some()); + assert!(def.lookup::((&root, &"wacky.index()".into())).is_some()); + assert!(def.lookup::((&root, &"wacky.sequence()".into())).is_some()); + + // The same hazard bites when recovering a local name from a stored one. Splitting on + // the last `.` turns `"wacky.index()"` into `"index()"`, which matches no def, so + // republishing such a module fails with "Index 0 not found in definition". The local + // name must be recovered by stripping the known namespace prefix instead. + use crate::schema::{Schema, TableSchema}; + let table_def = def.table("Deliveries").expect("table should exist"); + let schema = TableSchema::from_module_def(&def, table_def, (), spacetimedb_primitives::TableId::SENTINEL); + schema + .check_compatible(&def, table_def) + .expect("dotted V9 sub-object names must not break compatibility checking"); } #[test] diff --git a/crates/schema/src/error.rs b/crates/schema/src/error.rs index 0bab5445d92..e9408a35482 100644 --- a/crates/schema/src/error.rs +++ b/crates/schema/src/error.rs @@ -176,6 +176,13 @@ pub enum ValidationError { ok_type: PrettyAlgebraicType, err_type: PrettyAlgebraicType, }, + #[error( + "lifecycle event {lifecycle:?} is not permitted in submodule under namespace `{namespace}`; \ + lifecycle reducers may only be declared in the root module" + )] + LifecycleInSubmodule { lifecycle: Lifecycle, namespace: String }, + #[error("submodule namespace `{namespace}` is {len} bytes, which exceeds the 63-byte limit")] + NamespaceTooLong { namespace: RawIdentifier, len: usize }, } /// A wrapper around an `AlgebraicType` that implements `fmt::Display`. diff --git a/crates/schema/src/identifier.rs b/crates/schema/src/identifier.rs index 06a396ec012..a45d96b3f76 100644 --- a/crates/schema/src/identifier.rs +++ b/crates/schema/src/identifier.rs @@ -1,6 +1,6 @@ use crate::error::IdentifierError; use spacetimedb_data_structures::map::{Equivalent, HashSet}; -use spacetimedb_sats::raw_identifier::RawIdentifier; +use spacetimedb_sats::raw_identifier::{RawIdentifier, RawNamespacedIdentifier}; use spacetimedb_sats::{impl_deserialize, impl_serialize, impl_st}; use std::fmt::{self, Debug, Display}; use std::ops::Deref; @@ -33,51 +33,65 @@ pub struct Identifier { impl_st!([] Identifier, ts => RawIdentifier::make_type(ts)); impl_serialize!([] Identifier, (self, ser) => ser.serialize_str(&self.id)); -impl_deserialize!([] Identifier, de => RawIdentifier::deserialize(de).map(Self::new_assume_valid)); +impl_deserialize!([] Identifier, de => RawIdentifier::deserialize(de).map(Self::new_unsafe_assume_valid)); -impl Identifier { - /// Returns a new identifier without validating the input. - pub fn new_assume_valid(name: RawIdentifier) -> Self { - Self { id: name } +/// Validates that `name` is a valid identifier string +pub fn validate_identifier(name: &str) -> Result<(), IdentifierError> { + if name.is_empty() { + return Err(IdentifierError::Empty {}); } - /// Validates that the input string is a valid identifier. - /// - /// Currently, this rejects non-canonicalized identifiers. - /// Eventually, it will be changed to canonicalize the input string. - pub fn new(name: RawIdentifier) -> Result { - if name.is_empty() { - return Err(IdentifierError::Empty {}); - } + // Convert to Unicode Normalization Form C (canonical decomposition followed by composition). + if name.nfc().zip(name.chars()).any(|(a, b)| a != b) { + return Err(IdentifierError::NotCanonicalized { + name: RawIdentifier::new(name), + }); + } - // Convert to Unicode Normalization Form C (canonical decomposition followed by composition). - if name.nfc().zip(name.chars()).any(|(a, b)| a != b) { - return Err(IdentifierError::NotCanonicalized { name }); - } + let mut chars = name.chars(); - let mut chars = name.chars(); + let start = chars.next().ok_or(IdentifierError::Empty {})?; + if !is_xid_start(start) && start != '_' { + return Err(IdentifierError::InvalidStart { + name: RawIdentifier::new(name), + invalid_start: start, + }); + } - let start = chars.next().ok_or(IdentifierError::Empty {})?; - if !is_xid_start(start) && start != '_' { - return Err(IdentifierError::InvalidStart { - name, - invalid_start: start, + for char_ in chars { + if !is_xid_continue(char_) { + return Err(IdentifierError::InvalidContinue { + name: RawIdentifier::new(name), + invalid_continue: char_, }); } + } - for char_ in chars { - if !is_xid_continue(char_) { - return Err(IdentifierError::InvalidContinue { - name, - invalid_continue: char_, - }); - } - } + if Identifier::is_reserved(name) { + return Err(IdentifierError::Reserved { + name: RawIdentifier::new(name), + }); + } - if Identifier::is_reserved(&name) { - return Err(IdentifierError::Reserved { name }); - } + Ok(()) +} +impl Identifier { + /// Returns a new identifier **without validating the input**. + /// + /// Prefer [`Identifier::new`]. This exists only for names that were already + /// validated on the way in and are being reconstructed from storage or from + /// the wire; using it anywhere else defeats the point of the type. + pub fn new_unsafe_assume_valid(name: RawIdentifier) -> Self { + Self { id: name } + } + + /// Validates that the input string is a valid identifier. + /// + /// Currently, this rejects non-canonicalized identifiers. + /// Eventually, it will be changed to canonicalize the input string. + pub fn new(name: RawIdentifier) -> Result { + validate_identifier(&name)?; Ok(Identifier { id: name }) } @@ -122,12 +136,256 @@ impl Equivalent for str { } } +impl PartialEq for Identifier { + fn eq(&self, other: &str) -> bool { + &self.id[..] == other + } +} + impl From for RawIdentifier { fn from(id: Identifier) -> Self { id.id } } +/// A non-empty, dot-separated sequence of validated [`Identifier`] segments. +/// +/// Used for fully-qualified names of submodule items, e.g.: +/// - `"lib.library_table"` (table name) +/// - `"lib.library_table_id_idx_btree"` (index name) +/// +/// Root-level items have a single segment (e.g., `"user"`). +/// Constructed only from already-validated [`Identifier`]s. +#[derive(Clone)] +pub struct NamespacedIdentifier { + /// always non-empty. + segments: Box<[Identifier]>, + /// Cached dot-joined rendering of `segments`. + joined: Box, +} + +impl_st!([] NamespacedIdentifier, ts => RawNamespacedIdentifier::make_type(ts)); +impl_serialize!([] NamespacedIdentifier, (self, ser) => ser.serialize_str(&self.joined)); +// Stored names were validated when created; segments are trusted here, +// mirroring `Identifier`'s own deserialization. +impl_deserialize!([] NamespacedIdentifier, de => + RawNamespacedIdentifier::deserialize(de).map(|raw| Self::new_unsafe_assume_valid(&raw))); + +impl NamespacedIdentifier { + /// Construct from validated segments. Panics if `segments` is empty. + pub fn from_segments(segments: Vec) -> Self { + assert!( + !segments.is_empty(), + "NamespacedIdentifier must have at least one segment" + ); + let joined = segments.iter().map(|s| &**s).collect::>().join(".").into(); + Self { + segments: segments.into(), + joined, + } + } + + /// Validates each dot-separated segment of `name` as an [`Identifier`]. + pub fn new(name: &RawNamespacedIdentifier) -> Result { + let segments = name + .segments() + .map(|segment| Identifier::new(RawIdentifier::new(segment))) + .collect::, _>>()?; + Ok(Self::from_segments(segments)) + } + + /// Splits `name` on `.` **without validating** the resulting segments. + /// + /// Prefer [`NamespacedIdentifier::new`]. This is the multi-segment counterpart + /// of [`Identifier::new_unsafe_assume_valid`] and carries the same caveat: it is + /// only for names that were validated on the way in and are being reconstructed + /// from storage or from the wire. + pub fn new_unsafe_assume_valid(name: &RawNamespacedIdentifier) -> Self { + Self::from_segments( + name.segments() + .map(|segment| Identifier::new_unsafe_assume_valid(RawIdentifier::new(segment))) + .collect(), + ) + } + + pub fn segments(&self) -> &[Identifier] { + &self.segments + } + + /// The final segment, i.e. the name with any namespace prefix stripped. + pub fn local_name(&self) -> &Identifier { + self.segments.last().expect("NamespacedIdentifier is never empty") + } + + /// Whether this name carries a namespace prefix. + pub fn is_namespaced(&self) -> bool { + self.segments.len() > 1 + } + + /// This name as a single [`Identifier`], or `None` if it carries a namespace. + pub fn as_identifier(&self) -> Option<&Identifier> { + match &*self.segments { + [single] => Some(single), + _ => None, + } + } +} + +impl From for NamespacedIdentifier { + fn from(id: Identifier) -> Self { + Self::from_segments(vec![id]) + } +} + +impl FromIterator for NamespacedIdentifier { + /// Panics if the iterator is empty. + fn from_iter>(iter: I) -> Self { + Self::from_segments(iter.into_iter().collect()) + } +} + +impl From for RawNamespacedIdentifier { + fn from(id: NamespacedIdentifier) -> Self { + RawNamespacedIdentifier::new(&*id) + } +} + +// Comparisons and hashing use the joined form. This is equivalent to comparing +// segment-wise, since '.' orders below every character valid in an identifier. +impl PartialEq for NamespacedIdentifier { + fn eq(&self, other: &Self) -> bool { + self.joined == other.joined + } +} +impl Eq for NamespacedIdentifier {} +impl PartialOrd for NamespacedIdentifier { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} +impl Ord for NamespacedIdentifier { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.joined.cmp(&other.joined) + } +} +impl std::hash::Hash for NamespacedIdentifier { + fn hash(&self, state: &mut H) { + self.joined.hash(state) + } +} + +impl std::fmt::Debug for NamespacedIdentifier { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", &*self.joined) + } +} + +impl std::fmt::Display for NamespacedIdentifier { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.joined) + } +} + +impl std::ops::Deref for NamespacedIdentifier { + type Target = str; + fn deref(&self) -> &str { + &self.joined + } +} + +impl AsRef for NamespacedIdentifier { + fn as_ref(&self) -> &str { + &self.joined + } +} + +/// A possibly-empty path of validated namespace [`Identifier`]s. +/// +/// Displays as a dot-terminated prefix (`"lib."`, `"auth.baz."`, or `""` for the root), +/// matching how namespaced names are built by prepending the prefix to a local name. +#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub struct NamespacePath(Vec); + +impl NamespacePath { + /// The root (empty) path. + pub fn root() -> Self { + Self(Vec::new()) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn segments(&self) -> &[Identifier] { + &self.0 + } + + /// This path extended with the namespace `ns`. + pub fn child(&self, ns: Identifier) -> Self { + let mut segments = self.0.clone(); + segments.push(ns); + Self(segments) + } + + /// The full name of an item named `last` under this path. + pub fn join(&self, last: Identifier) -> NamespacedIdentifier { + let mut segments = self.0.clone(); + segments.push(last); + NamespacedIdentifier::from_segments(segments) + } + + /// `name`, prefixed with this namespace, preserving `name`'s own segments. + pub fn join_namespaced(&self, name: &NamespacedIdentifier) -> NamespacedIdentifier { + let mut segments = self.0.clone(); + segments.extend(name.segments().iter().cloned()); + NamespacedIdentifier::from_segments(segments) + } + + /// `name`, prefixed with this namespace. + /// + /// Unlike [`NamespacePath::join`], this takes an already-raw name. Index, + /// constraint and sequence names are system-generated and stored raw, so they + /// have no validated `Identifier` to join onto. + pub fn join_raw(&self, name: &RawNamespacedIdentifier) -> RawNamespacedIdentifier { + if self.is_empty() { + return name.clone(); + } + RawNamespacedIdentifier::new(format!("{self}{name}")) + } + + /// `name` with this namespace's `"a.b."` prefix removed, or `None` if `name` does not + /// carry it. + /// + /// This is the inverse of [`NamespacePath::join`] and friends, and is the correct way + /// to recover a local name from a stored one. Splitting on the last `.` is *not*: a V9 + /// index, constraint or sequence name may itself contain dots (see the `wacky_names` + /// test), so the final dot-separated segment is not necessarily the local name. Only + /// the prefix this path actually knows about is removed. + /// + /// Borrows rather than allocating, so it is cheap enough to call per sub-object. + pub fn strip_from<'a>(&self, name: &'a str) -> Option<&'a str> { + let mut rest = name; + for segment in &self.0 { + rest = rest.strip_prefix(&**segment)?.strip_prefix('.')?; + } + Some(rest) + } + + /// The segments joined with `sep` (e.g. `"/"` for a directory path). + pub fn join_segments(&self, sep: &str) -> String { + self.0.iter().map(|s| &**s).collect::>().join(sep) + } +} + +impl std::fmt::Display for NamespacePath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for segment in &self.0 { + write!(f, "{segment}.")?; + } + Ok(()) + } +} + #[cfg(test)] mod tests { use super::*; @@ -137,6 +395,31 @@ mod tests { Identifier::new(RawIdentifier::new(s)) } + #[test] + fn strip_from_removes_only_the_known_prefix() { + let root = NamespacePath::root(); + let lib = root.child(Identifier::for_test("lib")); + let nested = lib.child(Identifier::for_test("auth")); + + // The root strips nothing, so a name comes back whole -- dots and all. + assert_eq!(root.strip_from("my_table"), Some("my_table")); + assert_eq!(root.strip_from("wacky.index()"), Some("wacky.index()")); + + assert_eq!(lib.strip_from("lib.my_table"), Some("my_table")); + assert_eq!(nested.strip_from("lib.auth.my_table"), Some("my_table")); + + // Only the prefix this path knows about is removed: the remainder keeps its own + // dots, which is the whole point -- V9 sub-object names may contain them. + assert_eq!(lib.strip_from("lib.wacky.index()"), Some("wacky.index()")); + + // A name outside this namespace is not in it. + assert_eq!(lib.strip_from("my_table"), None); + assert_eq!(lib.strip_from("other.my_table"), None); + assert_eq!(nested.strip_from("lib.my_table"), None); + // A partial segment match is not a prefix match. + assert_eq!(lib.strip_from("library.my_table"), None); + } + #[test] fn test_a_bunch_of_identifiers() { assert!(new("friends").is_ok()); diff --git a/crates/schema/src/reducer_name.rs b/crates/schema/src/reducer_name.rs index 94158f9d4e3..f0935433db3 100644 --- a/crates/schema/src/reducer_name.rs +++ b/crates/schema/src/reducer_name.rs @@ -1,22 +1,31 @@ -use crate::identifier::Identifier; +use crate::identifier::{Identifier, NamespacedIdentifier}; use core::fmt; use core::ops::Deref; -use spacetimedb_sats::raw_identifier::RawIdentifier; +use spacetimedb_sats::raw_identifier::{RawIdentifier, RawNamespacedIdentifier}; /// The name of a reducer. +/// +/// Fully qualified: a reducer in a submodule mounted as `myauth` is named +/// `myauth.verify_token`, which is how clients address it. Use [`ReducerName::local`] +/// for the name within its own module. #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct ReducerName(pub Identifier); +pub struct ReducerName(pub NamespacedIdentifier); impl ReducerName { - pub fn new(id: Identifier) -> Self { - Self(id) + pub fn new(id: impl Into) -> Self { + Self(id.into()) } pub fn for_test(name: &str) -> Self { - Self(Identifier::for_test(name)) + Self(name.split('.').map(Identifier::for_test).collect()) } - pub fn as_identifier(&self) -> &Identifier { + /// The reducer's name within its own module, without the namespace it is mounted under. + pub fn local(&self) -> &Identifier { + self.0.local_name() + } + + pub fn as_namespaced(&self) -> &NamespacedIdentifier { &self.0 } } @@ -35,15 +44,22 @@ impl AsRef for ReducerName { } } -impl From for Identifier { +impl From for NamespacedIdentifier { fn from(id: ReducerName) -> Self { id.0 } } +impl From for RawNamespacedIdentifier { + fn from(id: ReducerName) -> Self { + id.0.into() + } +} + impl From for RawIdentifier { + /// Flattens the qualified name; only for the wire types that carry it as a plain string. fn from(id: ReducerName) -> Self { - Identifier::from(id).into() + RawIdentifier::new(&*id.0) } } diff --git a/crates/schema/src/schema.rs b/crates/schema/src/schema.rs index 4079d98a70a..db854fab15e 100644 --- a/crates/schema/src/schema.rs +++ b/crates/schema/src/schema.rs @@ -16,7 +16,7 @@ use spacetimedb_lib::db::raw_def::v9::RawSql; use spacetimedb_lib::db::raw_def::{generate_cols_name, RawConstraintDefV8}; use spacetimedb_primitives::*; use spacetimedb_sats::product_value::InvalidFieldError; -use spacetimedb_sats::raw_identifier::RawIdentifier; +use spacetimedb_sats::raw_identifier::{RawIdentifier, RawNamespacedIdentifier}; use spacetimedb_sats::{AlgebraicType, ProductType, ProductTypeElement, WithTypespace}; use std::collections::BTreeMap; use std::sync::Arc; @@ -25,7 +25,19 @@ use crate::def::{ ColumnDef, ConstraintData, ConstraintDef, IndexAlgorithm, IndexDef, ModuleDef, ModuleDefLookup, RawModuleDefVersion, ScheduleDef, SequenceDef, TableDef, UniqueConstraintData, ViewColumnDef, ViewDef, }; -use crate::identifier::Identifier; +use crate::identifier::{Identifier, NamespacePath, NamespacedIdentifier}; + +/// The local part of a stored, possibly namespaced name. +/// +/// Names in the database carry the namespace of the module that owns them, so recovering the +/// local name means removing exactly that prefix. Splitting on the last `.` is wrong: a V9 +/// index, constraint or sequence name may itself contain dots (see the `wacky_names` test), +/// in which case the final segment is not the local name and the comparison spuriously fails. +fn strip_namespace<'a>(stored: &'a str, namespace: &NamespacePath) -> anyhow::Result<&'a str> { + namespace + .strip_from(stored) + .ok_or_else(|| anyhow::anyhow!("Name `{stored}` is not in the expected namespace `{namespace}`")) +} /// Helper trait documenting allowing schema entities to be built from a validated `ModuleDef`. pub trait Schema: Sized { @@ -136,7 +148,9 @@ pub struct TableSchema { /// The name of the table. pub table_name: TableName, - pub alias: Option, + /// The source-level (accessor) name of the table, for client codegen. + /// Namespaced for submodule tables, matching `table_name`. + pub alias: Option, /// Is this the backing table of a view? pub view_info: Option, @@ -199,7 +213,7 @@ impl TableSchema { schedule: Option, primary_key: Option, is_event: bool, - alias: Option, + alias: Option, ) -> Self { Self { row_type: columns_to_row_type(&columns), @@ -233,7 +247,7 @@ impl TableSchema { col_name: element .name .clone() - .map(Identifier::new_assume_valid) + .map(Identifier::new_unsafe_assume_valid) .unwrap_or_else(|| Identifier::for_test(format!("col{col_pos}"))), col_type: element.algebraic_type.clone(), alias: None, @@ -663,12 +677,16 @@ impl TableSchema { /// This method works around this problem by copying the column types from the module def into the table schema. /// It can be removed once v8 is removed, since v9 will reject modules with an inconsistency like this. pub fn janky_fix_column_defs(&mut self, module_def: &ModuleDef) { - let table_name = self.table_name.clone().into(); + // `janky_fix_column_defs` only runs for v8 modules, which have no submodules, + // so the table is at the root. + let full_name: NamespacedIdentifier = self.table_name.clone().into(); + let local_name = full_name.local_name().clone(); for col in &mut self.columns { - let def: &ColumnDef = module_def.lookup((&table_name, &col.col_name)).unwrap(); + let def: &ColumnDef = module_def.lookup((&local_name, &col.col_name)).unwrap(); col.col_type = def.ty.clone(); } - let table_def: &TableDef = module_def.expect_lookup(&table_name); + let root = NamespacePath::root(); + let table_def: &TableDef = module_def.expect_lookup((&root, &local_name)); self.row_type = module_def.typespace()[table_def.product_type_ref] .as_product() .unwrap() @@ -826,7 +844,7 @@ impl TableSchema { columns.push(ColumnSchema { table_id: TableId::SENTINEL, col_pos: VIEW_ARG_HASH_COL, - col_name: Identifier::new_assume_valid("arg_hash".into()), + col_name: Identifier::new_unsafe_assume_valid("arg_hash".into()), col_type: AlgebraicType::U256, alias: None, }); @@ -854,7 +872,7 @@ impl TableSchema { let mut indexes = vec![IndexSchema { index_id: IndexId::SENTINEL, table_id: TableId::SENTINEL, - index_name: make_index_name(&arg_hash_cols), + index_name: make_index_name(&arg_hash_cols).into(), index_algorithm: IndexAlgorithm::BTree(arg_hash_cols.into()), alias: None, }]; @@ -869,7 +887,7 @@ impl TableSchema { constraints.push(ConstraintSchema { table_id: TableId::SENTINEL, constraint_id: ConstraintId::SENTINEL, - constraint_name: make_constraint_name(&cols), + constraint_name: make_constraint_name(&cols).into(), data: ConstraintData::Unique(UniqueConstraintData { columns: ColSet::from(cols.clone()), }), @@ -877,7 +895,7 @@ impl TableSchema { indexes.push(IndexSchema { index_id: IndexId::SENTINEL, table_id: TableId::SENTINEL, - index_name: make_index_name(&cols), + index_name: make_index_name(&cols).into(), index_algorithm: IndexAlgorithm::BTree(cols.into()), alias: None, }); @@ -907,7 +925,7 @@ impl TableSchema { None, None, false, - Some(accessor_name.clone()), + Some(accessor_name.clone().into()), ) } } @@ -978,12 +996,19 @@ impl Schema for TableSchema { schedule, *primary_key, *is_event, - Some(accessor_name.clone()), + Some(accessor_name.clone().into()), ) } fn check_compatible(&self, module_def: &ModuleDef, def: &Self::Def) -> Result<(), anyhow::Error> { - ensure_eq!(&self.table_name[..], &def.name[..], "Table name mismatch"); + // Both root and submodule tables are stored under their canonical name, qualified by + // the namespace their module is mounted under. Check the whole thing, not just the + // last segment: a table in `lib` must not validate against a root def of the same name. + ensure_eq!( + strip_namespace(&self.table_name, &def.namespace)?, + &def.name[..], + "Table name mismatch" + ); ensure_eq!(self.primary_key, def.primary_key, "Primary key mismatch"); let def_table_access: StAccess = (def.table_access).into(); ensure_eq!(self.table_access, def_table_access, "Table access mismatch"); @@ -999,19 +1024,22 @@ impl Schema for TableSchema { } ensure_eq!(self.columns.len(), def.columns.len(), "Column count mismatch"); + // Index names in the DB are prefixed for submodule tables (e.g. "lib.library_table_id_idx_btree"), + // but `def.indexes` is keyed by the local name. for index in &self.indexes { let index_def = def .indexes - .get(&index.index_name) + .get(strip_namespace(&index.index_name, &def.namespace)?) .ok_or_else(|| anyhow::anyhow!("Index {} not found in definition", index.index_id.0))?; index.check_compatible(module_def, index_def)?; } ensure_eq!(self.indexes.len(), def.indexes.len(), "Index count mismatch"); + // Like index names, constraint names are namespaced in the DB for submodule tables. for constraint in &self.constraints { let constraint_def = def .constraints - .get(&constraint.constraint_name) + .get(strip_namespace(&constraint.constraint_name, &def.namespace)?) .ok_or_else(|| anyhow::anyhow!("Constraint {} not found in definition", constraint.constraint_id.0))?; constraint.check_compatible(module_def, constraint_def)?; } @@ -1021,10 +1049,11 @@ impl Schema for TableSchema { "Constraint count mismatch" ); + // Like index names, sequence names are namespaced in the DB for submodule tables. for sequence in &self.sequences { let sequence_def = def .sequences - .get(&sequence.sequence_name) + .get(strip_namespace(&sequence.sequence_name, &def.namespace)?) .ok_or_else(|| anyhow::anyhow!("Sequence {} not found in definition", sequence.sequence_id.0))?; sequence.check_compatible(module_def, sequence_def)?; } @@ -1220,9 +1249,9 @@ impl From> for ProductTypeElement { pub struct SequenceSchema { /// The unique identifier for the sequence within a database. pub sequence_id: SequenceId, - /// The name of the sequence. + /// The name of the sequence. Namespaced for submodule tables. /// Deprecated. In the future, sequences will be identified by col_pos. - pub sequence_name: RawIdentifier, + pub sequence_name: RawNamespacedIdentifier, /// The ID of the table associated with the sequence. pub table_id: TableId, /// The position of the column associated with this sequence. @@ -1270,7 +1299,7 @@ impl Schema for SequenceSchema { SequenceSchema { sequence_id: id, - sequence_name: def.name.clone(), + sequence_name: def.name.clone().into(), table_id: parent_id, col_pos: def.column, increment: def.increment, @@ -1282,7 +1311,12 @@ impl Schema for SequenceSchema { } fn check_compatible(&self, _module_def: &ModuleDef, def: &Self::Def) -> Result<(), anyhow::Error> { - ensure_eq!(&self.sequence_name[..], &def.name[..], "Sequence name mismatch"); + // Sequence names are namespaced in the DB for submodule tables; def.name is local. + ensure_eq!( + strip_namespace(&self.sequence_name, &def.namespace)?, + &def.name[..], + "Sequence name mismatch" + ); ensure_eq!(self.col_pos, def.column, "Sequence column mismatch"); ensure_eq!(self.increment, def.increment, "Sequence increment mismatch"); if let Some(start) = &def.start { @@ -1311,7 +1345,10 @@ pub struct ScheduleSchema { pub schedule_name: Identifier, /// The name of the reducer or procedure to call. - pub function_name: Identifier, + /// + /// Namespaced for submodule tables (e.g. `"lib.library_scheduled_procedure"`), + /// since that is how the scheduler resolves it. + pub function_name: NamespacedIdentifier, /// The column containing the `ScheduleAt` enum. pub at_column: ColId, @@ -1324,7 +1361,7 @@ impl ScheduleSchema { table_id: TableId::SENTINEL, schedule_id: ScheduleId::SENTINEL, schedule_name: Identifier::for_test(name.as_ref()), - function_name: Identifier::for_test(function.as_ref()), + function_name: Identifier::for_test(function.as_ref()).into(), at_column: at.into(), } } @@ -1344,7 +1381,7 @@ impl Schema for ScheduleSchema { table_id: parent_id, schedule_id: id, schedule_name: def.name.clone(), - function_name: def.function_name.clone(), + function_name: def.function_name.clone().into(), at_column: def.at_column, // Ignore def.at_column and id_column. Those are recovered at runtime. } @@ -1352,8 +1389,10 @@ impl Schema for ScheduleSchema { fn check_compatible(&self, _module_def: &ModuleDef, def: &Self::Def) -> Result<(), anyhow::Error> { ensure_eq!(&self.schedule_name[..], &def.name[..], "Schedule name mismatch"); + // For submodule tables the stored function name is namespaced + // (e.g. `"lib.library_scheduled_procedure"`) while `def.function_name` is local. ensure_eq!( - &self.function_name[..], + strip_namespace(&self.function_name, &def.namespace)?, &def.function_name[..], "Schedule function name mismatch" ); @@ -1369,10 +1408,10 @@ pub struct IndexSchema { /// The ID of the table associated with the index. pub table_id: TableId, /// The name of the index. This should not be assumed to follow any particular format. - /// Unique within the database. - pub index_name: RawIdentifier, + /// Unique within the database. Namespaced for submodule tables. + pub index_name: RawNamespacedIdentifier, - pub alias: Option, + pub alias: Option, /// The data for the schema. pub index_algorithm: IndexAlgorithm, } @@ -1395,7 +1434,7 @@ impl IndexSchema { Self { index_id: IndexId::SENTINEL, table_id: TableId::SENTINEL, - index_name: RawIdentifier::new(name.as_ref()), + index_name: RawNamespacedIdentifier::new(name.as_ref()), index_algorithm: algo.into(), alias: None, } @@ -1414,14 +1453,19 @@ impl Schema for IndexSchema { IndexSchema { index_id: id, table_id: parent_id, - index_name: def.name.clone(), + index_name: def.name.clone().into(), index_algorithm, - alias: Some(def.source_name.clone()), + alias: Some(def.source_name.clone().into()), } } fn check_compatible(&self, _module_def: &ModuleDef, def: &Self::Def) -> Result<(), anyhow::Error> { - ensure_eq!(&self.index_name[..], &def.name[..], "Index name mismatch"); + // Index names are namespaced in the DB for submodule tables; def.name is local. + ensure_eq!( + strip_namespace(&self.index_name, &def.namespace)?, + &def.name[..], + "Index name mismatch" + ); ensure_eq!(&self.index_algorithm, &def.algorithm, "Index algorithm mismatch"); Ok(()) } @@ -1438,7 +1482,7 @@ pub struct ConstraintSchema { /// The unique ID of the constraint within the database. pub constraint_id: ConstraintId, /// The name of the constraint. - pub constraint_name: RawIdentifier, + pub constraint_name: RawNamespacedIdentifier, /// The data for the constraint. pub data: ConstraintData, // this reuses the type from Def, which is fine, neither of `schema` nor `def` are ABI modules. } @@ -1460,7 +1504,7 @@ impl ConstraintSchema { Self { table_id: TableId::SENTINEL, constraint_id: ConstraintId::SENTINEL, - constraint_name: RawIdentifier::new(name.as_ref()), + constraint_name: RawNamespacedIdentifier::new(name.as_ref()), data: ConstraintData::Unique(UniqueConstraintData { columns: cols.into() }), } } @@ -1476,7 +1520,7 @@ impl ConstraintSchema { if constraint.constraints.has_unique() { Some(ConstraintSchema { constraint_id: ConstraintId::SENTINEL, // Set to 0 as it may be assigned later. - constraint_name: RawIdentifier::new(constraint.constraint_name.trim()), + constraint_name: RawNamespacedIdentifier::new(constraint.constraint_name.trim()), table_id, data: ConstraintData::Unique(UniqueConstraintData { columns: constraint.columns.into(), @@ -1498,14 +1542,19 @@ impl Schema for ConstraintSchema { ConstraintSchema { constraint_id: id, - constraint_name: def.name.clone(), + constraint_name: def.name.clone().into(), table_id: parent_id, data: def.data.clone(), } } fn check_compatible(&self, _module_def: &ModuleDef, def: &Self::Def) -> Result<(), anyhow::Error> { - ensure_eq!(&self.constraint_name[..], &def.name[..], "Constraint name mismatch"); + // Constraint names are namespaced in the DB for submodule tables; def.name is local. + ensure_eq!( + strip_namespace(&self.constraint_name, &def.namespace)?, + &def.name[..], + "Constraint name mismatch" + ); ensure_eq!(&self.data, &def.data, "Constraint data mismatch"); Ok(()) } @@ -1517,3 +1566,95 @@ pub struct RowLevelSecuritySchema { pub table_id: TableId, pub sql: RawSql, } + +#[cfg(test)] +mod tests { + use super::*; + use spacetimedb_lib::db::raw_def::v10::{RawModuleDefV10Builder, RawModuleDefV10Section, RawSubmoduleV10}; + use spacetimedb_sats::AlgebraicType; + + fn module_with_table() -> ModuleDef { + let mut builder = RawModuleDefV10Builder::new(); + builder + .build_table_with_new_type("MyTable", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + builder + .finish() + .try_into() + .expect("should be a valid module definition") + } + + /// The same table, mounted in a `lib` submodule. + fn module_with_submodule_table() -> ModuleDef { + let mut sub = RawModuleDefV10Builder::new(); + sub.build_table_with_new_type("MyTable", ProductType::from([("id", AlgebraicType::U64)]), true) + .finish(); + let mut root = RawModuleDefV10Builder::new().finish(); + root.sections + .push(RawModuleDefV10Section::Submodules(vec![RawSubmoduleV10 { + namespace: "lib".to_string(), + module: sub.finish(), + }])); + root.try_into().expect("should be a valid module definition") + } + + #[test] + fn check_compatible_table_name() { + let def = module_with_table(); + let table_def = def.tables().next().expect("table should exist"); + assert_eq!(&table_def.name[..], "my_table"); + assert_eq!(&table_def.accessor_name[..], "MyTable"); + + let mut schema = TableSchema::from_module_def(&def, table_def, (), TableId::SENTINEL); + assert!(schema.check_compatible(&def, table_def).is_ok()); + + // A root table must match the def's canonical name exactly. + schema.table_name = TableName::for_test("MyTable"); + assert!(schema.check_compatible(&def, table_def).is_err()); + schema.table_name = TableName::for_test("other"); + assert!(schema.check_compatible(&def, table_def).is_err()); + + // A namespaced name does not match a def mounted at the root. + schema.table_name = TableName::for_test("lib.my_table"); + assert!(schema.check_compatible(&def, table_def).is_err()); + } + + /// A submodule table is stored under its namespace-qualified canonical name, and must be + /// checked against the namespace its def is actually mounted under. + #[test] + fn check_compatible_submodule_table_name() { + let def = module_with_submodule_table(); + let (prefix, owning, table_def) = def + .all_tables_with_prefix() + .into_iter() + .next() + .expect("submodule table should exist"); + assert_eq!(&table_def.name[..], "my_table"); + assert_eq!(&table_def.accessor_name[..], "MyTable"); + + // `from_module_def` yields local names; the engine qualifies them when it creates the + // table (see `create_table_from_def_with_prefix`), so do the same here. + let mut schema = TableSchema::from_module_def(owning, table_def, (), TableId::SENTINEL); + schema.table_name = TableName::from(prefix.join(table_def.name.clone())); + for index in &mut schema.indexes { + index.index_name = prefix.join_raw(&index.index_name); + } + for constraint in &mut schema.constraints { + constraint.constraint_name = prefix.join_raw(&constraint.constraint_name); + } + for sequence in &mut schema.sequences { + sequence.sequence_name = prefix.join_raw(&sequence.sequence_name); + } + assert!(schema.check_compatible(owning, table_def).is_ok()); + + // The accessor name is an alias, never an identity. + schema.table_name = TableName::for_test("lib.MyTable"); + assert!(schema.check_compatible(owning, table_def).is_err()); + // Nor does the bare canonical name match a def mounted in `lib`. + schema.table_name = TableName::for_test("my_table"); + assert!(schema.check_compatible(owning, table_def).is_err()); + // Nor a different namespace. + schema.table_name = TableName::for_test("other.my_table"); + assert!(schema.check_compatible(owning, table_def).is_err()); + } +} diff --git a/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print no color.snap b/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print no color.snap index bb6cbc98ef0..dea71a64749 100644 --- a/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print no color.snap +++ b/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print no color.snap @@ -9,7 +9,7 @@ Database Migration Plan ▸ Removed index Apples_id_name_idx_btree on [id, name] of table Apples ▸ Removed unique constraint Apples_id_key on [id] of table Apples ▸ Removed auto-increment constraint Apples_id_seq on column id of table Apples -▸ Removed schedule for table Deliveries_sched calling reducer check_deliveries +▸ Removed schedule for table Deliveries calling reducer check_deliveries ▸ ▸ Removed anonymous view: my_view Parameters: • x: U32 @@ -38,7 +38,7 @@ Database Migration Plan ▸ Created index Apples_id_count_idx_btree on [id, count] of table Apples ▸ Created auto-increment constraint Bananas_id_seq on column id of table Bananas -▸ Created schedule for table Inspections_sched calling reducer perform_inspection +▸ Created schedule for table Inspections calling reducer perform_inspection ▸ ▸ Created anonymous view: my_view Parameters: • x: U32 diff --git a/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print.snap b/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print.snap index b9a9b7827bd..6d643270705 100644 --- a/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print.snap +++ b/crates/schema/src/snapshots/spacetimedb_schema__auto_migrate__tests__updated pretty print.snap @@ -9,7 +9,7 @@ expression: "plan.pretty_print(PrettyPrintStyle::AnsiColor).expect(\"should pret ▸ Removed index Apples_id_name_idx_btree on [id, name] of table Apples ▸ Removed unique constraint Apples_id_key on [id] of table Apples ▸ Removed auto-increment constraint Apples_id_seq on column id of table Apples -▸ Removed schedule for table Deliveries_sched calling reducer check_deliveries +▸ Removed schedule for table Deliveries calling reducer check_deliveries ▸ ▸ Removed anonymous view: my_view Parameters: • x: U32 @@ -38,7 +38,7 @@ expression: "plan.pretty_print(PrettyPrintStyle::AnsiColor).expect(\"should pret ▸ Created index Apples_id_count_idx_btree on [id, count] of table Apples ▸ Created auto-increment constraint Bananas_id_seq on column id of table Bananas -▸ Created schedule for table Inspections_sched calling reducer perform_inspection +▸ Created schedule for table Inspections calling reducer perform_inspection ▸ ▸ Created anonymous view: my_view Parameters: • x: U32 diff --git a/crates/schema/src/table_name.rs b/crates/schema/src/table_name.rs index 3fe32ed70da..31c53728cb9 100644 --- a/crates/schema/src/table_name.rs +++ b/crates/schema/src/table_name.rs @@ -1,24 +1,53 @@ -use crate::identifier::Identifier; +use crate::identifier::{Identifier, NamespacedIdentifier}; use core::fmt; use core::ops::Deref; -use spacetimedb_sats::{impl_deserialize, impl_serialize, impl_st, raw_identifier::RawIdentifier}; +use spacetimedb_sats::{impl_deserialize, impl_serialize, impl_st, raw_identifier::RawNamespacedIdentifier}; /// The name of a table. +/// +/// Root tables have a single-segment name; submodule tables are namespaced +/// (e.g. `"lib.library_table"`). #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct TableName(Identifier); +pub struct TableName(NamespacedIdentifier); -impl_st!([] TableName, ts => Identifier::make_type(ts)); -impl_serialize!([] TableName, (self, ser) => self.0.serialize(ser)); -impl_deserialize!([] TableName, de => Identifier::deserialize(de).map(Self)); +impl_st!([] TableName, ts => RawNamespacedIdentifier::make_type(ts)); +impl_serialize!([] TableName, (self, ser) => ser.serialize_str(&self.0)); +// Stored names were validated when the table was created; segments are trusted +// here, mirroring `Identifier`'s own deserialization. +impl_deserialize!([] TableName, de => RawNamespacedIdentifier::deserialize(de) + .map(|raw| TableName(NamespacedIdentifier::new_unsafe_assume_valid(&raw)))); impl TableName { + /// The name of a root table. pub fn new(id: Identifier) -> Self { - Self(id) + Self(id.into()) + } + + /// The final segment, i.e. the name with any namespace prefix stripped. + pub fn local_name(&self) -> &Identifier { + self.0.local_name() + } + + /// Whether this table lives in a submodule namespace. + pub fn is_namespaced(&self) -> bool { + self.0.is_namespaced() } #[cfg(any(test, feature = "test"))] pub fn for_test(name: &str) -> Self { - Self(Identifier::for_test(name)) + Self(name.split('.').map(Identifier::for_test).collect()) + } +} + +impl From for TableName { + fn from(id: NamespacedIdentifier) -> Self { + Self(id) + } +} + +impl From for NamespacedIdentifier { + fn from(id: TableName) -> Self { + id.0 } } @@ -36,15 +65,22 @@ impl AsRef for TableName { } } +/// Panics if the `TableName` is a namespaced submodule table name, +/// since those are not single identifiers. Use `NamespacedIdentifier::from` +/// for names that may be namespaced. impl From for Identifier { fn from(id: TableName) -> Self { - id.0 + id.0.as_identifier() + .cloned() + .unwrap_or_else(|| panic!("TableName `{}` is namespaced; use NamespacedIdentifier instead", &*id)) } } -impl From for RawIdentifier { +// A `TableName` may carry a namespace, so its raw form is a `RawNamespacedIdentifier`, +// not a `RawIdentifier`: the latter can never be validated back into one name. +impl From for RawNamespacedIdentifier { fn from(id: TableName) -> Self { - Identifier::from(id).into() + id.0.into() } } diff --git a/crates/schema/tests/ensure_same_schema.rs b/crates/schema/tests/ensure_same_schema.rs index 7e5cb28f626..fb6d572b2a1 100644 --- a/crates/schema/tests/ensure_same_schema.rs +++ b/crates/schema/tests/ensure_same_schema.rs @@ -3,6 +3,7 @@ use serial_test::serial; use spacetimedb_schema::auto_migrate::{ponder_auto_migrate, AutoMigrateStep}; use spacetimedb_schema::def::ModuleDef; +use spacetimedb_schema::identifier::NamespacePath; use spacetimedb_testing::modules::{CompilationMode, CompiledModule}; fn get_normalized_schema(module_name: &str) -> ModuleDef { @@ -10,6 +11,36 @@ fn get_normalized_schema(module_name: &str) -> ModuleDef { module.extract_schema_blocking() } +/// The namespace a step's item lives in, if the step refers to a named item. +/// +/// Returns `None` for steps whose payload is not an item key (row-level security, which is +/// keyed by SQL text, and `DisconnectAllUsers`, which has no payload). +fn step_namespace<'a, 'def>(step: &'a AutoMigrateStep<'def>) -> Option<&'a NamespacePath> { + match step { + AutoMigrateStep::RemoveSchedule((ns, _)) + | AutoMigrateStep::RemoveView((ns, _)) + | AutoMigrateStep::RemoveTable((ns, _)) + | AutoMigrateStep::ChangeColumns((ns, _)) + | AutoMigrateStep::ReschemaEventTable((ns, _)) + | AutoMigrateStep::AddColumns((ns, _)) + | AutoMigrateStep::AddTable((ns, _)) + | AutoMigrateStep::AddSchedule((ns, _)) + | AutoMigrateStep::AddView((ns, _)) + | AutoMigrateStep::ChangeAccess((ns, _)) + | AutoMigrateStep::ChangePrimaryKey((ns, _)) + | AutoMigrateStep::UpdateView((ns, _)) + | AutoMigrateStep::RemoveIndex((ns, _)) + | AutoMigrateStep::RemoveConstraint((ns, _)) + | AutoMigrateStep::RemoveSequence((ns, _)) + | AutoMigrateStep::AddIndex((ns, _)) + | AutoMigrateStep::AddConstraint((ns, _)) + | AutoMigrateStep::AddSequence((ns, _)) => Some(ns), + AutoMigrateStep::AddRowLevelSecurity(_) + | AutoMigrateStep::RemoveRowLevelSecurity(_) + | AutoMigrateStep::DisconnectAllUsers => None, + } +} + fn assert_identical_modules(module_name_prefix: &str, lang_name: &str, suffix: &str) { let rs = get_normalized_schema(module_name_prefix); let cs = get_normalized_schema(&format!("{module_name_prefix}-{suffix}")); @@ -39,6 +70,12 @@ fn assert_identical_modules(module_name_prefix: &str, lang_name: &str, suffix: & ) }); + // TODO: Remove this once Rust and C# support submodules. + // Only TypeScript can mount submodules today, so its modules emit items in a namespace + // that the other languages cannot produce. Ignoring those steps keeps the root-level + // schema comparison meaningful instead of disabling it wholesale. + diff.retain(|step| step_namespace(step).is_none_or(|ns| ns.is_empty())); + assert!( diff.is_empty(), "Rust and {lang_name} modules are not identical. Here are the steps to migrate from {lang_name} to Rust: {diff:#?}" diff --git a/crates/sql-parser/src/ast/mod.rs b/crates/sql-parser/src/ast/mod.rs index b3222980471..57a87ebb1cf 100644 --- a/crates/sql-parser/src/ast/mod.rs +++ b/crates/sql-parser/src/ast/mod.rs @@ -1,4 +1,4 @@ -use spacetimedb_lib::sats::raw_identifier::RawIdentifier; +use spacetimedb_lib::sats::raw_identifier::RawNamespacedIdentifier; use spacetimedb_lib::Identity; use sqlparser::ast::Ident; use std::fmt::{Display, Formatter}; @@ -186,13 +186,17 @@ pub enum Parameter { /// A SQL identifier or named reference. /// Currently case sensitive. +/// +/// This holds a [`RawNamespacedIdentifier`] rather than a [`RawIdentifier`] because a +/// name written in SQL may be namespace-qualified (e.g. `lib.library_table`), and such +/// a name can never be validated into a single `Identifier`. #[derive(Debug, Clone)] -pub struct SqlIdent(pub RawIdentifier); +pub struct SqlIdent(pub RawNamespacedIdentifier); /// Case insensitivity should be implemented here if at all impl From for SqlIdent { fn from(Ident { value, .. }: Ident) -> Self { - SqlIdent(RawIdentifier::new(value)) + SqlIdent(RawNamespacedIdentifier::new(value)) } } diff --git a/crates/sql-parser/src/parser/errors.rs b/crates/sql-parser/src/parser/errors.rs index 953a031b8b8..e980970b8a0 100644 --- a/crates/sql-parser/src/parser/errors.rs +++ b/crates/sql-parser/src/parser/errors.rs @@ -1,10 +1,7 @@ use std::fmt::Display; use sqlparser::{ - ast::{ - BinaryOperator, Expr, Function, ObjectName, Query, Select, SelectItem, SetExpr, TableFactor, TableWithJoins, - Value, - }, + ast::{BinaryOperator, Expr, Function, Query, Select, SelectItem, SetExpr, TableFactor, TableWithJoins, Value}, parser::ParserError, }; use thiserror::Error; @@ -59,8 +56,6 @@ pub enum SqlUnsupported { DeleteTable(TableWithJoins), #[error("Unsupported column/variable assignment expression: {0}")] Assignment(Expr), - #[error("Multi-part names are not supported: {0}")] - MultiPartName(ObjectName), #[error("Unsupported: {0}")] Feature(String), #[error("Non-inner joins are not supported")] diff --git a/crates/sql-parser/src/parser/mod.rs b/crates/sql-parser/src/parser/mod.rs index b6fbe15c0dd..a6f9ff98b4b 100644 --- a/crates/sql-parser/src/parser/mod.rs +++ b/crates/sql-parser/src/parser/mod.rs @@ -5,6 +5,8 @@ use sqlparser::ast::{ WildcardAdditionalOptions, }; +use spacetimedb_lib::sats::raw_identifier::RawNamespacedIdentifier; + use crate::ast::{ BinOp, LogOp, Parameter, Project, ProjectElem, ProjectExpr, SqlExpr, SqlFrom, SqlIdent, SqlJoin, SqlLiteral, }; @@ -198,15 +200,23 @@ pub(crate) fn parse_project_elem(item: SelectItem) -> SqlParseResult SqlParseResult { match expr { Expr::Identifier(ident) => Ok(ProjectExpr::Var(ident.into())), - Expr::CompoundIdentifier(mut idents) if idents.len() == 2 => { - let table = idents.swap_remove(0).into(); - let field = idents.swap_remove(0).into(); + Expr::CompoundIdentifier(idents) if idents.len() >= 2 => { + let (table, field) = parse_qualified_field(idents)?; Ok(ProjectExpr::Field(table, field)) } _ => Err(SqlUnsupported::ProjectionExpr(expr).into()), } } +/// Parse a qualified column reference. +/// The last part is the column; the rest is the (possibly namespaced) table name or alias, +/// joined with dots to match [`parse_parts`]. +pub(crate) fn parse_qualified_field(mut idents: Vec) -> SqlParseResult<(SqlIdent, SqlIdent)> { + let field = idents.pop().expect("caller checked idents.len() >= 2").into(); + let table = parse_parts(idents)?; + Ok((table, field)) +} + // These types determine the size of [`parse_expr`]'s stack frame on 64-bit targets. // Changing their sizes will require updating the recursion limit to avoid stack overflows. // wasm32 has different type layouts, so this guard does not apply there. @@ -239,9 +249,8 @@ fn parse_expr(expr: Expr, depth: usize) -> SqlParseResult { SqlUnsupported::Expr, )?)), Expr::Identifier(ident) => Ok(SqlExpr::Var(ident.into())), - Expr::CompoundIdentifier(mut idents) if idents.len() == 2 => { - let table = idents.swap_remove(0).into(); - let field = idents.swap_remove(0).into(); + Expr::CompoundIdentifier(idents) if idents.len() >= 2 => { + let (table, field) = parse_qualified_field(idents)?; Ok(SqlExpr::Field(table, field)) } Expr::BinaryOp { @@ -343,10 +352,17 @@ pub(crate) fn parse_ident(ObjectName(parts): ObjectName) -> SqlParseResult) -> SqlParseResult { if parts.len() == 1 { return Ok(parts.swap_remove(0).into()); } - Err(SqlUnsupported::MultiPartName(ObjectName(parts)).into()) + let joined = parts.iter().map(|p| p.value.as_str()).collect::>().join("."); + Ok(SqlIdent(RawNamespacedIdentifier::new(joined))) } diff --git a/crates/sql-parser/src/parser/sql.rs b/crates/sql-parser/src/parser/sql.rs index e689817cc4c..0508baae5b2 100644 --- a/crates/sql-parser/src/parser/sql.rs +++ b/crates/sql-parser/src/parser/sql.rs @@ -403,8 +403,6 @@ mod tests { for sql in [ // FROM is required "select 1", - // Multi-part table names - "select a from s.t", // Bit-string literals "select * from t where a = B'1010'", // Wildcard with non-wildcard projections @@ -430,6 +428,13 @@ mod tests { fn supported() { for sql in [ "select a from t", + // Multi-part names are joined with dots for namespace-qualified tables + "select a from s.t", + // Qualified columns of namespaced tables: last part is the column + "select a from s.t where s.t.a = 1", + "select s.t.a from s.t", + "select a from x.s.t where x.s.t.a = 1", + r#"SELECT * FROM "s"."t" WHERE "s"."t"."a" = 1"#, "select a from t where x = :sender", "select count(*) as n from t", "select count(*) as n from t join s on t.id = s.id where s.x = 1", diff --git a/crates/sql-parser/src/parser/sub.rs b/crates/sql-parser/src/parser/sub.rs index 6a8ef34a1e8..1ae226fded8 100644 --- a/crates/sql-parser/src/parser/sub.rs +++ b/crates/sql-parser/src/parser/sub.rs @@ -178,6 +178,9 @@ mod tests { "select t.* from t join s on t.c = s.d", "select a.* from t as a join s as b on a.c = b.d", "select * from t where x = :sender", + // Qualified columns of namespaced tables: last part is the column + "select * from ns.t where ns.t.a = 1", + r#"SELECT ns.t.* FROM "ns"."s" JOIN "ns"."t" ON "ns"."s"."a" = "ns"."t"."a""#, ] { assert!(parse_subscription(sql).is_ok()); } diff --git a/crates/table/src/table.rs b/crates/table/src/table.rs index 554f840741a..7fb63a2e429 100644 --- a/crates/table/src/table.rs +++ b/crates/table/src/table.rs @@ -44,7 +44,7 @@ use spacetimedb_sats::{ layout::{AlgebraicTypeLayout, IncompatibleTypeLayoutError, PrimitiveType, RowTypeLayout, Size}, Typespace, }; -use spacetimedb_sats::{memory_usage::MemoryUsage, raw_identifier::RawIdentifier}; +use spacetimedb_sats::{memory_usage::MemoryUsage, raw_identifier::RawNamespacedIdentifier}; use spacetimedb_schema::{ def::IndexAlgorithm, identifier::Identifier, @@ -2328,7 +2328,7 @@ impl<'a> Iterator for IndexScanRangeIter<'a> { #[derive(Error, Debug, PartialEq, Eq)] #[error("Unique constraint violation '{}' in table '{}': column(s): '{:?}' value: {}", constraint_name, table_name, cols, value.to_satn())] pub struct UniqueConstraintViolation { - pub constraint_name: RawIdentifier, + pub constraint_name: RawNamespacedIdentifier, pub table_name: TableName, pub cols: Vec, pub value: AlgebraicValue, diff --git a/crates/testing/src/modules.rs b/crates/testing/src/modules.rs index cb44d4903e8..338dbf4a325 100644 --- a/crates/testing/src/modules.rs +++ b/crates/testing/src/modules.rs @@ -9,11 +9,14 @@ use std::time::Instant; use bytes::{Bytes, BytesMut}; use futures::{FutureExt as _, TryStreamExt as _}; use spacetimedb::config::CertificateAuthority; +use spacetimedb::host::ModuleHost; use spacetimedb::messages::control_db::HostType; use spacetimedb::util::jobs::JobCores; use spacetimedb::Identity; use spacetimedb_client_api::auth::SpacetimeAuth; use spacetimedb_client_api::routes::subscribe::{generate_random_connection_id, WebSocketOptions}; +use spacetimedb_lib::http as st_http; +use spacetimedb_lib::AlgebraicValue; use spacetimedb_paths::{RootDir, SpacetimePaths}; use spacetimedb_schema::auto_migrate::MigrationPolicy; use spacetimedb_schema::def::ModuleDef; @@ -166,6 +169,56 @@ impl ModuleHandle { .expect("failed to collect log stream"); String::from_utf8(bytes.into()).unwrap() } + + async fn module_host(&self) -> ModuleHost { + let database = self + .env + .get_database_by_identity(&self.db_identity) + .await + .unwrap() + .unwrap(); + let host = self.env.leader(database.id).await.expect("host should be running"); + host.module().await.expect("module should be running") + } + + /// Call a procedure by name with JSON-encoded args, returning the raw `AlgebraicValue` on success. + pub async fn call_procedure_with_args(&self, procedure: &str, args_json: &str) -> anyhow::Result { + let module = self.module_host().await; + let ret = module + .call_procedure( + Identity::ZERO, + None, + None, + procedure, + FunctionArgs::Json(args_json.into()), + ) + .await; + ret.result + .map(|r| r.return_val) + .map_err(|e| anyhow::anyhow!("procedure {procedure} failed: {e:#}")) + } + + /// Dispatch a GET request to a module HTTP route by path, returning the response body on success. + pub async fn call_http_route_get(&self, path: &str) -> anyhow::Result { + let module = self.module_host().await; + let (handler_id, _, _) = module + .info() + .module_def + .match_http_route(&st_http::Method::Get, path) + .ok_or_else(|| anyhow::anyhow!("no GET route registered for {path}"))?; + let request = st_http::Request { + method: st_http::Method::Get, + headers: std::iter::empty::<(Option>, Box<[u8]>)>().collect(), + timeout: None, + uri: format!("http://localhost{path}"), + version: st_http::Version::Http11, + }; + let (_response, body) = module + .call_http_handler(handler_id, request, Bytes::new()) + .await + .map_err(|e| anyhow::anyhow!("HTTP handler error: {e}"))?; + Ok(body) + } } pub struct CompiledModule { diff --git a/crates/testing/tests/standalone_integration_test.rs b/crates/testing/tests/standalone_integration_test.rs index 8de5cfe5bc4..8214e527120 100644 --- a/crates/testing/tests/standalone_integration_test.rs +++ b/crates/testing/tests/standalone_integration_test.rs @@ -459,3 +459,50 @@ fn test_calling_bench_db_ia_loop_typescript() { fn test_calling_bench_db_ia_loop_cpp() { test_calling_bench_db_ia_loop::(); } + +fn test_submodule_in_module(module_name: &'static str) { + init(); + + CompiledModule::compile(module_name, CompilationMode::Debug).with_module_async( + DEFAULT_CONFIG, + |mut module| async move { + // ── 1. Cross-namespace reducer call ────────────────────────────────── + // `useSubmodule` is exported as camelCase; its wire name is the canonical + // snake_case form. + let json = + r#"{"CallReducer": {"reducer": "use_submodule", "args": "[\"hello_submodule\"]", "request_id": 0, "flags": 0}}"# + .to_string(); + module.send_reducer_and_recv_update(json, 0).await.unwrap(); + + let logs = read_logs(&module).await; + let relevant: Vec<_> = logs + .into_iter() + .filter(|l| !is_scheduled_test_log(l)) + .collect(); + assert_eq!(relevant, ["libInsert: hello_submodule"].map(String::from)); + + // ── 2. Cross-namespace procedure call ───────────────────────────────── + // useSubmoduleProcedure calls libCount in the lib submodule. + // We inserted one row above, so the count should be 1. + let return_val = module + .call_procedure_with_args("use_submodule_procedure", "[]") + .await + .expect("use_submodule_procedure should succeed"); + assert_eq!(return_val, AlgebraicValue::U64(1), "libCount should return 1 after one insert"); + + // ── 3. Cross-namespace HTTP handler ─────────────────────────────────── + // The root module's /lib-hello route delegates to lib_submodule's libHello handler. + let body = module + .call_http_route_get("/lib-hello") + .await + .expect("GET /lib-hello should succeed"); + assert_eq!(body.as_ref(), b"Hello from lib submodule!"); + }, + ); +} + +#[test] +#[serial] +fn test_submodule_typescript() { + test_submodule_in_module("module-test-ts"); +} diff --git a/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md b/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md new file mode 100644 index 00000000000..242888f333d --- /dev/null +++ b/docs/docs/00200-core-concepts/00100-databases/00600-submodules.md @@ -0,0 +1,348 @@ +--- +title: Submodules +slug: /submodules +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +A **submodule** is a SpacetimeDB module that can be included in another module's database. The submodule's tables and functions register under a **namespace** you choose, keeping them separate from the consumer's own tables and from other submodules. + +Submodules let you package reusable database logic as a library that any consumer can integrate without coordinating table names. + +:::note +Submodules are currently supported in TypeScript only. Support for Rust, C#, and C++ is coming soon. +::: + +## Writing a Submodule + +A submodule is a regular SpacetimeDB module. Nothing special marks a module as a submodule. Export the schema as the default export and export every function (reducers, procedures, views, HTTP handlers) the consumer needs to register. + + + + +```typescript +// auth_lib/src/index.ts +import { schema, table, t, SyncResponse, Router } from 'spacetimedb/server'; + +const users = table( + { name: 'users', public: true }, + { identity: t.identity().primaryKey(), username: t.string() } +); + +const sessions = table( + { name: 'sessions' }, + { + id: t.u64().primaryKey().autoInc(), + userIdentity: t.identity(), + token: t.string(), + } +); + +const spacetimedb = schema({ users, sessions }); +export default spacetimedb; + +export const verifyToken = spacetimedb.reducer( + { token: t.string() }, + (ctx, { token }) => { /* ... */ } +); + +export const sessionCount = spacetimedb.procedure( + t.u64(), + (ctx) => ctx.withTx(tx => tx.db.sessions.count()) +); + +export const activeSessions = spacetimedb.anonymousView( + { name: 'activeSessions', public: true }, + t.array(sessions.rowType), + (ctx) => [...ctx.db.sessions.iter()] +); + +export const health = spacetimedb.httpHandler( + (_ctx, _req) => new SyncResponse('ok') +); + +export const router = spacetimedb.httpRouter( + new Router().get('/health', health) +); +``` + +The same file can be published as a standalone database or used as a submodule by another module. The module does not need to declare which role it plays. + + + + +## Using a Submodule + +The consumer controls the namespace name. Pass the submodule's module-namespace object under the alias you choose. + + + + +```typescript +// my-database/src/index.ts +import { schema } from 'spacetimedb/server'; +import * as authLib from 'auth_lib'; + +const players = table({ name: 'players', public: true }, { /* ... */ }); + +const spacetimedb = schema({ + players, + myauth: authLib, // register auth_lib under the namespace "myauth" +}); +export default spacetimedb; +``` + +`export default spacetimedb` is the only JS export required from the consumer. Registering the submodule adds all of its reducers, procedures, views, scheduled tables, and HTTP handlers automatically. + +:::warning Use `import * as`, not a default import +```typescript +import authLib from 'auth_lib'; // ❌ misses all named exports +import * as authLib from 'auth_lib'; // ✅ correct +``` +A default-only import exposes only the submodule's schema, not its named exports (reducers, procedures, views, handlers). The submodule walker requires the full module-namespace object. A clear error is raised if you use the wrong form. +::: + + + + +## Accessing Submodule Tables and Views + +Submodule tables appear under a namespace field on `ctx.db`. The field matches the alias you chose. Views exported by a submodule behave like tables from the client's perspective: they are accessible as `.` in subscriptions and SQL queries, where `` is the canonical snake_case name — `activeSessions` in TypeScript is `active_sessions` in SQL. The generated client bindings expose the camelCase accessor instead, as shown below. + + + + +```typescript +spacetimedb.reducer('example', {}, (ctx) => { + // Consumer's own tables (no namespace) + for (const player of ctx.db.players.iter()) { /* ... */ } + + // Submodule tables, registered as "myauth" + const user = ctx.db.myauth.users.identity.find(ctx.sender); + for (const session of ctx.db.myauth.sessions.iter()) { /* ... */ } +}); +``` + + + + +## Calling Submodule Functions + +A submodule can expose reducers, procedures, views, and HTTP handlers that the consumer calls from its own functions. Because the submodule's context type and the consumer's context type are distinct, use `ctx.as.` to narrow the context before passing it to a submodule function. + +### From a Reducer + + + + +Call a submodule reducer or a plain helper function typed against the submodule's schema using `ctx.as.`: + +```typescript +// auth_lib: plain helper function typed against the submodule's own schema +export function sessionCountHelper(ctx: ReducerContext): number { + return ctx.db.sessions.count(); +} + +// my-database: call submodule reducer and helper from a consumer reducer +spacetimedb.reducer('onLogin', { token: t.string() }, (ctx, { token }) => { + // call a submodule reducer + authLib.verifyToken(ctx.as.myauth, { token }); + + // call a submodule helper function + const count = authLib.sessionCountHelper(ctx.as.myauth); + console.log(`Active sessions: ${count}`); +}); +``` + +`ctx.as.myauth` is a `ReducerContext` scoped to the `myauth` namespace. It shares the same sender, timestamp, and connectionId as the parent context, but its `ctx.db` points at `ctx.db.myauth`. + +For reducers registered through the submodule's own schema (via `schema.reducer(...)`), the host passes a scoped context automatically when invoked directly. `ctx.as` is only needed when the consumer calls a submodule function explicitly. + + + + +### From a Procedure + + + + +Use `ctx.as.` to pass a submodule-scoped `ProcedureContext` to a submodule procedure. To call a submodule reducer from inside a procedure, open a transaction first with `ctx.withTx` and then narrow with `tx.as.`: + +```typescript +// call a submodule procedure +export const stats = spacetimedb.procedure( + t.u64(), + (ctx) => authLib.sessionCount(ctx.as.myauth) +); + +// call a submodule reducer inside a withTx block +export const transactAndCount = spacetimedb.procedure( + { token: t.string() }, + t.u64(), + (ctx, { token }) => { + ctx.withTx(tx => { + // tx is a root ReducerContext; narrow to the submodule namespace + authLib.verifyToken(tx.as.myauth, { token }); + }); + return authLib.sessionCount(ctx.as.myauth); + } +); +``` + + + + +### From an HTTP Handler + + + + +Delegate to a submodule's HTTP handler by passing `ctx.as.` and the request to the submodule handler function, then register it on the consumer's router: + +```typescript +import { Router } from 'spacetimedb/server'; + +// delegate the /health route to the submodule's handler +export const healthCheck = spacetimedb.httpHandler((ctx, req) => { + return authLib.health(ctx.as.myauth, req); +}); + +export const router = spacetimedb.httpRouter( + new Router().get('/health', healthCheck) +); +``` + + + + +## Multiple and Nested Submodules + +Multiple submodules compose freely. Each gets its own namespace, so name collisions between submodules are impossible. + + + + +```typescript +import * as authLib from 'auth_lib'; +import * as paymentLib from 'payment_lib'; + +const spacetimedb = schema({ + players, + myauth: authLib, + payments: paymentLib, +}); +export default spacetimedb; +``` + +A submodule can itself include other submodules using the same syntax. The nested submodule's tables appear under a two-level path in the top-level consumer: + +```typescript +// auth_lib includes session_lib as "sessions" +const authSchema = schema({ users, sessions: sessionLib }); +export default authSchema; + +// consumer includes auth_lib as "myauth" +// session_lib's tables are at ctx.db.myauth.sessions.
+``` + +**Lifecycle reducers** (`init`, `clientConnected`, `clientDisconnected`) are an exception: these are only allowed for the root module. Modules containing submodules which define lifecycle reducers will fail to publish. + + + + +## Client Subscriptions + +Client subscriptions use the same namespace structure as server-side access. Submodule tables and views are queried as `.`. + + + + +```typescript +conn.subscriptionBuilder().subscribe(tables => [ + tables.players.build(), // public.players + tables.myauth.users.build(), // myauth.users + tables.myauth.activeSessions.build(), // myauth.activeSessions view +]); +``` + + + + +## Calling Submodule Reducers and Procedures from the Client + +Submodule reducers and procedures are identified by their fully-qualified name, using `/` as the separator between namespace and function name. + +### Client SDK + +In generated bindings, submodule tables, views, reducers, and procedures appear as nested objects under the namespace alias. The `tables`, `reducers`, and `procedures` exports all reflect the same nesting. + + + + +React hooks: + +```typescript +import { tables, reducers, procedures } from './module_bindings'; +import { useTable, useReducer, useProcedure } from 'spacetimedb/react'; + +// subscribe to a submodule table and view +const [users] = useTable(tables.myauth.users); +const [activeSessions] = useTable(tables.myauth.activeSessions); + +// call a submodule reducer +const verifyToken = useReducer(reducers.myauth.verifyToken); +verifyToken({ token: 'abc123' }); + +// call a submodule procedure +const sessionCount = useProcedure(procedures.myauth.sessionCount); +sessionCount().then(count => console.log(`Sessions: ${count}`)); +``` + +Vanilla (non-React): + +```typescript +import { DbConnection, tables, reducers, procedures } from './module_bindings'; + +const conn = DbConnection.builder() + .withUri(SPACETIMEDB_URI) + .withDatabaseName('my-database') + .onConnect(ctx => { + ctx.subscriptionBuilder() + .subscribe([tables.myauth.users, tables.myauth.activeSessions]); + }) + .build(); + +conn.reducers.myauth.verifyToken({ token: 'abc123' }); +``` + + + + +### HTTP API + +``` +POST /v1/database/my-database/call/myauth/verify_token +``` + +### CLI + +```bash +spacetime call my-database "myauth/verify_token" '{"token": "abc123"}' +``` + +The namespace prefix is the alias you chose, and the function name after `/` is the canonical +snake_case form of the submodule's export name -- `verifyToken` in TypeScript is `verify_token` +on the wire. Generated client bindings expose the camelCase accessor instead, as shown above. + +## Namespace Name Rules + +The alias you choose becomes the SQL-level namespace name. It must be a valid SpacetimeDB identifier: starts with a letter or underscore, continues with letters, digits, or underscores, maximum 63 characters, case-insensitive for resolution. A submodule can be registered under at most one alias per consumer module. + +The reserved namespaces `public`, `st`, `spacetimedb`, and `pg_*` cannot be used as submodule aliases. + +## Limitations + +### Submodule routers are not applied automatically + +A submodule can define its own `httpRouter`, but when used as a submodule that router is ignored. Only the consumer's root router is used. To expose a submodule's HTTP handlers, register them explicitly on the consumer's router using `ctx.as.` as shown in the [HTTP handler section](#from-an-http-handler). diff --git a/modules/module-test-ts/src/index.ts b/modules/module-test-ts/src/index.ts index 1412ddad7bf..0eba467a7d0 100644 --- a/modules/module-test-ts/src/index.ts +++ b/modules/module-test-ts/src/index.ts @@ -3,7 +3,6 @@ // ───────────────────────────────────────────────────────────────────────────── import { ScheduleAt } from 'spacetimedb'; import { - Router, schema, SyncResponse, table, @@ -11,7 +10,9 @@ import { type Infer, type InferTypeOfRow, errors, + Router, } from 'spacetimedb/server'; +import * as libSubmodule from './lib_submodule'; // ───────────────────────────────────────────────────────────────────────────── // TYPE ALIASES @@ -247,6 +248,7 @@ const spacetimedb = schema({ playerLikeRow ), tableToRemove: table({ name: 'table_to_remove' }, { id: t.u32() }), + lib: libSubmodule, }); export default spacetimedb; @@ -525,10 +527,28 @@ export const getMySchemaViaHttp = spacetimedb.procedure(t.string(), ctx => { } }); +// useSubmodule: calls the lib submodule's libInsert reducer cross-namespace. +export const useSubmodule = spacetimedb.reducer( + { value: t.string() }, + (ctx, { value }) => { + libSubmodule.libInsert(ctx.as.lib, { value }); + } +); + +// useSubmoduleProcedure: calls the lib submodule's libCount procedure and returns the result. +export const useSubmoduleProcedure = spacetimedb.procedure(t.u64(), ctx => + libSubmodule.libCount(ctx.as.lib, {}) +); + export const getSimple = spacetimedb.httpHandler( (_ctx, _req) => new SyncResponse('ok') ); +// Delegates to the lib submodule's HTTP handler, demonstrating cross-namespace HTTP dispatch. +export const libHello = spacetimedb.httpHandler((ctx, req) => { + return libSubmodule.libHello(ctx.as.lib, req); +}); + export const router = spacetimedb.httpRouter( - new Router().get('/get', getSimple) + new Router().get('/get', getSimple).get('/lib-hello', libHello) ); diff --git a/modules/module-test-ts/src/lib_submodule.ts b/modules/module-test-ts/src/lib_submodule.ts new file mode 100644 index 00000000000..3c2832d602c --- /dev/null +++ b/modules/module-test-ts/src/lib_submodule.ts @@ -0,0 +1,28 @@ +import { schema, table, t, SyncResponse } from 'spacetimedb/server'; + +const libData = table( + { name: 'libData', public: true }, + { + id: t.u64().primaryKey().autoInc(), + value: t.string(), + } +); + +const libSubmoduleSchema = schema({ libData }); +export default libSubmoduleSchema; + +export const libInsert = libSubmoduleSchema.reducer( + { value: t.string() }, + (ctx, { value }) => { + console.info(`libInsert: ${value}`); + ctx.db.libData.insert({ id: 0n, value }); + } +); + +export const libCount = libSubmoduleSchema.procedure(t.u64(), ctx => + ctx.withTx(tx => tx.db.libData.count()) +); + +export const libHello = libSubmoduleSchema.httpHandler((_ctx, _req) => { + return new SyncResponse('Hello from lib submodule!'); +}); diff --git a/tools/ci/src/main.rs b/tools/ci/src/main.rs index 4d506a4e5f4..be61a790379 100644 --- a/tools/ci/src/main.rs +++ b/tools/ci/src/main.rs @@ -655,6 +655,15 @@ fn main() -> Result<()> { } Some(CiCmd::WasmBindings) => { + pnpm([ + "install", + "--filter", + "./crates/bindings-typescript...", + "--filter", + "./modules/module-test-ts...", + ]) + .run()?; + pnpm(["build"]).dir("crates/bindings-typescript").run()?; cmd!("cargo", "test", "-p", "spacetimedb-codegen").run()?; // Pre-build the CLI so that it _doesn't_ get `cargo update`d, since that may break the build. cmd!("cargo", "build", "-p", "spacetimedb-cli").run()?;