From 19706953874798170bcaab654c3c29e306693eee Mon Sep 17 00:00:00 2001 From: Luxian Date: Mon, 5 Oct 2026 21:29:47 +0800 Subject: [PATCH 1/3] feat: expire unconsumed native metadata preparations atomically --- src/callisto/mst2_metadata_prepare.rs | 1 + ...000500_add_mst2_metadata_prepare_expiry.rs | 130 +++ src/jupiter/migration/mod.rs | 3 + src/jupiter/storage/mod.rs | 1 + .../storage/native_metadata_install.rs | 282 +++++-- .../storage/native_metadata_install_tests.rs | 146 ++-- src/jupiter/storage/native_metadata_lease.rs | 24 +- .../storage/native_metadata_lease_tests.rs | 85 +- .../storage/native_metadata_prepare_expiry.rs | 589 ++++++++++++++ .../native_metadata_prepare_expiry_tests.rs | 764 ++++++++++++++++++ 10 files changed, 1860 insertions(+), 165 deletions(-) create mode 100644 src/jupiter/migration/m20261005_000500_add_mst2_metadata_prepare_expiry.rs create mode 100644 src/jupiter/storage/native_metadata_prepare_expiry.rs create mode 100644 src/jupiter/storage/native_metadata_prepare_expiry_tests.rs diff --git a/src/callisto/mst2_metadata_prepare.rs b/src/callisto/mst2_metadata_prepare.rs index 9401282b..e48c3b67 100644 --- a/src/callisto/mst2_metadata_prepare.rs +++ b/src/callisto/mst2_metadata_prepare.rs @@ -23,6 +23,7 @@ pub struct Model { pub edge_count: i32, pub total_bytes: i64, pub state: String, + pub deadline_managed: bool, pub created_at: DateTimeWithTimeZone, pub committed_at: Option, } diff --git a/src/jupiter/migration/m20261005_000500_add_mst2_metadata_prepare_expiry.rs b/src/jupiter/migration/m20261005_000500_add_mst2_metadata_prepare_expiry.rs new file mode 100644 index 00000000..c58ea2d1 --- /dev/null +++ b/src/jupiter/migration/m20261005_000500_add_mst2_metadata_prepare_expiry.rs @@ -0,0 +1,130 @@ +//! Forward-only preparation deadlines; historical rows retain unknown deadlines. + +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager.get_connection().execute_unprepared( + "ALTER TABLE mst2_metadata_prepare + ADD COLUMN deadline_managed boolean NOT NULL DEFAULT false, + DROP CONSTRAINT mst2_metadata_prepare_state_check, + DROP CONSTRAINT mst2_metadata_prepare_commit_check, + ADD CONSTRAINT mst2_metadata_prepare_state_check CHECK (state IN ('PREPARING','COMMITTED','CONSUMED','EXPIRED')), + ADD CONSTRAINT mst2_metadata_prepare_commit_check CHECK ( + (state='PREPARING' AND committed_at IS NULL) OR + (state IN ('COMMITTED','CONSUMED') AND committed_at IS NOT NULL) OR + (state='EXPIRED' AND deadline_managed)); + CREATE TABLE mst2_metadata_prepare_deadline ( + prepare_id text PRIMARY KEY REFERENCES mst2_metadata_prepare(prepare_id), + storage_uuid text NOT NULL REFERENCES mst2_metadata_storage_scope(storage_uuid), + operation_id text NOT NULL UNIQUE CHECK (octet_length(operation_id) BETWEEN 1 AND 255), + manifest_digest bytea NOT NULL CHECK (octet_length(manifest_digest)=32), + request_digest bytea NOT NULL CHECK (octet_length(request_digest)=32), + grant_digest bytea NOT NULL CHECK (octet_length(grant_digest)=32), + duration_ms bigint NOT NULL CHECK (duration_ms BETWEEN 1 AND 3600000), + granted_at_ms bigint NOT NULL CHECK (granted_at_ms>0), + expires_at_ms bigint NOT NULL CHECK (expires_at_ms=granted_at_ms+duration_ms) + ); + CREATE INDEX idx_mst2_metadata_prepare_deadline_expiry ON mst2_metadata_prepare_deadline(expires_at_ms); + CREATE TABLE mst2_metadata_prepare_expiry ( + prepare_id text PRIMARY KEY REFERENCES mst2_metadata_prepare_deadline(prepare_id), + operation_id text NOT NULL UNIQUE CHECK (octet_length(operation_id) BETWEEN 1 AND 255), + operation_digest bytea NOT NULL CHECK (octet_length(operation_digest)=32), + grant_digest bytea NOT NULL CHECK (octet_length(grant_digest)=32), + previous_state text NOT NULL CHECK (previous_state IN ('PREPARING','COMMITTED')), + pin_root text NOT NULL CHECK (pin_root='prepare:' || prepare_id), + expires_at_ms bigint NOT NULL CHECK (expires_at_ms>0), + expired_at_ms bigint NOT NULL CHECK (expired_at_ms>=expires_at_ms) + ); + CREATE TRIGGER mst2_metadata_prepare_deadline_immutable BEFORE UPDATE OR DELETE + ON mst2_metadata_prepare_deadline FOR EACH ROW EXECUTE FUNCTION mst2_metadata_payload_immutable(); + CREATE TRIGGER mst2_metadata_prepare_expiry_immutable BEFORE UPDATE OR DELETE + ON mst2_metadata_prepare_expiry FOR EACH ROW EXECUTE FUNCTION mst2_metadata_payload_immutable(); + CREATE FUNCTION mst2_metadata_prepare_deadline_guard() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.deadline_managed IS DISTINCT FROM OLD.deadline_managed THEN + RAISE EXCEPTION 'MST2 preparation deadline management cannot be changed'; + END IF; + IF OLD.state='EXPIRED' AND NEW.state<>'EXPIRED' THEN + RAISE EXCEPTION 'Expired MST2 preparations cannot be revived'; + END IF; + IF NEW.state='EXPIRED' AND + (OLD.state NOT IN ('PREPARING','COMMITTED','EXPIRED') OR + NEW.committed_at IS DISTINCT FROM OLD.committed_at) THEN + RAISE EXCEPTION 'MST2 preparation expiry must preserve its unconsumed history'; + END IF; + RETURN NEW; + END $$; + CREATE TRIGGER mst2_metadata_prepare_deadline_guard BEFORE UPDATE ON mst2_metadata_prepare + FOR EACH ROW EXECUTE FUNCTION mst2_metadata_prepare_deadline_guard(); + CREATE FUNCTION mst2_metadata_prepare_deadline_check() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM mst2_metadata_prepare_deadline d JOIN mst2_metadata_storage_scope s ON s.singleton=1 + WHERE d.prepare_id=NEW.prepare_id AND d.operation_id=NEW.operation_id AND + d.manifest_digest=NEW.manifest_digest AND d.storage_uuid=s.storage_uuid + ) THEN + RAISE EXCEPTION 'Managed MST2 preparations require their fixed deadline grant in the same transaction'; + END IF; + RETURN NEW; + END $$; + CREATE CONSTRAINT TRIGGER mst2_metadata_prepare_deadline_check AFTER INSERT OR UPDATE ON mst2_metadata_prepare + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW WHEN (NEW.deadline_managed) + EXECUTE FUNCTION mst2_metadata_prepare_deadline_check(); + CREATE FUNCTION mst2_metadata_prepare_grant_check() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM mst2_metadata_prepare p JOIN mst2_metadata_storage_scope s ON s.singleton=1 + WHERE p.prepare_id=NEW.prepare_id AND p.deadline_managed AND p.operation_id=NEW.operation_id AND + p.manifest_digest=NEW.manifest_digest AND s.storage_uuid=NEW.storage_uuid + ) THEN + RAISE EXCEPTION 'MST2 preparation grants require their fixed managed preparation'; + END IF; + RETURN NEW; + END $$; + CREATE CONSTRAINT TRIGGER mst2_metadata_prepare_grant_check AFTER INSERT ON mst2_metadata_prepare_deadline + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_prepare_grant_check(); + CREATE FUNCTION mst2_metadata_prepare_expiry_check() RETURNS trigger LANGUAGE plpgsql AS $$ + DECLARE prior text; + BEGIN + IF TG_OP='INSERT' THEN + RAISE EXCEPTION 'MST2 preparations cannot start in EXPIRED state'; + END IF; + SELECT previous_state INTO prior FROM mst2_metadata_prepare_expiry WHERE prepare_id=NEW.prepare_id; + IF prior IS NULL OR (OLD.state<>'EXPIRED' AND prior<>OLD.state) OR + ((prior='COMMITTED')<>(NEW.committed_at IS NOT NULL)) OR + EXISTS (SELECT 1 FROM mst2_metadata_prepare_consumption WHERE prepare_id=NEW.prepare_id) OR + EXISTS (SELECT 1 FROM mst2_retention_root WHERE root_key='prepare:' || NEW.prepare_id) THEN + RAISE EXCEPTION 'MST2 preparation expiry requires its immutable event and released Prepare pins'; + END IF; + RETURN NEW; + END $$; + CREATE CONSTRAINT TRIGGER mst2_metadata_prepare_expiry_check AFTER INSERT OR UPDATE ON mst2_metadata_prepare + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW WHEN (NEW.state='EXPIRED') + EXECUTE FUNCTION mst2_metadata_prepare_expiry_check(); + CREATE FUNCTION mst2_metadata_prepare_expiry_event_check() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM mst2_metadata_prepare p JOIN mst2_metadata_prepare_deadline d ON d.prepare_id=p.prepare_id + WHERE p.prepare_id=NEW.prepare_id AND p.state='EXPIRED' AND p.deadline_managed AND + d.grant_digest=NEW.grant_digest AND d.expires_at_ms=NEW.expires_at_ms AND + ((NEW.previous_state='COMMITTED')=(p.committed_at IS NOT NULL)) + ) OR EXISTS (SELECT 1 FROM mst2_metadata_prepare_consumption WHERE prepare_id=NEW.prepare_id) OR + EXISTS (SELECT 1 FROM mst2_retention_root WHERE root_key=NEW.pin_root) THEN + RAISE EXCEPTION 'MST2 preparation expiry events require their matching terminal grant and released pins'; + END IF; + RETURN NEW; + END $$; + CREATE CONSTRAINT TRIGGER mst2_metadata_prepare_expiry_event_check AFTER INSERT ON mst2_metadata_prepare_expiry + DEFERRABLE INITIALLY DEFERRED FOR EACH ROW EXECUTE FUNCTION mst2_metadata_prepare_expiry_event_check();" + ).await.map(|_| ()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + Ok(()) + } +} diff --git a/src/jupiter/migration/mod.rs b/src/jupiter/migration/mod.rs index 4fefa4fb..c75a761c 100644 --- a/src/jupiter/migration/mod.rs +++ b/src/jupiter/migration/mod.rs @@ -145,6 +145,7 @@ mod m20261005_000200_add_mst2_native_head; mod m20261005_000200_harden_mst2_retention_graph; mod m20261005_000300_add_mst2_metadata_install; mod m20261005_000400_add_mst2_metadata_lease_ledger; +mod m20261005_000500_add_mst2_metadata_prepare_expiry; mod runner; pub use m20260905_000100_add_push_queue::ensure_queue_control_seed; pub use runner::apply_migrations; @@ -278,6 +279,7 @@ impl MigratorTrait for Migrator { Box::new(m20261005_000200_harden_mst2_retention_graph::Migration), Box::new(m20261005_000300_add_mst2_metadata_install::Migration), Box::new(m20261005_000400_add_mst2_metadata_lease_ledger::Migration), + Box::new(m20261005_000500_add_mst2_metadata_prepare_expiry::Migration), ] } } @@ -496,6 +498,7 @@ mod tests { "m20261005_000200_harden_mst2_retention_graph".to_string(), "m20261005_000300_add_mst2_metadata_install".to_string(), "m20261005_000400_add_mst2_metadata_lease_ledger".to_string(), + "m20261005_000500_add_mst2_metadata_prepare_expiry".to_string(), ]; assert_eq!( &names[names.len() - expected.len()..], diff --git a/src/jupiter/storage/mod.rs b/src/jupiter/storage/mod.rs index 77372864..61d95d02 100644 --- a/src/jupiter/storage/mod.rs +++ b/src/jupiter/storage/mod.rs @@ -20,6 +20,7 @@ pub(crate) mod mst2_publication_storage; pub mod mst2_retention; pub mod native_metadata_install; pub(crate) mod native_metadata_lease; +pub(crate) mod native_metadata_prepare_expiry; pub(crate) mod native_publication_storage; pub mod notification_storage; pub mod object_storage; diff --git a/src/jupiter/storage/native_metadata_install.rs b/src/jupiter/storage/native_metadata_install.rs index 35b3adb6..db8a42d1 100644 --- a/src/jupiter/storage/native_metadata_install.rs +++ b/src/jupiter/storage/native_metadata_install.rs @@ -13,7 +13,13 @@ use sea_orm::{ }; use serde_json::json; -use super::mst2_retention::{PostgresRetentionRepository, RETENTION_LOCK_KEY}; +use super::{ + mst2_retention::{PostgresRetentionRepository, RETENTION_LOCK_KEY}, + native_metadata_prepare_expiry::{ + DEFAULT_PREPARE_DURATION_MS, PrepareDeadline, clock, load_deadline, load_expiry, + request_digest, + }, +}; use crate::{ callisto::{ mst2_metadata_payload, mst2_metadata_prepare, mst2_metadata_prepare_page, @@ -46,6 +52,7 @@ pub enum MetadataInstallError { operation_id: String, manifest_digest: [u8; 32], phase: MetadataCommitPhase, + recovery: Box, }, } @@ -54,6 +61,7 @@ pub struct MetadataPrepareIntent { prepare_id: String, operation_id: String, manifest_digest: [u8; 32], + deadline_digest: Option<[u8; 32]>, } impl MetadataPrepareIntent { @@ -66,6 +74,31 @@ impl MetadataPrepareIntent { pub fn manifest_digest(&self) -> [u8; 32] { self.manifest_digest } + pub fn deadline_digest(&self) -> Option<[u8; 32]> { + self.deadline_digest + } + pub fn recovery(&self, phase: MetadataCommitPhase) -> MetadataPrepareRecovery { + MetadataPrepareRecovery { + operation_id: self.operation_id.clone(), + manifest_digest: self.manifest_digest, + phase, + binding: PrepareRecoveryBinding::Exact(self.clone()), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MetadataPrepareRecovery { + operation_id: String, + manifest_digest: [u8; 32], + phase: MetadataCommitPhase, + binding: PrepareRecoveryBinding, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum PrepareRecoveryBinding { + Exact(MetadataPrepareIntent), + Request([u8; 32]), } #[derive(Debug, Clone, PartialEq, Eq)] @@ -92,15 +125,20 @@ pub enum MetadataPrepareObservation { Absent, Preparing(MetadataPrepareIntent), Committed(PreparedMetadataReceipt), + DeadlinePassed { + intent: MetadataPrepareIntent, + expires_at_ms: i64, + }, } pub(super) struct StoredPlan { pub(super) record: mst2_metadata_prepare::Model, pub(super) plan: MetadataInstallPlan, + pub(super) deadline: Option, } impl StoredPlan { - fn intent(&self) -> Result { + pub(super) fn intent(&self) -> Result { Ok(MetadataPrepareIntent { prepare_id: self.record.prepare_id.clone(), operation_id: self.record.operation_id.clone(), @@ -110,6 +148,7 @@ impl StoredPlan { .as_slice() .try_into() .map_err(|_| integrity("invalid stored metadata manifest digest"))?, + deadline_digest: self.deadline.as_ref().map(|grant| grant.grant_digest), }) } fn receipt(&self) -> Result { @@ -154,15 +193,32 @@ impl PostgresMetadataInstallRepository { &self, operation_id: &str, prepared: &PreparedNativeMetadataRetention, + ) -> Result { + self.begin_intent_with_duration(operation_id, prepared, DEFAULT_PREPARE_DURATION_MS) + .await + } + + pub(crate) async fn begin_intent_with_duration( + &self, + operation_id: &str, + prepared: &PreparedNativeMetadataRetention, + duration_ms: i64, ) -> Result { validate_operation_id(operation_id)?; let plan = prepared.install_plan()?; let digest = plan.digest()?; + let expected_request = + request_digest(self.storage_uuid(), operation_id, &digest, duration_ms)?; let txn = self.transaction().await?; let result = async { self.barrier(&txn).await?; + lock_prepare_operation(&txn, operation_id).await?; if let Some(stored) = load_plan(&txn, operation_id, &digest).await? { - reject_consumed(&stored)?; + reject_terminal(&stored)?; + if stored.deadline.as_ref().map(|grant| grant.request_digest) != Some(expected_request) { + return Err(conflict("metadata operation does not bind this preparation duration")); + } + require_unelapsed(&txn, &stored).await?; return stored.intent(); } let id = uuid::Uuid::new_v4().to_string(); @@ -171,8 +227,8 @@ impl PostgresMetadataInstallRepository { "INSERT INTO mst2_metadata_prepare (prepare_id, operation_id, manifest_digest, canonical_plan, source_domain, tagged_root_tree_oid, scope, schema_version, metadata_codec, materialization_policy, fs_semantics, access_projection, verification_revision, projection_revision, metadata_root, - node_count, edge_count, total_bytes, state) - VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,'PREPARING')", + node_count, edge_count, total_bytes, state, deadline_managed) + VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,'PREPARING',true)", [id.clone().into(), operation_id.into(), digest.to_vec().into(), plan.encode()?.into(), identity.source_domain.clone().into(), identity.tagged_root_tree_oid.clone().into(), identity.scope.clone().into(), (identity.schema_version as i16).into(), (identity.metadata_codec as i16).into(), @@ -187,15 +243,16 @@ impl PostgresMetadataInstallRepository { SELECT $1,decode(p.page_id,'hex'),p.size FROM jsonb_to_recordset($2::jsonb) AS p(page_id text,size integer)", [id.clone().into(), serde_json::to_string(&pages).map_err(internal)?.into()], )).await.map_err(internal)?; - Ok(MetadataPrepareIntent { prepare_id:id, operation_id:operation_id.into(), manifest_digest:digest }) + let deadline = PrepareDeadline::new( + self.storage_uuid(), &id, operation_id, digest, duration_ms, clock(&txn).await?, + )?; + deadline.insert(&txn).await?; + Ok(MetadataPrepareIntent { prepare_id:id, operation_id:operation_id.into(), manifest_digest:digest, + deadline_digest:Some(deadline.grant_digest) }) }.await; - commit( - txn, - result, - operation_id, - digest, - MetadataCommitPhase::Intent, - ) + commit(txn, result, |intent| { + intent.recovery(MetadataCommitPhase::Intent) + }) .await } @@ -208,6 +265,7 @@ impl PostgresMetadataInstallRepository { let txn = self.transaction().await?; let result = async { self.barrier(&txn).await?; + lock_prepare_operation(&txn, &intent.operation_id).await?; let stored = require_plan(&txn, intent).await?; if stored.plan.pages.get(&payload.id) != Some(&payload.size) { return Err(integrity( @@ -242,13 +300,9 @@ impl PostgresMetadataInstallRepository { Ok(()) } .await; - commit( - txn, - result, - &intent.operation_id, - intent.manifest_digest, - MetadataCommitPhase::Payload, - ) + commit(txn, result, |_| { + intent.recovery(MetadataCommitPhase::Payload) + }) .await } @@ -269,13 +323,9 @@ impl PostgresMetadataInstallRepository { let dag = self.load_installed_dag(intent).await?; let txn = self.transaction().await?; let result = self.finalize_in_txn(&txn, intent, &dag).await; - commit( - txn, - result, - &intent.operation_id, - intent.manifest_digest, - MetadataCommitPhase::Finalize, - ) + commit(txn, result, |_| { + intent.recovery(MetadataCommitPhase::Finalize) + }) .await } @@ -288,6 +338,7 @@ impl PostgresMetadataInstallRepository { dag: &ValidatedMetadataDag, ) -> Result { self.barrier(txn).await?; + lock_prepare_operation(txn, &intent.operation_id).await?; let stored = require_plan(txn, intent).await?; let expected: BTreeSet<_> = dag .payloads() @@ -353,51 +404,100 @@ impl PostgresMetadataInstallRepository { pub async fn inspect_prepare( &self, fresh_primary: &DatabaseConnection, - operation_id: &str, - digest: [u8; 32], - phase: MetadataCommitPhase, + recovery: &MetadataPrepareRecovery, ) -> Result { + let operation_id = &recovery.operation_id; + let digest = recovery.manifest_digest; validate_operation_id(operation_id)?; let txn = fresh_primary .begin_with_config(Some(IsolationLevel::ReadCommitted), None) .await - .map_err(|_| uncertain(operation_id, digest, phase))?; + .map_err(|_| uncertain(recovery))?; if self.barrier(&txn).await.is_err() { let _ = txn.rollback().await; - return Err(uncertain(operation_id, digest, phase)); + return Err(uncertain(recovery)); } let result = async { + lock_prepare_operation(&txn, operation_id).await?; match load_plan(&txn, operation_id, &digest).await? { None => Ok(MetadataPrepareObservation::Absent), - Some(stored) if stored.record.state == "COMMITTED" => { - // Recovery rechecks the bounded bytes/DAG under the barrier, - // so corruption cannot turn a stored state into a receipt. - // This may read 64 MiB; normal finalization verifies outside - // its short graph transaction instead. - load_installed_dag(&txn, &stored).await?; - check_payload_coverage(&txn, &stored).await?; - verify_graph(&txn, &stored).await?; - Ok(MetadataPrepareObservation::Committed(stored.receipt()?)) - } Some(stored) => { - reject_consumed(&stored)?; - Ok(MetadataPrepareObservation::Preparing(stored.intent()?)) + let matches = match &recovery.binding { + PrepareRecoveryBinding::Exact(intent) => &stored.intent()? == intent, + PrepareRecoveryBinding::Request(digest) => { + stored.deadline.as_ref().map(|grant| grant.request_digest) + == Some(*digest) + } + }; + if !matches { + return Err(conflict( + "metadata recovery does not bind this fixed preparation grant", + )); + } + reject_terminal(&stored)?; + if let Some(grant) = &stored.deadline + && clock(&txn).await? >= grant.expires_at_ms + { + return Ok(MetadataPrepareObservation::DeadlinePassed { + intent: stored.intent()?, + expires_at_ms: grant.expires_at_ms, + }); + } + if stored.record.state == "COMMITTED" { + // Recovery rechecks bounded bytes/DAG under the barrier; + // corruption cannot turn a stored state into a receipt. + load_installed_dag(&txn, &stored).await?; + check_payload_coverage(&txn, &stored).await?; + verify_graph(&txn, &stored).await?; + Ok(MetadataPrepareObservation::Committed(stored.receipt()?)) + } else { + Ok(MetadataPrepareObservation::Preparing(stored.intent()?)) + } } } } .await; - txn.rollback() - .await - .map_err(|_| uncertain(operation_id, digest, phase))?; + txn.rollback().await.map_err(|_| uncertain(recovery))?; result.map_err(|error: SnapshotError| { if error.code == SnapshotErrorCode::Internal { - uncertain(operation_id, digest, phase) + uncertain(recovery) } else { MetadataInstallError::Rejected(error) } }) } + /// A process restart may retain the original operation and requested duration + /// without a server-issued intent. Recovery must still match that fixed request. + pub fn recovery_request( + &self, + operation: &str, + digest: [u8; 32], + phase: MetadataCommitPhase, + ) -> Result { + self.recovery_request_with_duration(operation, digest, phase, DEFAULT_PREPARE_DURATION_MS) + } + + pub(crate) fn recovery_request_with_duration( + &self, + operation: &str, + digest: [u8; 32], + phase: MetadataCommitPhase, + duration_ms: i64, + ) -> Result { + Ok(MetadataPrepareRecovery { + operation_id: operation.into(), + manifest_digest: digest, + phase, + binding: PrepareRecoveryBinding::Request(request_digest( + self.storage_uuid(), + operation, + &digest, + duration_ms, + )?), + }) + } + pub(super) fn storage_uuid(&self) -> &str { &self.storage_scope.storage_uuid } @@ -538,8 +638,13 @@ pub(super) async fn load_plan( || record.node_count as usize != plan.pages.len() || record.edge_count as usize != plan.edges.len() || record.total_bytes as u64 != plan.total_bytes - || !["PREPARING", "COMMITTED", "CONSUMED"].contains(&record.state.as_str()) - || (record.state != "PREPARING") != record.committed_at.is_some() + || !["PREPARING", "COMMITTED", "CONSUMED", "EXPIRED"].contains(&record.state.as_str()) + || match record.state.as_str() { + "PREPARING" => record.committed_at.is_some(), + "COMMITTED" | "CONSUMED" => record.committed_at.is_none(), + "EXPIRED" => !record.deadline_managed, + _ => true, + } { return Err(integrity( "stored metadata preparation fields disagree with their canonical plan", @@ -570,7 +675,14 @@ pub(super) async fn load_plan( "stored metadata preparation coverage differs from its canonical plan", )); } - Ok(Some(StoredPlan { record, plan })) + let deadline = load_deadline(connection, &record, digest).await?; + let stored = StoredPlan { + record, + plan, + deadline, + }; + load_expiry(connection, &stored).await?; + Ok(Some(stored)) } async fn require_plan( @@ -581,23 +693,48 @@ async fn require_plan( let stored = load_plan(connection, &intent.operation_id, &intent.manifest_digest) .await? .ok_or_else(|| unavailable("metadata preparation intent is not durable"))?; - if stored.record.prepare_id != intent.prepare_id { + if stored.record.prepare_id != intent.prepare_id + || stored.deadline.as_ref().map(|grant| grant.grant_digest) != intent.deadline_digest + { return Err(integrity("metadata intent identity mismatch")); } - reject_consumed(&stored)?; + reject_terminal(&stored)?; + require_unelapsed(connection, &stored).await?; Ok(stored) } -fn reject_consumed(stored: &StoredPlan) -> Result<(), SnapshotError> { - if stored.record.state == "CONSUMED" { - return Err(SnapshotError::new( - SnapshotErrorCode::Conflict, - "metadata preparation has been consumed into a lease binding", - )); +fn reject_terminal(stored: &StoredPlan) -> Result<(), SnapshotError> { + if ["CONSUMED", "EXPIRED"].contains(&stored.record.state.as_str()) { + return Err(conflict("metadata preparation is terminal")); } Ok(()) } +pub(super) async fn require_unelapsed( + connection: &C, + stored: &StoredPlan, +) -> Result<(), SnapshotError> { + if let Some(grant) = &stored.deadline + && clock(connection).await? >= grant.expires_at_ms + { + return Err(conflict("metadata preparation deadline has passed")); + } + Ok(()) +} + +pub(super) async fn lock_prepare_operation( + txn: &DatabaseTransaction, + operation: &str, +) -> Result<(), SnapshotError> { + txn.query_one_raw(statement( + "SELECT prepare_id FROM mst2_metadata_prepare WHERE operation_id=$1 FOR UPDATE", + [operation.into()], + )) + .await + .map_err(internal)?; + Ok(()) +} + pub(super) async fn load_installed_dag( connection: &C, stored: &StoredPlan, @@ -775,7 +912,7 @@ fn validate_payload(payload: &MetadataPagePayload) -> Result<(), SnapshotError> Ok(()) } -fn validate_operation_id(operation: &str) -> Result<(), SnapshotError> { +pub(super) fn validate_operation_id(operation: &str) -> Result<(), SnapshotError> { if operation.is_empty() || operation.len() > 255 || operation.contains('\0') { return Err(SnapshotError::new( SnapshotErrorCode::InvalidRequest, @@ -788,15 +925,12 @@ fn validate_operation_id(operation: &str) -> Result<(), SnapshotError> { async fn commit( txn: DatabaseTransaction, result: Result, - operation: &str, - digest: [u8; 32], - phase: MetadataCommitPhase, + recovery: impl FnOnce(&T) -> MetadataPrepareRecovery, ) -> Result { match result { Ok(value) => { - txn.commit() - .await - .map_err(|_| uncertain(operation, digest, phase))?; + let recovery = recovery(&value); + txn.commit().await.map_err(|_| uncertain(&recovery))?; Ok(value) } Err(error) => { @@ -805,15 +939,12 @@ async fn commit( } } } -fn uncertain( - operation: &str, - digest: [u8; 32], - phase: MetadataCommitPhase, -) -> MetadataInstallError { +fn uncertain(recovery: &MetadataPrepareRecovery) -> MetadataInstallError { MetadataInstallError::CommitUncertain { - operation_id: operation.into(), - manifest_digest: digest, - phase, + operation_id: recovery.operation_id.clone(), + manifest_digest: recovery.manifest_digest, + phase: recovery.phase, + recovery: Box::new(recovery.clone()), } } fn node_id(id: &[u8; 32]) -> String { @@ -831,6 +962,9 @@ fn integrity(message: &str) -> SnapshotError { fn unavailable(message: &str) -> SnapshotError { SnapshotError::new(SnapshotErrorCode::ObjectUnavailable, message) } +fn conflict(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::Conflict, message) +} #[cfg(test)] #[path = "native_metadata_install_tests.rs"] diff --git a/src/jupiter/storage/native_metadata_install_tests.rs b/src/jupiter/storage/native_metadata_install_tests.rs index bba815aa..b1ef1905 100644 --- a/src/jupiter/storage/native_metadata_install_tests.rs +++ b/src/jupiter/storage/native_metadata_install_tests.rs @@ -22,7 +22,7 @@ use crate::{ }, }; -fn prepared(scope: &str) -> PreparedNativeMetadataRetention { +pub(in crate::jupiter::storage) fn prepared(scope: &str) -> PreparedNativeMetadataRetention { let child_entries = [Entry::file(EntryKind::Regular, b"file", 3, [42; 32])]; let child = Page::build(&child_entries).unwrap(); let entries = [ @@ -39,7 +39,7 @@ fn prepared(scope: &str) -> PreparedNativeMetadataRetention { ) } -async fn fixture() -> ( +pub(in crate::jupiter::storage) async fn fixture() -> ( DatabaseConnection, DatabaseConnection, TestSchemaGuard, @@ -53,7 +53,7 @@ async fn fixture() -> ( (first, second, schema, config.db_url) } -async fn install( +pub(in crate::jupiter::storage) async fn install( repository: &PostgresMetadataInstallRepository, intent: &MetadataPrepareIntent, prepared: &PreparedNativeMetadataRetention, @@ -63,6 +63,44 @@ async fn install( } } +#[tokio::test] +async fn native_metadata_recovery_requires_the_original_preparation_and_deadline_grant() { + let (first, second, _schema, _url) = fixture().await; + let repo = PostgresMetadataInstallRepository::new(first.clone()) + .await + .unwrap(); + let pages = prepared("/"); + let original = repo.begin_intent("original-witness", &pages).await.unwrap(); + let other = repo.begin_intent("other-witness", &pages).await.unwrap(); + assert_eq!(original.manifest_digest(), other.manifest_digest()); + let mut wrong = original.clone(); + wrong.prepare_id = other.prepare_id.clone(); + assert_eq!( + rejected( + repo.inspect_prepare(&second, &wrong.recovery(MetadataCommitPhase::Intent)) + .await + .unwrap_err() + ), + SnapshotErrorCode::Conflict + ); + wrong = original.clone(); + wrong.deadline_digest = other.deadline_digest; + assert_eq!( + rejected( + repo.inspect_prepare(&second, &wrong.recovery(MetadataCommitPhase::Intent)) + .await + .unwrap_err() + ), + SnapshotErrorCode::Conflict + ); + assert_eq!( + repo.inspect_prepare(&second, &original.recovery(MetadataCommitPhase::Intent)) + .await + .unwrap(), + MetadataPrepareObservation::Preparing(original) + ); +} + fn rejected(error: MetadataInstallError) -> SnapshotErrorCode { match error { MetadataInstallError::Rejected(error) => error.code, @@ -285,12 +323,7 @@ async fn native_metadata_partial_installation_survives_repository_and_connection .unwrap(); assert!(matches!( restarted - .inspect_prepare( - &second, - intent.operation_id(), - intent.manifest_digest(), - MetadataCommitPhase::Payload - ) + .inspect_prepare(&second, &intent.recovery(MetadataCommitPhase::Payload)) .await .unwrap(), MetadataPrepareObservation::Preparing(_) @@ -386,7 +419,12 @@ async fn native_metadata_half_installation_survives_real_process_kill() { let prepared = prepared("/"); let digest = prepared.install_plan().unwrap().digest().unwrap(); let original = match restarted - .inspect_prepare(&second, OPERATION, digest, MetadataCommitPhase::Payload) + .inspect_prepare( + &second, + &restarted + .recovery_request(OPERATION, digest, MetadataCommitPhase::Payload) + .unwrap(), + ) .await .unwrap() { @@ -505,12 +543,7 @@ async fn native_metadata_committed_recovery_rejects_same_length_corruption_and_p assert_eq!( rejected( repository - .inspect_prepare( - &second, - intent.operation_id(), - intent.manifest_digest(), - MetadataCommitPhase::Finalize - ) + .inspect_prepare(&second, &intent.recovery(MetadataCommitPhase::Finalize)) .await .unwrap_err() ), @@ -549,12 +582,7 @@ async fn native_metadata_committed_recovery_rejects_same_length_corruption_and_p overwrite_installed_payload_for_test(&first, page, page.bytes.clone()).await; assert_eq!( repository - .inspect_prepare( - &second, - intent.operation_id(), - intent.manifest_digest(), - MetadataCommitPhase::Finalize - ) + .inspect_prepare(&second, &intent.recovery(MetadataCommitPhase::Finalize)) .await .unwrap(), MetadataPrepareObservation::Committed(receipt) @@ -598,12 +626,7 @@ async fn native_metadata_outer_rollback_preserves_intent_but_exposes_no_graph_or txn.rollback().await.unwrap(); assert!(matches!( repository - .inspect_prepare( - &second, - intent.operation_id(), - intent.manifest_digest(), - MetadataCommitPhase::Finalize - ) + .inspect_prepare(&second, &intent.recovery(MetadataCommitPhase::Finalize)) .await .unwrap(), MetadataPrepareObservation::Preparing(_) @@ -690,12 +713,7 @@ async fn native_metadata_receipt_replay_rejects_lost_pin_without_recreating_it() assert_eq!( rejected( repository - .inspect_prepare( - &second, - intent.operation_id(), - intent.manifest_digest(), - MetadataCommitPhase::Finalize - ) + .inspect_prepare(&second, &intent.recovery(MetadataCommitPhase::Finalize)) .await .unwrap_err() ), @@ -739,12 +757,7 @@ async fn native_metadata_recovery_lock_timeout_is_unknown_and_never_releases_pin recovery.barrier_timeout = Duration::from_millis(25); assert!(matches!( recovery - .inspect_prepare( - &second, - intent.operation_id(), - intent.manifest_digest(), - MetadataCommitPhase::Finalize - ) + .inspect_prepare(&second, &intent.recovery(MetadataCommitPhase::Finalize)) .await .unwrap_err(), MetadataInstallError::CommitUncertain { @@ -762,12 +775,7 @@ async fn native_metadata_recovery_lock_timeout_is_unknown_and_never_releases_pin blocker.rollback().await.unwrap(); assert!(matches!( recovery - .inspect_prepare( - &second, - intent.operation_id(), - intent.manifest_digest(), - MetadataCommitPhase::Finalize - ) + .inspect_prepare(&second, &intent.recovery(MetadataCommitPhase::Finalize)) .await .unwrap(), MetadataPrepareObservation::Committed(_) @@ -791,12 +799,7 @@ async fn native_metadata_stored_identity_and_plan_tampering_fail_closed_on_resta assert_eq!( rejected( repository - .inspect_prepare( - &second, - intent.operation_id(), - intent.manifest_digest(), - MetadataCommitPhase::Intent - ) + .inspect_prepare(&second, &intent.recovery(MetadataCommitPhase::Intent)) .await .unwrap_err() ), @@ -806,12 +809,7 @@ async fn native_metadata_stored_identity_and_plan_tampering_fail_closed_on_resta assert_eq!( rejected( repository - .inspect_prepare( - &second, - intent.operation_id(), - intent.manifest_digest(), - MetadataCommitPhase::Intent - ) + .inspect_prepare(&second, &intent.recovery(MetadataCommitPhase::Intent)) .await .unwrap_err() ), @@ -1043,12 +1041,7 @@ async fn commit_fault_case(fault: Fault) { )); proxy.wait_for_fault().await; let observed = repository - .inspect_prepare( - &recovery, - intent.operation_id(), - intent.manifest_digest(), - MetadataCommitPhase::Finalize, - ) + .inspect_prepare(&recovery, &intent.recovery(MetadataCommitPhase::Finalize)) .await .unwrap(); match fault { @@ -1136,9 +1129,7 @@ async fn native_metadata_wrong_schema_primary_cannot_report_absent_for_another_c repository .inspect_prepare( &wrong_primary, - intent.operation_id(), - intent.manifest_digest(), - MetadataCommitPhase::Finalize + &intent.recovery(MetadataCommitPhase::Finalize) ) .await .unwrap_err(), @@ -1160,12 +1151,7 @@ async fn native_metadata_wrong_schema_primary_cannot_report_absent_for_another_c ); assert!(matches!( repository - .inspect_prepare( - &second, - intent.operation_id(), - intent.manifest_digest(), - MetadataCommitPhase::Finalize - ) + .inspect_prepare(&second, &intent.recovery(MetadataCommitPhase::Finalize)) .await .unwrap(), MetadataPrepareObservation::Committed(_) @@ -1192,12 +1178,7 @@ async fn native_metadata_query_loss_after_recovery_barrier_preserves_typed_unkno proxy.armed.store(true, Ordering::SeqCst); assert!(matches!( repository - .inspect_prepare( - &fresh, - intent.operation_id(), - intent.manifest_digest(), - MetadataCommitPhase::Finalize - ) + .inspect_prepare(&fresh, &intent.recovery(MetadataCommitPhase::Finalize)) .await .unwrap_err(), MetadataInstallError::CommitUncertain { .. } @@ -1213,12 +1194,7 @@ async fn native_metadata_query_loss_after_recovery_barrier_preserves_typed_unkno ); assert!(matches!( repository - .inspect_prepare( - &second, - intent.operation_id(), - intent.manifest_digest(), - MetadataCommitPhase::Finalize - ) + .inspect_prepare(&second, &intent.recovery(MetadataCommitPhase::Finalize)) .await .unwrap(), MetadataPrepareObservation::Committed(_) diff --git a/src/jupiter/storage/native_metadata_lease.rs b/src/jupiter/storage/native_metadata_lease.rs index fb2f1040..92c472f6 100644 --- a/src/jupiter/storage/native_metadata_lease.rs +++ b/src/jupiter/storage/native_metadata_lease.rs @@ -13,7 +13,8 @@ use super::{ mst2_retention::PostgresRetentionRepository, native_metadata_install::{ PostgresMetadataInstallRepository, PreparedMetadataReceipt, StoredPlan, - check_payload_coverage, load_installed_dag, load_plan, verify_graph_root, + check_payload_coverage, load_installed_dag, load_plan, require_unelapsed, + verify_graph_root, }, }; use crate::ceres::snapshot::{ @@ -45,6 +46,7 @@ pub(crate) struct VerifiedMetadataHandoff { prepare_id: String, install_operation: String, install_digest: [u8; 32], + deadline_digest: Option<[u8; 32]>, storage_uuid: String, plan: MetadataInstallPlan, dag: ValidatedMetadataDag, @@ -148,8 +150,11 @@ impl PostgresMetadataLeaseRepository { ) .await? .ok_or_else(|| unavailable("metadata preparation is absent"))?; + if !["COMMITTED", "CONSUMED"].contains(&stored.record.state.as_str()) { + return Err(conflict("metadata handoff preparation is not committed")); + } if stored.record.prepare_id != intent.prepare_id() - || stored.record.state == "PREPARING" + || stored.deadline.as_ref().map(|grant| grant.grant_digest) != intent.deadline_digest() || stored.plan.root != receipt.metadata_root() || stored.plan.total_bytes != receipt.payload_bytes() { @@ -157,6 +162,9 @@ impl PostgresMetadataLeaseRepository { "metadata handoff does not bind its durable preparation", )); } + if stored.record.state == "COMMITTED" { + require_unelapsed(self.install.connection(), &stored).await?; + } let descriptor = checked_descriptor(&binding, &stored.plan)?; let snapshot_id = descriptor.snapshot_id().map_err(internal)?; let dag = load_installed_dag(self.install.connection(), &stored).await?; @@ -166,6 +174,7 @@ impl PostgresMetadataLeaseRepository { prepare_id: stored.record.prepare_id, install_operation: intent.operation_id().into(), install_digest: intent.manifest_digest(), + deadline_digest: intent.deadline_digest(), storage_uuid: self.install.storage_uuid().into(), plan: stored.plan, dag, @@ -187,7 +196,10 @@ impl PostgresMetadataLeaseRepository { ) .await? .ok_or_else(|| unavailable("metadata handoff preparation is absent"))?; - if stored.record.prepare_id != handoff.prepare_id || stored.plan != handoff.plan { + if stored.record.prepare_id != handoff.prepare_id + || stored.plan != handoff.plan + || stored.deadline.as_ref().map(|grant| grant.grant_digest) != handoff.deadline_digest + { return Err(integrity("metadata handoff changed before consumption").into()); } // Revalidate the current bounded bytes for each mutation attempt; @@ -239,7 +251,10 @@ impl PostgresMetadataLeaseRepository { let stored = load_plan(txn, &handoff.install_operation, &handoff.install_digest) .await? .ok_or_else(|| unavailable("handoff plan is absent"))?; - if stored.record.prepare_id != handoff.prepare_id || stored.plan != handoff.plan { + if stored.record.prepare_id != handoff.prepare_id + || stored.plan != handoff.plan + || stored.deadline.as_ref().map(|grant| grant.grant_digest) != handoff.deadline_digest + { return Err(integrity("durable handoff plan changed")); } require_access(txn, access).await?; @@ -257,6 +272,7 @@ impl PostgresMetadataLeaseRepository { "preparation is not an unconsumed committed handoff", )); } + require_unelapsed(txn, &stored).await?; check_payload_coverage(txn, &stored).await?; verify_graph_root( txn, diff --git a/src/jupiter/storage/native_metadata_lease_tests.rs b/src/jupiter/storage/native_metadata_lease_tests.rs index 1a0d3bfa..cb4a6c4e 100644 --- a/src/jupiter/storage/native_metadata_lease_tests.rs +++ b/src/jupiter/storage/native_metadata_lease_tests.rs @@ -316,6 +316,88 @@ fn rejection(error: MetadataLeaseError) -> SnapshotErrorCode { } } +#[tokio::test] +async fn metadata_lease_consumed_prepare_deadline_cannot_expire_or_break_operation_replay() { + let (first, second, _schema) = fixture().await; + let install = PostgresMetadataInstallRepository::new(first.clone()) + .await + .unwrap(); + let pages = prepared(); + let intent = install + .begin_intent_with_duration("consumed-deadline", &pages, 3000) + .await + .unwrap(); + for page in pages.dag().payloads() { + install.install_page(&intent, page).await.unwrap(); + } + let prepared_receipt = install.finalize(&intent).await.unwrap(); + let ledger = PostgresMetadataLeaseRepository::new(first.clone()) + .await + .unwrap(); + let handoff = ledger + .verify_handoff( + binding(prepared_receipt.metadata_root(), 1), + &prepared_receipt, + ) + .await + .unwrap(); + let access = access(&first).await; + let lease = ledger + .consume("consume-before-deadline", &handoff, &access, 60_000) + .await + .unwrap(); + let grant = load_plan(&first, intent.operation_id(), &intent.manifest_digest()) + .await + .unwrap() + .unwrap() + .deadline + .unwrap(); + tokio::time::timeout(Duration::from_secs(10), async { + while super::super::native_metadata_prepare_expiry::clock(&first) + .await + .unwrap() + < grant.expires_at_ms + { + tokio::time::sleep(Duration::from_millis(50)).await; + } + }) + .await + .unwrap(); + assert!( + matches!(install.expire_prepare("expire-consumed", &intent).await.unwrap_err(), + super::super::native_metadata_prepare_expiry::PrepareExpiryError::Rejected(error) + if error.code==SnapshotErrorCode::Conflict) + ); + assert_eq!( + ledger + .consume("consume-before-deadline", &handoff, &access, 60_000) + .await + .unwrap(), + lease + ); + assert!( + matches!(ledger.inspect_operation(&second, lease.operation_id(), lease.operation_digest()).await.unwrap(), + MetadataLeaseObservation::Committed { event, state, .. } if *event==lease && state=="ACTIVE") + ); + assert_eq!(count(&first, "mst2_metadata_prepare_expiry").await, 0); + assert_eq!(count(&first, "mst2_metadata_prepare_consumption").await, 1); + assert_eq!(count(&first, "mst2_metadata_lease_operation").await, 1); + assert_eq!(count(&first, "mst2_retention_root").await, 2); + let stored = load_plan(&first, intent.operation_id(), &intent.manifest_digest()) + .await + .unwrap() + .unwrap(); + assert_eq!(stored.record.state, "CONSUMED"); + verify_graph_root( + &second, + &stored, + &format!("lease:{}", lease.lease_id()), + "lease", + ) + .await + .unwrap(); +} + #[test] fn metadata_lease_descriptor_fixture_requires_exact_plan_sid_profile_and_tagged_source() { let prepared = prepared(); @@ -391,8 +473,7 @@ async fn metadata_lease_two_real_primary_connections_consume_once_and_recover_ex super::super::native_metadata_install::MetadataInstallError::Rejected(error) if error.code==SnapshotErrorCode::Conflict) ); assert!( - matches!(install.inspect_prepare(&second,prepared_receipt.intent().operation_id(),prepared_receipt.intent().manifest_digest(), - super::super::native_metadata_install::MetadataCommitPhase::Finalize).await.unwrap_err(), + matches!(install.inspect_prepare(&second, &prepared_receipt.intent().recovery(super::super::native_metadata_install::MetadataCommitPhase::Finalize)).await.unwrap_err(), super::super::native_metadata_install::MetadataInstallError::Rejected(error) if error.code==SnapshotErrorCode::Conflict) ); } diff --git a/src/jupiter/storage/native_metadata_prepare_expiry.rs b/src/jupiter/storage/native_metadata_prepare_expiry.rs new file mode 100644 index 00000000..1422bf9c --- /dev/null +++ b/src/jupiter/storage/native_metadata_prepare_expiry.rs @@ -0,0 +1,589 @@ +//! Private preparation deadlines and terminal metadata-only pin release. + +use sea_orm::{ + ConnectionTrait, DatabaseConnection, DatabaseTransaction, DbBackend, IsolationLevel, Statement, + TransactionTrait, +}; +use sha2::{Digest, Sha256}; + +use super::{ + mst2_retention::PostgresRetentionRepository, + native_metadata_install::{ + MetadataPrepareIntent, PostgresMetadataInstallRepository, StoredPlan, + check_payload_coverage, load_installed_dag, load_plan, lock_prepare_operation, + validate_operation_id, verify_graph_root, + }, +}; +use crate::{ + callisto::mst2_metadata_prepare, + ceres::snapshot::{ + error::{SnapshotError, SnapshotErrorCode}, + retention::RetentionRoot, + }, +}; + +pub(super) const DEFAULT_PREPARE_DURATION_MS: i64 = 3_600_000; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct PrepareDeadline { + pub(super) storage_uuid: String, + pub(super) prepare_id: String, + pub(super) operation_id: String, + pub(super) manifest_digest: [u8; 32], + pub(super) request_digest: [u8; 32], + pub(super) grant_digest: [u8; 32], + pub(super) duration_ms: i64, + pub(super) granted_at_ms: i64, + pub(super) expires_at_ms: i64, +} + +pub(super) fn request_digest( + storage_uuid: &str, + operation_id: &str, + manifest_digest: &[u8; 32], + duration_ms: i64, +) -> Result<[u8; 32], SnapshotError> { + validate_operation_id(operation_id)?; + if !(1..=DEFAULT_PREPARE_DURATION_MS).contains(&duration_ms) { + return Err(SnapshotError::new( + SnapshotErrorCode::InvalidRequest, + "metadata preparation duration must be 1..=3600000 milliseconds", + )); + } + let storage = uuid::Uuid::parse_str(storage_uuid).map_err(internal)?; + if storage.is_nil() || storage.to_string() != storage_uuid { + return Err(integrity("invalid preparation storage UUID")); + } + Ok(framed_digest( + b"mega.mst2.prepare-request.v1\0", + &[ + storage_uuid.as_bytes(), + operation_id.as_bytes(), + manifest_digest, + &duration_ms.to_be_bytes(), + ], + )) +} + +impl PrepareDeadline { + pub(super) fn new( + storage_uuid: &str, + prepare_id: &str, + operation_id: &str, + manifest_digest: [u8; 32], + duration_ms: i64, + granted_at_ms: i64, + ) -> Result { + let request_digest = + request_digest(storage_uuid, operation_id, &manifest_digest, duration_ms)?; + let id = uuid::Uuid::parse_str(prepare_id).map_err(internal)?; + if id.is_nil() || id.to_string() != prepare_id || granted_at_ms <= 0 { + return Err(integrity("invalid preparation deadline identity or clock")); + } + let expires_at_ms = granted_at_ms + .checked_add(duration_ms) + .ok_or_else(|| integrity("preparation deadline overflow"))?; + let grant_digest = framed_digest( + b"mega.mst2.prepare-grant.v1\0", + &[ + &request_digest, + prepare_id.as_bytes(), + &granted_at_ms.to_be_bytes(), + &expires_at_ms.to_be_bytes(), + ], + ); + Ok(Self { + storage_uuid: storage_uuid.into(), + prepare_id: prepare_id.into(), + operation_id: operation_id.into(), + manifest_digest, + request_digest, + grant_digest, + duration_ms, + granted_at_ms, + expires_at_ms, + }) + } + + pub(super) async fn insert(&self, txn: &DatabaseTransaction) -> Result<(), SnapshotError> { + txn.execute_raw(statement( + "INSERT INTO mst2_metadata_prepare_deadline(prepare_id,storage_uuid,operation_id,manifest_digest, + request_digest,grant_digest,duration_ms,granted_at_ms,expires_at_ms) + VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9)", + [self.prepare_id.clone().into(),self.storage_uuid.clone().into(),self.operation_id.clone().into(), + self.manifest_digest.to_vec().into(),self.request_digest.to_vec().into(),self.grant_digest.to_vec().into(), + self.duration_ms.into(),self.granted_at_ms.into(),self.expires_at_ms.into()], + )).await.map_err(internal)?; + Ok(()) + } +} + +pub(super) async fn load_deadline( + connection: &C, + record: &mst2_metadata_prepare::Model, + manifest_digest: &[u8; 32], +) -> Result, SnapshotError> { + let row = connection.query_one_raw(statement( + "SELECT d.*,s.storage_uuid AS current_storage_uuid FROM mst2_metadata_prepare_deadline d + JOIN mst2_metadata_storage_scope s ON s.singleton=1 WHERE d.prepare_id=$1", + [record.prepare_id.clone().into()], + )).await.map_err(internal)?; + let Some(row) = row else { + if record.deadline_managed { + return Err(integrity( + "managed preparation is missing its immutable deadline", + )); + } + return Ok(None); + }; + if !record.deadline_managed { + return Err(integrity("historical preparation has an unbound deadline")); + } + let storage_uuid: String = row.try_get("", "storage_uuid").map_err(internal)?; + if storage_uuid + != row + .try_get::("", "current_storage_uuid") + .map_err(internal)? + || row.try_get::("", "prepare_id").map_err(internal)? != record.prepare_id + || row + .try_get::("", "operation_id") + .map_err(internal)? + != record.operation_id + || row + .try_get::>("", "manifest_digest") + .map_err(internal)? + .as_slice() + != manifest_digest + { + return Err(integrity( + "preparation deadline does not bind its storage and fixed plan", + )); + } + let duration_ms = row.try_get("", "duration_ms").map_err(internal)?; + let granted_at_ms = row.try_get("", "granted_at_ms").map_err(internal)?; + let expected = PrepareDeadline::new( + &storage_uuid, + &record.prepare_id, + &record.operation_id, + *manifest_digest, + duration_ms, + granted_at_ms, + ) + .map_err(|_| integrity("stored preparation deadline parameters are invalid"))?; + if row + .try_get::>("", "request_digest") + .map_err(internal)? + != expected.request_digest + || row + .try_get::>("", "grant_digest") + .map_err(internal)? + != expected.grant_digest + || row.try_get::("", "expires_at_ms").map_err(internal)? != expected.expires_at_ms + { + return Err(integrity( + "preparation deadline fields disagree with their canonical grant", + )); + } + Ok(Some(expected)) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PrepareExpiryRequest { + operation_id: String, + operation_digest: [u8; 32], + intent: MetadataPrepareIntent, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct PrepareExpiryReceipt { + request: PrepareExpiryRequest, + previous_state: String, + expires_at_ms: i64, + expired_at_ms: i64, +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum PrepareExpiryError { + #[error(transparent)] + Rejected(#[from] SnapshotError), + #[error("preparation expiry commit outcome is unknown")] + CommitUncertain { recovery: PrepareExpiryRequest }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum PrepareExpiryObservation { + NotExpired { state: String }, + Expired(PrepareExpiryReceipt), +} + +fn expiry_request( + storage_uuid: &str, + operation: &str, + intent: &MetadataPrepareIntent, +) -> Result { + validate_operation_id(operation)?; + let grant_digest = intent.deadline_digest().ok_or_else(|| { + conflict("historical preparation has an unknown deadline and cannot be expired") + })?; + let operation_digest = framed_digest( + b"mega.mst2.prepare-expiry.v1\0", + &[ + storage_uuid.as_bytes(), + operation.as_bytes(), + intent.prepare_id().as_bytes(), + intent.operation_id().as_bytes(), + &intent.manifest_digest(), + &grant_digest, + ], + ); + Ok(PrepareExpiryRequest { + operation_id: operation.into(), + operation_digest, + intent: intent.clone(), + }) +} + +impl PostgresMetadataInstallRepository { + pub(crate) async fn expire_prepare( + &self, + operation: &str, + intent: &MetadataPrepareIntent, + ) -> Result { + let request = expiry_request(self.storage_uuid(), operation, intent)?; + let txn = self.transaction().await?; + let result = self.expire_prepare_in_txn(&txn, &request).await; + match result { + Ok(receipt) => { + txn.commit().await.map_err(|_| expiry_uncertain(&request))?; + Ok(receipt) + } + Err(error) => { + txn.rollback() + .await + .map_err(|_| expiry_uncertain(&request))?; + Err(error.into()) + } + } + } + + // Only the successful outer COMMIT or a fresh-primary recovery may expose this receipt. + async fn expire_prepare_in_txn( + &self, + txn: &DatabaseTransaction, + request: &PrepareExpiryRequest, + ) -> Result { + self.barrier(txn).await?; + validate_expiry_request(self.storage_uuid(), request)?; + lock_prepare_operation(txn, request.intent.operation_id()).await?; + let stored = expiry_plan(txn, request).await?; + if let Some(event) = load_expiry(txn, &stored).await? { + if event.request != *request { + return Err(conflict( + "preparation was expired by another fixed operation", + )); + } + return Ok(event); + } + if !["PREPARING", "COMMITTED"].contains(&stored.record.state.as_str()) { + return Err(conflict("only an unconsumed preparation can expire")); + } + // Do not grant a new operation ID to a different preparation. + if txn + .query_one_raw(statement( + "SELECT prepare_id FROM mst2_metadata_prepare_expiry WHERE operation_id=$1", + [request.operation_id.clone().into()], + )) + .await + .map_err(internal)? + .is_some() + { + return Err(conflict( + "expiry operation is already bound to another preparation", + )); + } + let grant = stored + .deadline + .as_ref() + .ok_or_else(|| integrity("expiry grant is missing"))?; + let expired_at_ms = clock(txn).await?; + if expired_at_ms < grant.expires_at_ms { + return Err(conflict("preparation deadline has not passed")); + } + if stored.record.state == "COMMITTED" { + load_installed_dag(txn, &stored).await?; + check_payload_coverage(txn, &stored).await?; + verify_graph_root( + txn, + &stored, + &format!("prepare:{}", stored.record.prepare_id), + "prepare", + ) + .await?; + } else { + require_no_prepare_root(txn, &stored.record.prepare_id).await?; + } + require_no_consumption(txn, &stored.record.prepare_id).await?; + let receipt = PrepareExpiryReceipt { + request: request.clone(), + previous_state: stored.record.state.clone(), + expires_at_ms: grant.expires_at_ms, + expired_at_ms, + }; + txn.execute_raw(statement( + "INSERT INTO mst2_metadata_prepare_expiry(prepare_id,operation_id,operation_digest,grant_digest, + previous_state,pin_root,expires_at_ms,expired_at_ms) VALUES($1,$2,$3,$4,$5,$6,$7,$8)", + [stored.record.prepare_id.clone().into(),request.operation_id.clone().into(),request.operation_digest.to_vec().into(), + grant.grant_digest.to_vec().into(),stored.record.state.clone().into(),format!("prepare:{}",stored.record.prepare_id).into(), + grant.expires_at_ms.into(),expired_at_ms.into()], + )).await.map_err(internal)?; + let changed = txn + .execute_raw(statement( + "UPDATE mst2_metadata_prepare SET state='EXPIRED' WHERE prepare_id=$1 AND state=$2", + [ + stored.record.prepare_id.clone().into(), + stored.record.state.clone().into(), + ], + )) + .await + .map_err(internal)?; + if changed.rows_affected() != 1 { + return Err(integrity("preparation changed during expiry")); + } + PostgresRetentionRepository::release_root_in_txn( + txn, + &RetentionRoot::Prepare(stored.record.prepare_id.clone()), + ) + .await?; + // Read the final state rather than the earlier in-memory COMMITTED row. + let final_plan = expiry_plan(txn, request).await?; + if load_expiry(txn, &final_plan).await? != Some(receipt.clone()) { + return Err(integrity( + "expiry event differs from its atomic terminal state", + )); + } + Ok(receipt) + } + + pub(crate) async fn inspect_prepare_expiry( + &self, + fresh_primary: &DatabaseConnection, + recovery: &PrepareExpiryRequest, + ) -> Result { + validate_expiry_request(self.storage_uuid(), recovery)?; + let txn = fresh_primary + .begin_with_config(Some(IsolationLevel::ReadCommitted), None) + .await + .map_err(|_| expiry_uncertain(recovery))?; + if self.barrier(&txn).await.is_err() { + let _ = txn.rollback().await; + return Err(expiry_uncertain(recovery)); + } + let result = async { + lock_prepare_operation(&txn, recovery.intent.operation_id()).await?; + let stored = expiry_plan(&txn, recovery).await?; + match load_expiry(&txn, &stored).await? { + Some(receipt) if receipt.request == *recovery => Ok(PrepareExpiryObservation::Expired(receipt)), + Some(_) => Err(conflict("recovered preparation expiry belongs to another operation")), + None => { + if txn.query_one_raw(statement( + "SELECT prepare_id FROM mst2_metadata_prepare_expiry WHERE operation_id=$1", + [recovery.operation_id.clone().into()], + )).await.map_err(internal)?.is_some() { + return Err(conflict("recovered expiry operation belongs to another preparation")); + } + if stored.record.state == "COMMITTED" { + load_installed_dag(&txn, &stored).await?; + check_payload_coverage(&txn, &stored).await?; + verify_graph_root(&txn, &stored, &format!("prepare:{}", stored.record.prepare_id), "prepare").await?; + } + Ok(PrepareExpiryObservation::NotExpired { state: stored.record.state }) + } + } + }.await; + txn.rollback() + .await + .map_err(|_| expiry_uncertain(recovery))?; + result.map_err(|error: SnapshotError| { + if error.code == SnapshotErrorCode::Internal { + expiry_uncertain(recovery) + } else { + error.into() + } + }) + } +} + +fn validate_expiry_request( + storage_uuid: &str, + request: &PrepareExpiryRequest, +) -> Result<(), SnapshotError> { + if expiry_request(storage_uuid, &request.operation_id, &request.intent)? != *request { + return Err(conflict( + "expiry request does not bind the captured primary storage", + )); + } + Ok(()) +} + +async fn expiry_plan( + connection: &C, + request: &PrepareExpiryRequest, +) -> Result { + let stored = load_plan( + connection, + request.intent.operation_id(), + &request.intent.manifest_digest(), + ) + .await? + .ok_or_else(|| integrity("expiry preparation is missing"))?; + if stored.intent()? != request.intent { + return Err(conflict( + "expiry request does not bind the fixed preparation grant", + )); + } + Ok(stored) +} + +pub(super) async fn load_expiry( + connection: &C, + stored: &StoredPlan, +) -> Result, SnapshotError> { + let row = connection + .query_one_raw(statement( + "SELECT * FROM mst2_metadata_prepare_expiry WHERE prepare_id=$1", + [stored.record.prepare_id.clone().into()], + )) + .await + .map_err(internal)?; + let Some(row) = row else { + if stored.record.state == "EXPIRED" { + return Err(integrity( + "expired preparation is missing its immutable event", + )); + } + return Ok(None); + }; + if stored.record.state != "EXPIRED" { + return Err(integrity("expiry event has no terminal preparation state")); + } + let grant = stored + .deadline + .as_ref() + .ok_or_else(|| integrity("expired preparation has no known grant"))?; + let operation: String = row.try_get("", "operation_id").map_err(internal)?; + let request = expiry_request(&grant.storage_uuid, &operation, &stored.intent()?)?; + let previous_state: String = row.try_get("", "previous_state").map_err(internal)?; + let expires_at_ms: i64 = row.try_get("", "expires_at_ms").map_err(internal)?; + let expired_at_ms: i64 = row.try_get("", "expired_at_ms").map_err(internal)?; + if row + .try_get::>("", "operation_digest") + .map_err(internal)? + != request.operation_digest + || row + .try_get::>("", "grant_digest") + .map_err(internal)? + != grant.grant_digest + || row.try_get::("", "pin_root").map_err(internal)? + != format!("prepare:{}", stored.record.prepare_id) + || expires_at_ms != grant.expires_at_ms + || expired_at_ms < expires_at_ms + || !["PREPARING", "COMMITTED"].contains(&previous_state.as_str()) + || (previous_state == "COMMITTED") != stored.record.committed_at.is_some() + { + return Err(integrity( + "expiry event disagrees with its fixed grant and unconsumed history", + )); + } + require_no_prepare_root(connection, &stored.record.prepare_id).await?; + require_no_consumption(connection, &stored.record.prepare_id).await?; + Ok(Some(PrepareExpiryReceipt { + request, + previous_state, + expires_at_ms, + expired_at_ms, + })) +} + +async fn require_no_prepare_root( + connection: &C, + prepare_id: &str, +) -> Result<(), SnapshotError> { + if connection + .query_one_raw(statement( + "SELECT node_id FROM mst2_retention_root WHERE root_key=$1 LIMIT 1", + [format!("prepare:{prepare_id}").into()], + )) + .await + .map_err(internal)? + .is_some() + { + return Err(integrity( + "terminal or unfinalized preparation still has Prepare pins", + )); + } + Ok(()) +} + +async fn require_no_consumption( + connection: &C, + prepare_id: &str, +) -> Result<(), SnapshotError> { + if connection + .query_one_raw(statement( + "SELECT prepare_id FROM mst2_metadata_prepare_consumption WHERE prepare_id=$1", + [prepare_id.into()], + )) + .await + .map_err(internal)? + .is_some() + { + return Err(integrity("expired preparation has a consumption event")); + } + Ok(()) +} + +fn expiry_uncertain(request: &PrepareExpiryRequest) -> PrepareExpiryError { + PrepareExpiryError::CommitUncertain { + recovery: request.clone(), + } +} + +pub(super) async fn clock(connection: &C) -> Result { + connection + .query_one_raw(statement( + "SELECT floor(extract(epoch FROM clock_timestamp())*1000)::bigint", + [], + )) + .await + .map_err(internal)? + .ok_or_else(|| internal("database clock is missing"))? + .try_get_by_index(0) + .map_err(internal) +} + +fn framed_digest(domain: &[u8], fields: &[&[u8]]) -> [u8; 32] { + let mut digest = Sha256::new(); + digest.update(domain); + for bytes in fields { + digest.update((bytes.len() as u64).to_be_bytes()); + digest.update(bytes); + } + digest.finalize().into() +} + +fn statement(sql: &str, values: [sea_orm::Value; N]) -> Statement { + Statement::from_sql_and_values(DbBackend::Postgres, sql, values) +} +fn internal(error: impl std::fmt::Display) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::Internal, error.to_string()) +} +fn integrity(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::IntegrityError, message) +} +fn conflict(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::Conflict, message) +} + +#[cfg(test)] +#[path = "native_metadata_prepare_expiry_tests.rs"] +mod tests; diff --git a/src/jupiter/storage/native_metadata_prepare_expiry_tests.rs b/src/jupiter/storage/native_metadata_prepare_expiry_tests.rs new file mode 100644 index 00000000..f7c99dba --- /dev/null +++ b/src/jupiter/storage/native_metadata_prepare_expiry_tests.rs @@ -0,0 +1,764 @@ +use std::{sync::atomic::Ordering, time::Duration}; + +use sea_orm::Database; +use sea_orm_migration::MigratorTrait; + +use super::*; +use crate::{ + ceres::snapshot::retention::RetentionRoot, + jupiter::{ + migration::Migrator, + storage::native_metadata_install::{ + MetadataCommitPhase, MetadataInstallError, MetadataPrepareObservation, + tests::{Fault, PgCommitFaultProxy, fixture, install, prepared}, + }, + tests::test_db_config, + }, +}; + +async fn count(db: &C, table: &str) -> i64 { + db.query_one_raw(statement(&format!("SELECT count(*) FROM {table}"), [])) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap() +} + +async fn elapsed(db: &DatabaseConnection, intent: &MetadataPrepareIntent) { + let deadline: i64 = db + .query_one_raw(statement( + "SELECT expires_at_ms FROM mst2_metadata_prepare_deadline WHERE prepare_id=$1", + [intent.prepare_id().into()], + )) + .await + .unwrap() + .unwrap() + .try_get_by_index(0) + .unwrap(); + tokio::time::timeout(Duration::from_secs(10), async { + loop { + let now = clock(db).await.unwrap(); + if now >= deadline { + break; + } + tokio::time::sleep(Duration::from_millis(((deadline - now) as u64).min(100))).await; + } + }) + .await + .unwrap(); +} + +fn rejected(error: PrepareExpiryError) -> SnapshotErrorCode { + match error { + PrepareExpiryError::Rejected(error) => error.code, + other => panic!("expected definite expiry rejection: {other:?}"), + } +} + +#[tokio::test] +async fn prepare_deadline_replay_and_recovery_bind_original_duration_without_extending_it() { + let (first, second, _schema, _url) = fixture().await; + let repo = PostgresMetadataInstallRepository::new(first.clone()) + .await + .unwrap(); + let pages = prepared("/"); + let intent = repo + .begin_intent_with_duration("duration", &pages, 3000) + .await + .unwrap(); + let digest = intent.manifest_digest(); + let grant = load_plan(&first, "duration", &digest) + .await + .unwrap() + .unwrap() + .deadline + .unwrap(); + assert_eq!( + repo.begin_intent_with_duration("duration", &pages, 3000) + .await + .unwrap(), + intent + ); + assert_eq!( + load_plan(&first, "duration", &digest) + .await + .unwrap() + .unwrap() + .deadline + .unwrap(), + grant + ); + assert!( + matches!(repo.begin_intent_with_duration("duration", &pages, 3001).await.unwrap_err(), + MetadataInstallError::Rejected(error) if error.code==SnapshotErrorCode::Conflict) + ); + for duration in [0, DEFAULT_PREPARE_DURATION_MS + 1] { + assert!( + matches!(repo.begin_intent_with_duration("invalid", &pages, duration).await.unwrap_err(), + MetadataInstallError::Rejected(error) if error.code==SnapshotErrorCode::InvalidRequest) + ); + } + let wrong = repo + .recovery_request_with_duration("duration", digest, MetadataCommitPhase::Intent, 3001) + .unwrap(); + assert!( + matches!(repo.inspect_prepare(&second, &wrong).await.unwrap_err(), + MetadataInstallError::Rejected(error) if error.code==SnapshotErrorCode::Conflict) + ); + assert_eq!(count(&first, "mst2_metadata_prepare_deadline").await, 1); +} + +#[tokio::test] +async fn prepare_elapsed_receipt_is_unusable_but_inspection_keeps_all_pins() { + let (first, second, _schema, _url) = fixture().await; + let repo = PostgresMetadataInstallRepository::new(first.clone()) + .await + .unwrap(); + let pages = prepared("/"); + let intent = repo + .begin_intent_with_duration("elapsed", &pages, 3000) + .await + .unwrap(); + install(&repo, &intent, &pages).await; + repo.finalize(&intent).await.unwrap(); + elapsed(&first, &intent).await; + assert!( + matches!(repo.inspect_prepare(&second, &intent.recovery(MetadataCommitPhase::Finalize)).await.unwrap(), + MetadataPrepareObservation::DeadlinePassed { intent: observed, .. } if observed==intent) + ); + for error in [ + repo.install_page(&intent, &pages.dag().payloads()[0]) + .await + .unwrap_err(), + repo.finalize(&intent).await.unwrap_err(), + repo.begin_intent_with_duration("elapsed", &pages, 3000) + .await + .unwrap_err(), + ] { + assert!( + matches!(error, MetadataInstallError::Rejected(error) if error.code==SnapshotErrorCode::Conflict) + ); + } + assert_eq!(count(&first, "mst2_retention_root").await, 2); + assert_eq!(count(&first, "mst2_metadata_prepare_expiry").await, 0); +} + +#[tokio::test] +async fn prepare_partial_payload_expiry_preserves_orphan_bytes_and_one_terminal_event() { + let (first, second, _schema, _url) = fixture().await; + let repo = PostgresMetadataInstallRepository::new(first.clone()) + .await + .unwrap(); + let pages = prepared("/"); + let intent = repo + .begin_intent_with_duration("partial", &pages, 3000) + .await + .unwrap(); + repo.install_page(&intent, &pages.dag().payloads()[0]) + .await + .unwrap(); + elapsed(&first, &intent).await; + let event = repo + .expire_prepare("expire-partial", &intent) + .await + .unwrap(); + assert_eq!(event.previous_state, "PREPARING"); + assert!(event.expired_at_ms >= event.expires_at_ms); + let restarted = PostgresMetadataInstallRepository::new(second.clone()) + .await + .unwrap(); + assert_eq!( + restarted + .expire_prepare("expire-partial", &intent) + .await + .unwrap(), + event + ); + assert_eq!( + restarted + .inspect_prepare_expiry(&second, &event.request) + .await + .unwrap(), + PrepareExpiryObservation::Expired(event) + ); + assert_eq!(count(&first, "mst2_metadata_payload").await, 1); + assert_eq!(count(&first, "mst2_retention_root").await, 0); + assert_eq!(count(&first, "mst2_metadata_prepare_expiry").await, 1); + assert_eq!( + rejected( + repo.expire_prepare("other-expiry", &intent) + .await + .unwrap_err() + ), + SnapshotErrorCode::Conflict + ); + assert!( + matches!(repo.inspect_prepare(&second, &intent.recovery(MetadataCommitPhase::Payload)).await.unwrap_err(), + MetadataInstallError::Rejected(error) if error.code==SnapshotErrorCode::Conflict) + ); + assert!( + first + .execute_unprepared( + "UPDATE mst2_metadata_prepare SET state='PREPARING' WHERE state='EXPIRED'" + ) + .await + .is_err() + ); + assert!( + first + .execute_unprepared( + "UPDATE mst2_metadata_prepare_expiry SET expired_at_ms=expired_at_ms+1" + ) + .await + .is_err() + ); + assert!( + first + .execute_unprepared("DELETE FROM mst2_metadata_prepare_deadline") + .await + .is_err() + ); +} + +#[tokio::test] +async fn prepare_committed_expiry_releases_only_own_root_and_preserves_shared_graph() { + let (first, second, _schema, _url) = fixture().await; + let repo = PostgresMetadataInstallRepository::new(first.clone()) + .await + .unwrap(); + let pages = prepared("/"); + let intent = repo + .begin_intent_with_duration("committed", &pages, 3000) + .await + .unwrap(); + install(&repo, &intent, &pages).await; + repo.finalize(&intent).await.unwrap(); + let committed_at = load_plan(&first, "committed", &intent.manifest_digest()) + .await + .unwrap() + .unwrap() + .record + .committed_at; + let graph = PostgresRetentionRepository::new(first.clone()); + graph + .retain_group( + pages.dag().nodes(), + pages.dag().edges(), + &[ + RetentionRoot::Lease("other-lease".into()), + RetentionRoot::Pin("workspace:one".into()), + RetentionRoot::Pin("local:one".into()), + ], + ) + .await + .unwrap(); + elapsed(&first, &intent).await; + let event = repo + .expire_prepare("expire-committed", &intent) + .await + .unwrap(); + assert_eq!(event.previous_state, "COMMITTED"); + let final_plan = load_plan(&second, "committed", &intent.manifest_digest()) + .await + .unwrap() + .unwrap(); + assert_eq!(final_plan.record.committed_at, committed_at); + assert_eq!(final_plan.record.state, "EXPIRED"); + assert_eq!(count(&first, "mst2_metadata_payload").await, 2); + assert_eq!(count(&first, "mst2_retention_node").await, 2); + assert_eq!(count(&first, "mst2_retention_edge").await, 1); + assert_eq!(count(&first, "mst2_retention_root").await, 6); + for root in ["lease:other-lease", "pin:workspace:one", "pin:local:one"] { + verify_graph_root( + &second, + &final_plan, + root, + if root.starts_with("lease:") { + "lease" + } else { + "pin" + }, + ) + .await + .unwrap(); + } + assert_eq!( + repo.expire_prepare("expire-committed", &intent) + .await + .unwrap(), + event + ); + assert_eq!(count(&first, "mst2_retention_root").await, 6); +} + +#[tokio::test] +async fn prepare_expiry_early_rejection_and_outer_rollback_keep_preparation() { + let (first, second, _schema, _url) = fixture().await; + let repo = PostgresMetadataInstallRepository::new(first.clone()) + .await + .unwrap(); + let pages = prepared("/"); + let intent = repo + .begin_intent_with_duration("rollback", &pages, 3000) + .await + .unwrap(); + install(&repo, &intent, &pages).await; + repo.finalize(&intent).await.unwrap(); + assert_eq!( + rejected(repo.expire_prepare("early", &intent).await.unwrap_err()), + SnapshotErrorCode::Conflict + ); + assert_eq!(count(&first, "mst2_metadata_prepare_expiry").await, 0); + elapsed(&first, &intent).await; + let request = expiry_request(repo.storage_uuid(), "rollback-expire", &intent).unwrap(); + let txn = repo.transaction().await.unwrap(); + let provisional = repo.expire_prepare_in_txn(&txn, &request).await.unwrap(); + assert_eq!(provisional.previous_state, "COMMITTED"); + assert_eq!(count(&txn, "mst2_retention_root").await, 0); + txn.rollback().await.unwrap(); + assert_eq!( + repo.inspect_prepare_expiry(&second, &request) + .await + .unwrap(), + PrepareExpiryObservation::NotExpired { + state: "COMMITTED".into() + } + ); + assert_eq!(count(&first, "mst2_retention_root").await, 2); + assert_eq!(count(&first, "mst2_metadata_prepare_expiry").await, 0); +} + +#[tokio::test] +async fn prepare_expiry_missing_coverage_and_recovery_resurrected_pin_fail_without_repair() { + let (first, second, _schema, _url) = fixture().await; + let repo = PostgresMetadataInstallRepository::new(first.clone()) + .await + .unwrap(); + let pages = prepared("/"); + let intent = repo + .begin_intent_with_duration("coverage", &pages, 3000) + .await + .unwrap(); + install(&repo, &intent, &pages).await; + repo.finalize(&intent).await.unwrap(); + first + .execute_raw(statement( + "DELETE FROM mst2_retention_root WHERE root_key=$1 AND node_id=$2", + [ + format!("prepare:{}", intent.prepare_id()).into(), + pages.dag().nodes()[0].id.clone().into(), + ], + )) + .await + .unwrap(); + elapsed(&first, &intent).await; + assert_eq!( + rejected( + repo.expire_prepare("incomplete", &intent) + .await + .unwrap_err() + ), + SnapshotErrorCode::ObjectUnavailable + ); + assert_eq!(count(&first, "mst2_retention_root").await, 1); + assert_eq!(count(&first, "mst2_metadata_prepare_expiry").await, 0); + let other = repo + .begin_intent_with_duration("partial-recovery", &pages, 1) + .await + .unwrap(); + elapsed(&first, &other).await; + let event = repo.expire_prepare("expired-other", &other).await.unwrap(); + PostgresRetentionRepository::new(first.clone()) + .retain_group( + pages.dag().nodes(), + pages.dag().edges(), + &[RetentionRoot::Prepare(other.prepare_id().into())], + ) + .await + .unwrap(); + assert_eq!( + rejected( + repo.inspect_prepare_expiry(&second, &event.request) + .await + .unwrap_err() + ), + SnapshotErrorCode::IntegrityError + ); + assert_eq!(count(&first, "mst2_retention_root").await, 3); +} + +#[tokio::test] +async fn prepare_expiry_clock_is_sampled_after_actual_barrier_wait() { + let (first, second, _schema, _url) = fixture().await; + let repo = PostgresMetadataInstallRepository::new(first.clone()) + .await + .unwrap(); + let pages = prepared("/"); + let intent = repo + .begin_intent_with_duration("wait-expire", &pages, 3000) + .await + .unwrap(); + let blocker = repo.transaction().await.unwrap(); + repo.barrier(&blocker).await.unwrap(); + let worker_repo = PostgresMetadataInstallRepository::new(second.clone()) + .await + .unwrap(); + let worker_intent = intent.clone(); + let worker = tokio::spawn(async move { + worker_repo + .expire_prepare("waited-expiry", &worker_intent) + .await + }); + elapsed(&first, &intent).await; + assert!(!worker.is_finished()); + let released_at = clock(&first).await.unwrap(); + blocker.rollback().await.unwrap(); + let event = tokio::time::timeout(Duration::from_secs(5), worker) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(event.expired_at_ms >= released_at); + assert_eq!(count(&first, "mst2_metadata_prepare_expiry").await, 1); +} + +async fn commit_fault(fault: Fault) { + let (direct, _second, _schema, url) = fixture().await; + let original = PostgresMetadataInstallRepository::new(direct.clone()) + .await + .unwrap(); + let pages = prepared("/"); + let intent = original + .begin_intent_with_duration("fault-expiry", &pages, 3000) + .await + .unwrap(); + install(&original, &intent, &pages).await; + original.finalize(&intent).await.unwrap(); + elapsed(&direct, &intent).await; + let proxy = PgCommitFaultProxy::start(&url, fault).await; + let mut options = sea_orm::ConnectOptions::new(proxy.url.clone()); + options.min_connections(1).max_connections(1); + let repo = PostgresMetadataInstallRepository::new(Database::connect(options).await.unwrap()) + .await + .unwrap(); + proxy.armed.store(true, Ordering::SeqCst); + let recovery = match repo + .expire_prepare("fault-release", &intent) + .await + .unwrap_err() + { + PrepareExpiryError::CommitUncertain { recovery } => recovery, + other => panic!("COMMIT disconnect must return a typed unknown outcome: {other:?}"), + }; + proxy.wait_for_fault().await; + let observed = repo + .inspect_prepare_expiry(&direct, &recovery) + .await + .unwrap(); + match fault { + Fault::BeforeCommit => { + assert!(!proxy.commit_observed.load(Ordering::SeqCst)); + assert_eq!( + observed, + PrepareExpiryObservation::NotExpired { + state: "COMMITTED".into() + } + ); + assert_eq!(count(&direct, "mst2_retention_root").await, 2); + assert_eq!(count(&direct, "mst2_metadata_prepare_expiry").await, 0); + } + Fault::AfterCommit => { + assert!(proxy.commit_observed.load(Ordering::SeqCst)); + let original_event = match observed { + PrepareExpiryObservation::Expired(receipt) => receipt, + other => { + panic!("COMMIT reply loss must recover its immutable release event: {other:?}") + } + }; + assert_eq!( + original + .expire_prepare("fault-release", &intent) + .await + .unwrap(), + original_event + ); + } + Fault::PrepareRead => panic!("query failure is a separate recovery test"), + } + original + .expire_prepare("fault-release", &intent) + .await + .unwrap(); + assert_eq!(count(&direct, "mst2_metadata_prepare_expiry").await, 1); + assert_eq!(count(&direct, "mst2_retention_root").await, 0); + assert_eq!(count(&direct, "mst2_metadata_payload").await, 2); +} + +#[tokio::test] +async fn prepare_expiry_actual_disconnect_before_commit_keeps_refs_until_recovery() { + commit_fault(Fault::BeforeCommit).await; +} + +#[tokio::test] +async fn prepare_expiry_actual_commit_reply_loss_recovers_one_release_event() { + commit_fault(Fault::AfterCommit).await; +} + +#[tokio::test] +async fn prepare_expiry_recovery_query_failure_and_wrong_primary_stay_unknown() { + let (direct, second, _schema, url) = fixture().await; + let repo = PostgresMetadataInstallRepository::new(direct.clone()) + .await + .unwrap(); + let intent = repo + .begin_intent_with_duration("unknown-expiry", &prepared("/"), 1) + .await + .unwrap(); + elapsed(&direct, &intent).await; + let event = repo + .expire_prepare("unknown-release", &intent) + .await + .unwrap(); + let proxy = PgCommitFaultProxy::start(&url, Fault::PrepareRead).await; + let fresh = Database::connect(proxy.url.clone()).await.unwrap(); + proxy.armed.store(true, Ordering::SeqCst); + assert!( + matches!(repo.inspect_prepare_expiry(&fresh, &event.request).await.unwrap_err(), PrepareExpiryError::CommitUncertain { recovery } if recovery==event.request) + ); + proxy.wait_for_fault().await; + let (wrong, _wrong_second, _wrong_schema, _wrong_url) = fixture().await; + assert!(matches!( + repo.inspect_prepare_expiry(&wrong, &event.request) + .await + .unwrap_err(), + PrepareExpiryError::CommitUncertain { .. } + )); + assert_eq!( + repo.inspect_prepare_expiry(&second, &event.request) + .await + .unwrap(), + PrepareExpiryObservation::Expired(event) + ); + assert_eq!(count(&direct, "mst2_metadata_prepare_expiry").await, 1); +} + +#[tokio::test] +async fn prepare_deadline_and_expiry_ddl_reject_unbound_grants_and_incomplete_terminal_transactions() + { + let (db, _second, _schema, _url) = fixture().await; + let repo = PostgresMetadataInstallRepository::new(db.clone()) + .await + .unwrap(); + let intent = repo + .begin_intent_with_duration("ddl-prepare", &prepared("/"), 3000) + .await + .unwrap(); + let stored = load_plan(&db, intent.operation_id(), &intent.manifest_digest()) + .await + .unwrap() + .unwrap(); + let grant = stored.deadline.unwrap(); + assert!( + db.execute_unprepared("UPDATE mst2_metadata_prepare SET deadline_managed=false") + .await + .is_err() + ); + let txn = repo.transaction().await.unwrap(); + txn.execute_raw(statement( + "UPDATE mst2_metadata_prepare SET state='EXPIRED' WHERE prepare_id=$1", + [intent.prepare_id().into()], + )) + .await + .unwrap(); + assert!(txn.commit().await.is_err()); + assert_eq!(count(&db, "mst2_metadata_prepare_expiry").await, 0); + for wrong_grant in [false, true] { + let txn = repo.transaction().await.unwrap(); + txn.execute_raw(statement( + "INSERT INTO mst2_metadata_prepare_expiry(prepare_id,operation_id,operation_digest,grant_digest, + previous_state,pin_root,expires_at_ms,expired_at_ms) VALUES($1,'ddl-expiry',$2,$3,'PREPARING',$4,$5,$5)", + [intent.prepare_id().into(),vec![9u8;32].into(),if wrong_grant {vec![8u8;32]} else {grant.grant_digest.to_vec()}.into(), + format!("prepare:{}",intent.prepare_id()).into(),grant.expires_at_ms.into()], + )).await.unwrap(); + if wrong_grant { + txn.execute_raw(statement( + "UPDATE mst2_metadata_prepare SET state='EXPIRED' WHERE prepare_id=$1", + [intent.prepare_id().into()], + )) + .await + .unwrap(); + } + assert!(txn.commit().await.is_err()); + assert_eq!(count(&db, "mst2_metadata_prepare_expiry").await, 0); + } + let txn = repo.transaction().await.unwrap(); + txn.execute_raw(statement( + "INSERT INTO mst2_metadata_prepare SELECT replacement.* FROM jsonb_populate_record(NULL::mst2_metadata_prepare, + (SELECT to_jsonb(p)||jsonb_build_object('prepare_id',$1::text,'operation_id','missing-grant') + FROM mst2_metadata_prepare p WHERE prepare_id=$2)) AS replacement", + [uuid::Uuid::new_v4().to_string().into(),intent.prepare_id().into()], + )).await.unwrap(); + assert!(txn.commit().await.is_err()); + assert_eq!(count(&db, "mst2_metadata_prepare").await, 1); + assert_eq!(count(&db, "mst2_metadata_prepare_deadline").await, 1); + assert_eq!( + load_plan(&db, intent.operation_id(), &intent.manifest_digest()) + .await + .unwrap() + .unwrap() + .record + .state, + "PREPARING" + ); +} + +async fn intent_commit_fault(fault: Fault) { + let (direct, _second, _schema, url) = fixture().await; + let proxy = PgCommitFaultProxy::start(&url, fault).await; + let mut options = sea_orm::ConnectOptions::new(proxy.url.clone()); + options.min_connections(1).max_connections(1); + let repo = PostgresMetadataInstallRepository::new(Database::connect(options).await.unwrap()) + .await + .unwrap(); + let pages = prepared("/"); + proxy.armed.store(true, Ordering::SeqCst); + let witness = match repo + .begin_intent_with_duration("fault-intent-duration", &pages, 30_000) + .await + .unwrap_err() + { + MetadataInstallError::CommitUncertain { + phase: MetadataCommitPhase::Intent, + recovery, + .. + } => recovery, + other => panic!("intent COMMIT fault must retain its exact grant witness: {other:?}"), + }; + proxy.wait_for_fault().await; + let observed = repo.inspect_prepare(&direct, &witness).await.unwrap(); + let restart = PostgresMetadataInstallRepository::new(direct.clone()) + .await + .unwrap(); + match fault { + Fault::BeforeCommit => { + assert!(!proxy.commit_observed.load(Ordering::SeqCst)); + assert_eq!(observed, MetadataPrepareObservation::Absent); + assert_eq!(count(&direct, "mst2_metadata_prepare_deadline").await, 0); + } + Fault::AfterCommit => { + assert!(proxy.commit_observed.load(Ordering::SeqCst)); + let intent = match observed { + MetadataPrepareObservation::Preparing(intent) => intent, + other => panic!("expected one durable intent: {other:?}"), + }; + assert_eq!( + restart + .begin_intent_with_duration("fault-intent-duration", &pages, 30_000) + .await + .unwrap(), + intent + ); + let wrong = restart + .recovery_request_with_duration( + intent.operation_id(), + intent.manifest_digest(), + MetadataCommitPhase::Intent, + 30_001, + ) + .unwrap(); + assert!( + matches!(restart.inspect_prepare(&direct,&wrong).await.unwrap_err(),MetadataInstallError::Rejected(error) if error.code==SnapshotErrorCode::Conflict) + ); + } + Fault::PrepareRead => panic!("query fault is not intent COMMIT failure"), + } + restart + .begin_intent_with_duration("fault-intent-duration", &pages, 30_000) + .await + .unwrap(); + assert_eq!(count(&direct, "mst2_metadata_prepare").await, 1); + assert_eq!(count(&direct, "mst2_metadata_prepare_deadline").await, 1); +} + +#[tokio::test] +async fn prepare_intent_actual_commit_reply_loss_recovers_exact_duration_and_grant() { + intent_commit_fault(Fault::AfterCommit).await; +} + +#[tokio::test] +async fn prepare_intent_actual_disconnect_before_commit_recovers_absence_without_grant() { + intent_commit_fault(Fault::BeforeCommit).await; +} + +#[tokio::test] +async fn prepare_deadline_migration_keeps_historical_unknown_duration_and_pins() { + let temp = tempfile::TempDir::new().unwrap(); + let (config, _schema) = test_db_config(temp.path()).await; + let db = Database::connect(config.db_url.clone()).await.unwrap(); + Migrator::up(&db, Some(Migrator::migrations().len() as u32 - 1)) + .await + .unwrap(); + let pages = prepared("/"); + let plan = pages.install_plan().unwrap(); + let digest = plan.digest().unwrap(); + let id = uuid::Uuid::new_v4().to_string(); + let identity = &plan.identity; + db.execute_raw(statement( + "INSERT INTO mst2_metadata_prepare(prepare_id,operation_id,manifest_digest,canonical_plan,source_domain, + tagged_root_tree_oid,scope,schema_version,metadata_codec,materialization_policy,fs_semantics,access_projection, + verification_revision,projection_revision,metadata_root,node_count,edge_count,total_bytes,state,created_at,committed_at) + VALUES($1,'historical',$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,'COMMITTED',now()-interval '30 days',now()-interval '30 days')", + [id.clone().into(),digest.to_vec().into(),plan.encode().unwrap().into(),identity.source_domain.clone().into(), + identity.tagged_root_tree_oid.clone().into(),identity.scope.clone().into(),(identity.schema_version as i16).into(), + (identity.metadata_codec as i16).into(),(identity.materialization_policy as i16).into(),(identity.fs_semantics as i16).into(), + (identity.access_projection as i16).into(),identity.verification_revision.into(),(identity.projection_revision as i16).into(), + plan.root.to_vec().into(),(plan.pages.len() as i32).into(),(plan.edges.len() as i32).into(),(plan.total_bytes as i64).into()], + )).await.unwrap(); + for page in pages.dag().payloads() { + db.execute_raw(statement("INSERT INTO mst2_metadata_prepare_page(prepare_id,page_id,expected_size) VALUES($1,$2,$3)", + [id.clone().into(),page.id.to_vec().into(),(page.size as i32).into()])).await.unwrap(); + db.execute_raw(statement("INSERT INTO mst2_metadata_payload(page_id,metadata_codec,byte_size,payload) VALUES($1,1,$2,$3)", + [page.id.to_vec().into(),(page.size as i32).into(),page.bytes.clone().into()])).await.unwrap(); + } + PostgresRetentionRepository::new(db.clone()) + .retain_group( + pages.dag().nodes(), + pages.dag().edges(), + &[RetentionRoot::Prepare(id)], + ) + .await + .unwrap(); + Migrator::up(&db, None).await.unwrap(); + let repo = PostgresMetadataInstallRepository::new(db.clone()) + .await + .unwrap(); + let stored = load_plan(&db, "historical", &digest) + .await + .unwrap() + .unwrap(); + assert!(!stored.record.deadline_managed); + assert!(stored.deadline.is_none()); + let intent = stored.intent().unwrap(); + assert_eq!( + rejected( + repo.expire_prepare("legacy-expire", &intent) + .await + .unwrap_err() + ), + SnapshotErrorCode::Conflict + ); + assert!( + matches!(repo.begin_intent("historical",&pages).await.unwrap_err(), MetadataInstallError::Rejected(error) if error.code==SnapshotErrorCode::Conflict) + ); + let fresh = Database::connect(config.db_url).await.unwrap(); + assert!(matches!( + repo.inspect_prepare(&fresh, &intent.recovery(MetadataCommitPhase::Finalize)) + .await + .unwrap(), + MetadataPrepareObservation::Committed(_) + )); + assert_eq!(count(&db, "mst2_metadata_prepare_deadline").await, 0); + assert_eq!(count(&db, "mst2_metadata_prepare_expiry").await, 0); + assert_eq!(count(&db, "mst2_retention_root").await, 2); +} From 8ed0dfabfea1610ecf7526d57cac81577a245a36 Mon Sep 17 00:00:00 2001 From: Luxian Date: Mon, 5 Oct 2026 21:42:16 +0800 Subject: [PATCH 2/3] Reduce the size of opaque preparation expiry errors --- src/jupiter/storage/native_metadata_prepare_expiry.rs | 4 ++-- src/jupiter/storage/native_metadata_prepare_expiry_tests.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jupiter/storage/native_metadata_prepare_expiry.rs b/src/jupiter/storage/native_metadata_prepare_expiry.rs index 1422bf9c..2ebbe810 100644 --- a/src/jupiter/storage/native_metadata_prepare_expiry.rs +++ b/src/jupiter/storage/native_metadata_prepare_expiry.rs @@ -207,7 +207,7 @@ pub(crate) enum PrepareExpiryError { #[error(transparent)] Rejected(#[from] SnapshotError), #[error("preparation expiry commit outcome is unknown")] - CommitUncertain { recovery: PrepareExpiryRequest }, + CommitUncertain { recovery: Box }, } #[derive(Debug, Clone, PartialEq, Eq)] @@ -544,7 +544,7 @@ async fn require_no_consumption( fn expiry_uncertain(request: &PrepareExpiryRequest) -> PrepareExpiryError { PrepareExpiryError::CommitUncertain { - recovery: request.clone(), + recovery: Box::new(request.clone()), } } diff --git a/src/jupiter/storage/native_metadata_prepare_expiry_tests.rs b/src/jupiter/storage/native_metadata_prepare_expiry_tests.rs index f7c99dba..67947666 100644 --- a/src/jupiter/storage/native_metadata_prepare_expiry_tests.rs +++ b/src/jupiter/storage/native_metadata_prepare_expiry_tests.rs @@ -524,7 +524,7 @@ async fn prepare_expiry_recovery_query_failure_and_wrong_primary_stay_unknown() let fresh = Database::connect(proxy.url.clone()).await.unwrap(); proxy.armed.store(true, Ordering::SeqCst); assert!( - matches!(repo.inspect_prepare_expiry(&fresh, &event.request).await.unwrap_err(), PrepareExpiryError::CommitUncertain { recovery } if recovery==event.request) + matches!(repo.inspect_prepare_expiry(&fresh, &event.request).await.unwrap_err(), PrepareExpiryError::CommitUncertain { recovery } if *recovery==event.request) ); proxy.wait_for_fault().await; let (wrong, _wrong_second, _wrong_schema, _wrong_url) = fixture().await; From ddf38b14fa20c1b4189ef0c8d217b3d0aa338147 Mon Sep 17 00:00:00 2001 From: Xiaoyang Han Date: Mon, 5 Oct 2026 23:09:34 +0800 Subject: [PATCH 3/3] Reject corrupt MST/2 verified facts and preserve raw content (#31) * fix(git): respect shallow depth and v0 negotiation boundaries * test(redis): accept OS-assigned standalone ports * test(s3): allow isolated RustFS integration endpoints * fix(git): preserve SSH shallow negotiation and isolate test ports * Validate MST2 verified facts and preserve raw blob content * Isolate retention lifecycle test runtime * ci: run isolated Linux repository gates on pull requests * ci: avoid duplicate case-insensitive workflow environment keys * ci: handle generated future lint and retain failed gate cache * feat(mst2): index fixed namespace bindings by path components (#40) * feat(mst2): add fixed namespace component index * test(mst2): keep namespace oracle inside private domain * test(mst2): register internal namespace regression suite --- .github/workflows/repository-gates.yml | 79 +++ src/callisto/mst2_verified_object.rs | 6 +- src/ceres/pack/import_repo.rs | 4 + src/ceres/snapshot/error.rs | 6 + src/ceres/snapshot/mod.rs | 7 +- src/ceres/snapshot/namespace.rs | 405 ++++++++++++ src/ceres/snapshot/namespace/tests.rs | 838 +++++++++++++++++++++++++ src/ceres/snapshot/pages.rs | 610 ++++++++++++++++-- src/ceres/snapshot/runtime.rs | 10 +- src/ceres/snapshot/view.rs | 19 +- src/jupiter/storage/mono_storage.rs | 252 +++++++- tests/integration_api_write_trunk.rs | 11 +- 12 files changed, 2167 insertions(+), 80 deletions(-) create mode 100644 .github/workflows/repository-gates.yml create mode 100644 src/ceres/snapshot/namespace.rs create mode 100644 src/ceres/snapshot/namespace/tests.rs diff --git a/.github/workflows/repository-gates.yml b/.github/workflows/repository-gates.yml new file mode 100644 index 00000000..b656c665 --- /dev/null +++ b/.github/workflows/repository-gates.yml @@ -0,0 +1,79 @@ +name: Repository gates + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: repository-gates-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + linux: + name: Linux repository gates + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: "0" + CARGO_BUILD_JOBS: "2" + CARGO_PROFILE_DEV_DEBUG: "0" + CARGO_PROFILE_TEST_DEBUG: "0" + RUST_BACKTRACE: "1" + NO_PROXY: localhost,127.0.0.1,::1,host.docker.internal + steps: + - uses: actions/checkout@v4 + + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y pkg-config libssl-dev protobuf-compiler + + - uses: dtolnay/rust-toolchain@nightly + with: + components: rustfmt + + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - uses: Swatinem/rust-cache@v2 + with: + save-if: "true" + + - name: Check formatting + run: cargo +nightly fmt --all --check + + - name: Check all targets and features + run: cargo clippy --all-targets --all-features -- -D warnings + + - name: Build + run: cargo build + + - name: Build tests + run: cargo build --tests + + - name: Start isolated test dependencies + shell: bash + run: | + set -euo pipefail + source scripts/lib/mega2-it.sh + mega2_it_ensure_env_test + mega2_it_prepare_git_workdir + docker compose -p mega2-it -f docker/docker-compose.test.yml --profile git \ + up -d --build --wait postgres redis rustfs rustfs-init git-cli + + - name: Run repository tests with test environment + shell: bash + run: | + set -euo pipefail + source .env.test + export no_proxy="$NO_PROXY" + cargo test --all + + - name: Stop isolated test dependencies + if: always() + run: docker compose -p mega2-it -f docker/docker-compose.test.yml --profile git down -v diff --git a/src/callisto/mst2_verified_object.rs b/src/callisto/mst2_verified_object.rs index 851f5cd5..9933e936 100644 --- a/src/callisto/mst2_verified_object.rs +++ b/src/callisto/mst2_verified_object.rs @@ -2,9 +2,9 @@ //! //! One row per verified object: Git identity + raw content digest + 64-bit //! size + verification state. The natural key is -//! `(storage_domain, git_oid, object_kind)`; rows are insert-only with -//! ON CONFLICT DO NOTHING — a row only ever appears after the content was -//! fetched and hashed by this service (write-through verification). +//! `(storage_domain, git_oid, object_kind)`. Current-generation facts retain +//! first-write-wins semantics; valid legacy facts can be atomically upgraded +//! only after this service fetches and hashes the raw body again. use sea_orm::entity::prelude::*; use serde::{Deserialize, Serialize}; diff --git a/src/ceres/pack/import_repo.rs b/src/ceres/pack/import_repo.rs index 6b6bd6ce..b98cbb39 100644 --- a/src/ceres/pack/import_repo.rs +++ b/src/ceres/pack/import_repo.rs @@ -452,6 +452,10 @@ impl RepoHandler for ImportRepo { } } +#[allow( + clippy::double_must_use, + reason = "async_recursion adds must_use to the generated boxed Future" +)] #[async_recursion] pub(crate) async fn collect_git_blob_filepaths( storage: GitDbStorage, diff --git a/src/ceres/snapshot/error.rs b/src/ceres/snapshot/error.rs index 8d9cb316..4fe54274 100644 --- a/src/ceres/snapshot/error.rs +++ b/src/ceres/snapshot/error.rs @@ -34,6 +34,8 @@ pub enum SnapshotErrorCode { DigestMismatch, RangeNotSupported, SymlinkTraversal, + ObjectUnavailable, + IntegrityError, Internal, } @@ -60,6 +62,8 @@ impl SnapshotErrorCode { SnapshotErrorCode::DigestMismatch => "EXPECTED_DIGEST_MISMATCH", SnapshotErrorCode::RangeNotSupported => "RANGE_NOT_SUPPORTED", SnapshotErrorCode::SymlinkTraversal => "SYMLINK_TRAVERSAL", + SnapshotErrorCode::ObjectUnavailable => "OBJECT_UNAVAILABLE", + SnapshotErrorCode::IntegrityError => "INTEGRITY_ERROR", SnapshotErrorCode::Internal => "INTERNAL", } } @@ -79,6 +83,7 @@ impl SnapshotErrorCode { | SnapshotErrorCode::PathNotFound | SnapshotErrorCode::LeaseUnknown => 404, SnapshotErrorCode::SnapshotNotReady => 503, + SnapshotErrorCode::ObjectUnavailable => 503, SnapshotErrorCode::NotDirectory | SnapshotErrorCode::Conflict | SnapshotErrorCode::CursorStale @@ -87,6 +92,7 @@ impl SnapshotErrorCode { SnapshotErrorCode::UnsupportedEntry => 422, SnapshotErrorCode::LimitExceeded | SnapshotErrorCode::ProofBudgetExceeded => 413, SnapshotErrorCode::LeaseExpired | SnapshotErrorCode::SnapshotGone => 410, + SnapshotErrorCode::IntegrityError => 502, SnapshotErrorCode::Internal => 500, } } diff --git a/src/ceres/snapshot/mod.rs b/src/ceres/snapshot/mod.rs index 7ac7fc25..a7c44a61 100644 --- a/src/ceres/snapshot/mod.rs +++ b/src/ceres/snapshot/mod.rs @@ -1,12 +1,13 @@ //! MST/2 snapshot domain (spec 02/03): fixed views over pinned Git trees. //! -//! Slice scope (T02-a): native monorepo views only — no bindings, imports or -//! release policy yet (T04/T05). Everything here resolves against fixed OIDs; -//! no code path in this module may read current refs. +//! Serving remains native-only. The independent namespace index is an unwired +//! composition seam; identity, attestation and publication integration remain +//! separate gates. Fixed-source readers must never look up current refs. pub mod chunks; pub mod descriptor; pub mod error; pub mod frame_stream; +pub mod namespace; pub mod pages; pub mod publication; pub mod resolver; diff --git a/src/ceres/snapshot/namespace.rs b/src/ceres/snapshot/namespace.rs new file mode 100644 index 00000000..720a6171 --- /dev/null +++ b/src/ceres/snapshot/namespace.rs @@ -0,0 +1,405 @@ +//! Immutable component-prefix routing for an already attested namespace. +//! +//! `S` is a caller-owned fixed source handle, not a registry path or ref selector. +//! This module neither encodes NamespaceView identity nor grants authorization, +//! retention or release-write permission. It is not wired into serving/publication. + +use std::collections::{BTreeMap, BTreeSet}; + +use thiserror::Error; + +use super::{ + error::{SnapshotError, SnapshotErrorCode}, + resolver::FsKind, + view::validate_scope_relative_path, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BindingPolicy { + Mutable, + ImmutableRelease, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceKind { + Native, + Import, +} + +#[derive(Debug, Clone)] +pub struct FixedBinding { + pub mount_path: String, + pub source: S, + pub source_kind: SourceKind, + /// Canonical source-root-relative path; the source root is the empty string. + pub source_subpath: String, + pub policy: BindingPolicy, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NodeClass { + NativeTree, + NativeCheckoutRoot, + ImportRoot, + ImportTree, + Aggregate, +} + +#[derive(Debug, Clone)] +pub struct SourceNode { + pub fs_kind: FsKind, + pub oid: String, +} + +#[derive(Debug, Clone)] +pub struct SourceEntry { + pub name: String, + pub node: SourceNode, +} + +#[derive(Debug, Clone)] +pub enum SourceOutcome { + Found(SourceNode), + /// Absence established by reading a fixed parent tree, never an I/O failure. + AbsentProven, +} + +/// An adapter must read only the attested fixed root held by `source`, verify +/// membership, and preserve storage errors. Paths are relative to that root; +/// `scope_path` must not be prepended again. Attestation/auth/retention are external. +/// The adapter also enforces the attested scope plus source-path byte budget. +pub trait FixedSourceReader { + fn resolve(&mut self, source: &S, source_path: &str) -> Result; + fn list(&mut self, source: &S, source_path: &str) -> Result, SnapshotError>; +} + +#[derive(Debug)] +pub struct SourceRoute<'a, S> { + pub source: &'a S, + pub source_path: String, + pub source_kind: SourceKind, + pub mount_path: Option<&'a str>, + /// Selected binding provenance, not an inherited release protection or a + /// write permission. A persistent release registry must fence every writer. + pub policy: BindingPolicy, +} + +#[derive(Debug)] +pub struct NamespaceEntry<'a, S> { + pub name: String, + pub fs_kind: FsKind, + /// Synthetic ancestors have neither an object ID nor a source context. + pub oid: Option, + pub source_context: Option>, +} + +#[derive(Debug)] +pub struct NamespaceDirectory<'a, S> { + pub node_class: NodeClass, + pub source_context: Option>, + pub entries: Vec>, +} + +#[derive(Debug, Error)] +pub enum NamespaceError { + #[error(transparent)] + Snapshot(#[from] SnapshotError), + #[error("NAMESPACE_CONFLICT at {0}")] + Conflict(String), + #[error("duplicate mount path {0}")] + DuplicateMount(String), + #[error("binding source subpath is not a directory at {0}")] + SourceSubpathNotDirectory(String), + #[error("invalid source directory entry {0}")] + InvalidSourceEntry(String), +} + +#[derive(Debug, Default)] +struct PrefixNode { + binding: Option, + children: BTreeMap, +} + +/// Construct a new index for each fixed view; no binding/source mutators exist. +/// The trie restricts each request to its path and immediate indexed children. +#[derive(Debug)] +pub struct FixedNamespaceIndex { + native: S, + bindings: Vec>, + root: PrefixNode, + native_checkouts: BTreeSet, +} + +impl FixedNamespaceIndex { + pub fn new>( + native: S, + bindings: Vec>, + native_checkouts: BTreeSet, + reader: &mut R, + ) -> Result { + let mut root = PrefixNode::default(); + for (id, binding) in bindings.iter().enumerate() { + validate_scope_relative_path(&binding.mount_path)?; + validate_relative_path(&binding.source_subpath)?; + let mut node = &mut root; + for component in components(&binding.mount_path) { + node = node.children.entry(component.to_string()).or_default(); + } + if node.binding.replace(id).is_some() { + return Err(NamespaceError::DuplicateMount(binding.mount_path.clone())); + } + } + for checkout in &native_checkouts { + validate_scope_relative_path(checkout)?; + } + let index = Self { + native, + bindings, + root, + native_checkouts, + }; + index.validate_sources(reader)?; + Ok(index) + } + + pub fn route(&self, path: &str) -> Result, NamespaceError> { + self.route_inner(path, false) + } + + fn route_inner( + &self, + path: &str, + exclude_exact: bool, + ) -> Result, NamespaceError> { + validate_scope_relative_path(path)?; + let comps = components(path); + let mut prefix = &self.root; + let mut selected = prefix + .binding + .filter(|_| !(exclude_exact && comps.is_empty())); + let mut selected_depth = 0; + for (depth, component) in comps.iter().enumerate() { + let Some(child) = prefix.children.get(*component) else { + break; + }; + prefix = child; + if let Some(id) = prefix.binding + && !(exclude_exact && depth + 1 == comps.len()) + { + selected = Some(id); + selected_depth = depth + 1; + } + } + let route = if let Some(id) = selected { + let binding = &self.bindings[id]; + let suffix = comps[selected_depth..].join("/"); + let source_path = join_relative(&binding.source_subpath, &suffix); + SourceRoute { + source: &binding.source, + source_path, + source_kind: binding.source_kind, + mount_path: Some(&binding.mount_path), + policy: binding.policy, + } + } else { + SourceRoute { + source: &self.native, + source_path: comps.join("/"), + source_kind: SourceKind::Native, + mount_path: None, + policy: BindingPolicy::Mutable, + } + }; + validate_relative_path(&route.source_path)?; + Ok(route) + } + + fn prefix(&self, path: &str) -> Option<&PrefixNode> { + let mut node = &self.root; + for component in components(path) { + node = node.children.get(component)?; + } + Some(node) + } + + /// Validate all mount crossings once when constructing the index. The + /// binding at the endpoint is excluded so it cannot hide a source file. + fn validate_sources>( + &self, + reader: &mut R, + ) -> Result<(), NamespaceError> { + match reader.resolve(&self.native, "")? { + SourceOutcome::Found(node) if node.fs_kind == FsKind::Directory => {} + _ => return Err(NamespaceError::Conflict("/".to_string())), + } + for binding in &self.bindings { + match reader.resolve(&binding.source, &binding.source_subpath)? { + SourceOutcome::Found(node) if node.fs_kind == FsKind::Directory => {} + _ => { + return Err(NamespaceError::SourceSubpathNotDirectory( + binding.mount_path.clone(), + )); + } + } + let comps = components(&binding.mount_path); + for length in 0..=comps.len() { + let path = format!("/{}", comps[..length].join("/")); + let route = self.route_inner(&path, length == comps.len())?; + if let SourceOutcome::Found(node) = + reader.resolve(route.source, &route.source_path)? + && node.fs_kind != FsKind::Directory + { + return Err(NamespaceError::Conflict(path)); + } + } + } + Ok(()) + } + + /// Return proven absence as None; failures remain errors. Exact mounts + /// replace the underlying source subtree, with only explicit descendants + /// added. No registry enumeration or implicit native/import union occurs. + pub fn list>( + &self, + path: &str, + reader: &mut R, + ) -> Result>, NamespaceError> { + let route = self.route(path)?; + let indexed = self.prefix(path); + let has_children = indexed.is_some_and(|node| !node.children.is_empty()); + let mut entries = BTreeMap::new(); + let mut aggregate = false; + let source_context = match reader.resolve(route.source, &route.source_path)? { + SourceOutcome::AbsentProven if !has_children => return Ok(None), + SourceOutcome::AbsentProven => { + aggregate = true; + None + } + SourceOutcome::Found(node) if node.fs_kind != FsKind::Directory => { + return Err(SnapshotError::new( + SnapshotErrorCode::NotDirectory, + "namespace path is not a directory", + ) + .into()); + } + SourceOutcome::Found(_) => { + for entry in reader.list(route.source, &route.source_path)? { + validate_entry_name(&entry.name)?; + let child_path = append_absolute(path, &entry.name); + let context = self.route(&child_path)?; + let name = entry.name; + if entries + .insert( + name.clone(), + NamespaceEntry { + name: name.clone(), + fs_kind: entry.node.fs_kind, + oid: Some(entry.node.oid), + source_context: Some(context), + }, + ) + .is_some() + { + return Err(NamespaceError::InvalidSourceEntry(name)); + } + } + Some(route) + } + }; + if let Some(prefix) = indexed { + for (name, child) in &prefix.children { + let child_path = append_absolute(path, name); + if child.binding.is_some() { + let route = self.route(&child_path)?; + let node = match reader.resolve(route.source, &route.source_path)? { + SourceOutcome::Found(node) if node.fs_kind == FsKind::Directory => node, + _ => return Err(NamespaceError::SourceSubpathNotDirectory(child_path)), + }; + aggregate = true; + entries.insert( + name.clone(), + NamespaceEntry { + name: name.clone(), + fs_kind: node.fs_kind, + oid: Some(node.oid), + source_context: Some(route), + }, + ); + } else if let Some(entry) = entries.get(name) { + if entry.fs_kind != FsKind::Directory { + return Err(NamespaceError::Conflict(child_path)); + } + } else { + aggregate = true; + entries.insert( + name.clone(), + NamespaceEntry { + name: name.clone(), + fs_kind: FsKind::Directory, + oid: None, + source_context: None, + }, + ); + } + } + } + let node_class = if aggregate { + NodeClass::Aggregate + } else if let Some(route) = &source_context { + match route.source_kind { + SourceKind::Import if route.mount_path == Some(path) => NodeClass::ImportRoot, + SourceKind::Import => NodeClass::ImportTree, + SourceKind::Native if self.native_checkouts.contains(path) => { + NodeClass::NativeCheckoutRoot + } + SourceKind::Native => NodeClass::NativeTree, + } + } else { + NodeClass::Aggregate + }; + Ok(Some(NamespaceDirectory { + node_class, + source_context, + entries: entries.into_values().collect(), + })) + } +} + +fn components(path: &str) -> Vec<&str> { + if path == "/" { + Vec::new() + } else { + path[1..].split('/').collect() + } +} + +fn join_relative(parent: &str, suffix: &str) -> String { + match (parent.is_empty(), suffix.is_empty()) { + (true, _) => suffix.to_string(), + (_, true) => parent.to_string(), + _ => format!("{parent}/{suffix}"), + } +} + +fn append_absolute(parent: &str, name: &str) -> String { + if parent == "/" { + format!("/{name}") + } else { + format!("{parent}/{name}") + } +} + +fn validate_relative_path(path: &str) -> Result<(), SnapshotError> { + validate_scope_relative_path(&format!("/{path}")) +} + +fn validate_entry_name(name: &str) -> Result<(), NamespaceError> { + if name.contains('/') || validate_relative_path(name).is_err() || name.is_empty() { + return Err(NamespaceError::InvalidSourceEntry(name.to_string())); + } + Ok(()) +} + +#[cfg(test)] +#[path = "namespace/tests.rs"] +mod tests; diff --git a/src/ceres/snapshot/namespace/tests.rs b/src/ceres/snapshot/namespace/tests.rs new file mode 100644 index 00000000..6b2fa27e --- /dev/null +++ b/src/ceres/snapshot/namespace/tests.rs @@ -0,0 +1,838 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + io::Write, + path::Path, + process::{Command, Stdio}, +}; + +use tempfile::TempDir; + +use crate::ceres::snapshot::{ + error::{SnapshotError, SnapshotErrorCode}, + namespace::{ + BindingPolicy, FixedBinding, FixedNamespaceIndex, FixedSourceReader, NamespaceError, + NodeClass, SourceEntry, SourceKind, SourceNode, SourceOutcome, + }, + resolver::FsKind, +}; + +#[derive(Default)] +struct MemoryReader { + nodes: BTreeMap<(String, String), SourceNode>, + fail_path: Option, + fail_listing: bool, + resolves: usize, + lists: usize, +} + +impl MemoryReader { + fn add(&mut self, source: &str, path: &str, fs_kind: FsKind) { + self.nodes.insert( + (source.to_string(), path.to_string()), + SourceNode { + fs_kind, + oid: format!("{source}:{path}"), + }, + ); + } + + fn roots() -> Self { + let mut reader = Self::default(); + for source in ["native", "a20", "a21", "b30"] { + reader.add(source, "", FsKind::Directory); + } + reader + } +} + +impl FixedSourceReader for MemoryReader { + fn resolve(&mut self, source: &String, path: &str) -> Result { + self.resolves += 1; + if self.fail_path.as_deref() == Some(path) { + return Err(SnapshotError::new( + SnapshotErrorCode::ObjectUnavailable, + "injected missing backend object", + )); + } + Ok(self + .nodes + .get(&(source.clone(), path.to_string())) + .cloned() + .map_or(SourceOutcome::AbsentProven, SourceOutcome::Found)) + } + + fn list(&mut self, source: &String, path: &str) -> Result, SnapshotError> { + self.lists += 1; + if self.fail_listing { + return Err(SnapshotError::new( + SnapshotErrorCode::ObjectUnavailable, + "injected listing failure", + )); + } + let prefix = if path.is_empty() { + String::new() + } else { + format!("{path}/") + }; + Ok(self + .nodes + .iter() + .filter_map(|((owner, node_path), node)| { + if owner != source { + return None; + } + let suffix = node_path.strip_prefix(&prefix)?; + if suffix.is_empty() || suffix.contains('/') { + return None; + } + Some(SourceEntry { + name: suffix.to_string(), + node: node.clone(), + }) + }) + .collect()) + } +} + +fn binding(path: &str, source: &str, subpath: &str) -> FixedBinding { + FixedBinding { + mount_path: path.to_string(), + source: source.to_string(), + source_kind: SourceKind::Import, + source_subpath: subpath.to_string(), + policy: BindingPolicy::Mutable, + } +} + +#[test] +fn component_prefix_source_subpath_and_nested_routing_match_manual_table() { + let mut reader = MemoryReader::roots(); + reader.add("a20", "src", FsKind::Directory); + let index = FixedNamespaceIndex::new( + "native".to_string(), + vec![ + binding("/x/lib", "a20", "src"), + binding("/x/lib/vendor/b", "b30", ""), + ], + BTreeSet::new(), + &mut reader, + ) + .unwrap(); + for (path, expected_source, expected_path) in [ + ("/", "native", ""), + ("/x/lib", "a20", "src"), + ("/x/lib/a", "a20", "src/a"), + ("/x/library/a", "native", "x/library/a"), + ("/x/lib/vendor", "a20", "src/vendor"), + ("/x/lib/vendor/b/z", "b30", "z"), + ("/x/lib/vendor/beta/z", "a20", "src/vendor/beta/z"), + ("/x/lib/é+\\name", "a20", "src/é+\\name"), + ] { + let route = index.route(path).unwrap(); + assert_eq!(route.source, expected_source, "{path}"); + assert_eq!(route.source_path, expected_path, "{path}"); + } +} + +#[test] +fn synthetic_ancestors_have_no_source_and_exact_mount_has_no_native_union() { + let mut reader = MemoryReader::roots(); + reader.add("native", "lib", FsKind::Directory); + reader.add("native", "lib/native-only", FsKind::Regular); + reader.add("a20", "src", FsKind::Directory); + reader.add("a20", "src/import-only", FsKind::Executable); + let index = FixedNamespaceIndex::new( + "native".to_string(), + vec![ + binding("/lib", "a20", "src"), + binding("/missing/deep/mount", "b30", ""), + ], + BTreeSet::new(), + &mut reader, + ) + .unwrap(); + let synthetic = index.list("/missing", &mut reader).unwrap().unwrap(); + assert_eq!(synthetic.node_class, NodeClass::Aggregate); + assert!(synthetic.source_context.is_none()); + assert_eq!(synthetic.entries[0].name, "deep"); + assert!(synthetic.entries[0].oid.is_none()); + assert!(synthetic.entries[0].source_context.is_none()); + let imported = index.list("/lib", &mut reader).unwrap().unwrap(); + assert_eq!(imported.node_class, NodeClass::ImportRoot); + assert_eq!(imported.entries.len(), 1); + assert_eq!(imported.entries[0].name, "import-only"); + assert_eq!(imported.entries[0].fs_kind, FsKind::Executable); + assert!(index.list("/absent", &mut reader).unwrap().is_none()); +} + +#[test] +fn node_class_depends_on_direct_composition_and_explicit_checkout() { + let mut reader = MemoryReader::roots(); + reader.add("native", "app", FsKind::Directory); + reader.add("native", "plain", FsKind::Directory); + let index = FixedNamespaceIndex::new( + "native".to_string(), + vec![binding("/app/vendor/lib", "a20", "")], + BTreeSet::from(["/app".to_string(), "/plain".to_string()]), + &mut reader, + ) + .unwrap(); + assert_eq!( + index.list("/", &mut reader).unwrap().unwrap().node_class, + NodeClass::NativeTree + ); + let app = index.list("/app", &mut reader).unwrap().unwrap(); + assert_eq!(app.node_class, NodeClass::Aggregate); + assert!(app.source_context.is_some()); + assert_eq!( + index + .list("/plain", &mut reader) + .unwrap() + .unwrap() + .node_class, + NodeClass::NativeCheckoutRoot + ); +} + +#[test] +fn file_and_symlink_crossings_and_exact_mount_targets_are_conflicts() { + for kind in [FsKind::Regular, FsKind::Executable, FsKind::Symlink] { + for mount in ["/blocked", "/blocked/child"] { + let mut reader = MemoryReader::roots(); + reader.add("native", "blocked", kind); + let error = FixedNamespaceIndex::new( + "native".to_string(), + vec![binding(mount, "a20", "")], + BTreeSet::new(), + &mut reader, + ) + .unwrap_err(); + assert!(matches!(error, NamespaceError::Conflict(path) if path == "/blocked")); + } + } + let mut reader = MemoryReader::roots(); + reader.add("a20", "file", FsKind::Regular); + assert!(matches!( + FixedNamespaceIndex::new( + "native".to_string(), + vec![binding("/import", "a20", "file")], + BTreeSet::new(), + &mut reader, + ), + Err(NamespaceError::SourceSubpathNotDirectory(_)) + )); +} + +#[test] +fn nested_binding_cannot_hide_an_outer_source_file() { + let mut reader = MemoryReader::roots(); + reader.add("a20", "blocked", FsKind::Symlink); + let result = FixedNamespaceIndex::new( + "native".to_string(), + vec![ + binding("/import", "a20", ""), + binding("/import/blocked/nested", "b30", ""), + ], + BTreeSet::new(), + &mut reader, + ); + assert!(matches!(result, Err(NamespaceError::Conflict(path)) if path == "/import/blocked")); +} + +#[test] +fn removing_binding_restores_native_in_new_index_and_preserves_old_index() { + let mut reader = MemoryReader::roots(); + reader.add("native", "lib", FsKind::Directory); + reader.add("native", "lib/native-only", FsKind::Regular); + reader.add("a20", "old-only", FsKind::Regular); + reader.add("a21", "new-only", FsKind::Regular); + let old = FixedNamespaceIndex::new( + "native".to_string(), + vec![binding("/lib", "a20", "")], + BTreeSet::new(), + &mut reader, + ) + .unwrap(); + let new = FixedNamespaceIndex::new( + "native".to_string(), + Vec::new(), + BTreeSet::new(), + &mut reader, + ) + .unwrap(); + assert_eq!( + old.list("/lib", &mut reader).unwrap().unwrap().entries[0].name, + "old-only" + ); + assert_eq!( + new.list("/lib", &mut reader).unwrap().unwrap().entries[0].name, + "native-only" + ); + assert_eq!(old.route("/lib/unvisited/deep").unwrap().source, "a20"); +} + +#[test] +fn duplicate_mounts_and_noncanonical_paths_are_rejected() { + let mut reader = MemoryReader::roots(); + assert!(matches!( + FixedNamespaceIndex::new( + "native".to_string(), + vec![binding("/same", "a20", ""), binding("/same", "b30", "")], + BTreeSet::new(), + &mut reader, + ), + Err(NamespaceError::DuplicateMount(_)) + )); + for path in ["", "relative", "/a/", "/a//b", "/a/..", "/a\0"] { + assert!( + FixedNamespaceIndex::new( + "native".to_string(), + vec![binding(path, "a20", "")], + BTreeSet::new(), + &mut reader, + ) + .is_err(), + "{path:?}" + ); + } + for subpath in ["/src", "src/", "src//a", "src/..", "src\0"] { + assert!( + FixedNamespaceIndex::new( + "native".to_string(), + vec![binding("/lib", "a20", subpath)], + BTreeSet::new(), + &mut reader, + ) + .is_err(), + "{subpath:?}" + ); + } +} + +#[test] +fn composed_source_path_limits_are_checked_after_prefix_replacement() { + let mut reader = MemoryReader::roots(); + let subpath = vec!["a"; 256].join("/"); + reader.add("a20", &subpath, FsKind::Directory); + let index = FixedNamespaceIndex::new( + "native".to_string(), + vec![binding("/lib", "a20", &subpath)], + BTreeSet::new(), + &mut reader, + ) + .unwrap(); + assert!(index.route("/lib").is_ok()); + assert!(index.route("/lib/extra").is_err()); + assert!(index.route(&format!("/{}", "a".repeat(256))).is_err()); + assert!( + index + .route(&format!("/{}", vec!["a"; 257].join("/"))) + .is_err() + ); + let max_bytes = vec!["a".repeat(255); 16].join("/"); + assert_eq!(max_bytes.len() + 1, 4096); + reader.add("a20", &max_bytes, FsKind::Directory); + let index = FixedNamespaceIndex::new( + "native".to_string(), + vec![binding("/lib", "a20", &max_bytes)], + BTreeSet::new(), + &mut reader, + ) + .unwrap(); + assert!(index.route("/lib").is_ok()); + assert!(index.route("/lib/x").is_err()); +} + +#[test] +fn import_root_class_yields_to_direct_binding_and_import_tree_class_is_retained() { + let mut reader = MemoryReader::roots(); + reader.add("a20", "src", FsKind::Directory); + reader.add("a20", "src/plain", FsKind::Directory); + let index = FixedNamespaceIndex::new( + "native".to_string(), + vec![ + binding("/lib", "a20", "src"), + binding("/lib/nested", "b30", ""), + ], + BTreeSet::from(["/lib".to_string()]), + &mut reader, + ) + .unwrap(); + let root = index.list("/lib", &mut reader).unwrap().unwrap(); + assert_eq!(root.node_class, NodeClass::Aggregate); + assert_eq!(root.source_context.as_ref().unwrap().source, "a20"); + assert_eq!( + index + .list("/lib/plain", &mut reader) + .unwrap() + .unwrap() + .node_class, + NodeClass::ImportTree + ); + assert_eq!( + index + .list("/lib/nested", &mut reader) + .unwrap() + .unwrap() + .node_class, + NodeClass::ImportRoot + ); +} + +#[test] +fn storage_failure_is_preserved_in_construction_and_listing() { + let mut reader = MemoryReader::roots(); + let index = FixedNamespaceIndex::new( + "native".to_string(), + Vec::new(), + BTreeSet::new(), + &mut reader, + ) + .unwrap(); + reader.fail_path = Some("missing".to_string()); + assert!(matches!( + index.list("/missing", &mut reader), + Err(NamespaceError::Snapshot(error)) if error.code == SnapshotErrorCode::ObjectUnavailable + )); + assert!(matches!( + FixedNamespaceIndex::new( + "native".to_string(), + vec![binding("/lib", "a20", "missing")], + BTreeSet::new(), + &mut reader, + ), + Err(NamespaceError::Snapshot(error)) if error.code == SnapshotErrorCode::ObjectUnavailable + )); + reader.fail_path = None; + reader.fail_listing = true; + assert!(matches!( + index.list("/", &mut reader), + Err(NamespaceError::Snapshot(error)) if error.code == SnapshotErrorCode::ObjectUnavailable + )); +} + +#[test] +fn root_binding_replaces_entire_native_source() { + let mut reader = MemoryReader::roots(); + reader.add("native", "native-only", FsKind::Regular); + reader.add("a20", "import-only", FsKind::Regular); + let index = FixedNamespaceIndex::new( + "native".to_string(), + vec![binding("/", "a20", "")], + BTreeSet::new(), + &mut reader, + ) + .unwrap(); + assert_eq!(index.route("/").unwrap().source, "a20"); + assert_eq!(index.route("/anything").unwrap().source_path, "anything"); + let root = index.list("/", &mut reader).unwrap().unwrap(); + assert_eq!(root.node_class, NodeClass::ImportRoot); + assert_eq!(root.entries.len(), 1); + assert_eq!(root.entries[0].name, "import-only"); +} + +#[test] +fn invalid_and_duplicate_source_entries_are_rejected() { + struct MalformedReader(Vec); + impl FixedSourceReader for MalformedReader { + fn resolve(&mut self, _: &String, _: &str) -> Result { + Ok(SourceOutcome::Found(SourceNode { + fs_kind: FsKind::Directory, + oid: "root".to_string(), + })) + } + + fn list(&mut self, _: &String, _: &str) -> Result, SnapshotError> { + Ok(self + .0 + .iter() + .map(|name| SourceEntry { + name: name.clone(), + node: SourceNode { + fs_kind: FsKind::Regular, + oid: "blob".to_string(), + }, + }) + .collect()) + } + } + for names in [vec![""], vec![".."], vec!["a/b"], vec!["same", "same"]] { + let mut reader = MalformedReader(names.iter().map(|s| s.to_string()).collect()); + let index = FixedNamespaceIndex::new( + "native".to_string(), + Vec::new(), + BTreeSet::new(), + &mut reader, + ) + .unwrap(); + assert!(matches!( + index.list("/", &mut reader), + Err(NamespaceError::InvalidSourceEntry(_)) + )); + } +} + +#[test] +fn unrelated_binding_count_does_not_expand_source_reads_for_a_listing() { + let mut reader = MemoryReader::roots(); + reader.add("native", "app", FsKind::Directory); + let bindings = (0..512) + .map(|i| binding(&format!("/elsewhere-{i}/lib"), "a20", "")) + .collect(); + let index = + FixedNamespaceIndex::new("native".to_string(), bindings, BTreeSet::new(), &mut reader) + .unwrap(); + reader.resolves = 0; + reader.lists = 0; + index.list("/app", &mut reader).unwrap().unwrap(); + assert_eq!((reader.resolves, reader.lists), (1, 1)); +} + +fn git(repo: &Path, args: &[&str], input: Option<&[u8]>) -> String { + let mut child = Command::new("git") + .current_dir(repo) + .env("GIT_AUTHOR_NAME", "MST2 Oracle") + .env("GIT_AUTHOR_EMAIL", "mst2-oracle@example.invalid") + .env("GIT_COMMITTER_NAME", "MST2 Oracle") + .env("GIT_COMMITTER_EMAIL", "mst2-oracle@example.invalid") + .env("GIT_AUTHOR_DATE", "2026-09-15T00:00:00Z") + .env("GIT_COMMITTER_DATE", "2026-09-15T00:00:00Z") + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + if let Some(input) = input { + child.stdin.take().unwrap().write_all(input).unwrap(); + } else { + drop(child.stdin.take()); + } + let output = child.wait_with_output().unwrap(); + assert!( + output.status.success(), + "{args:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).unwrap() +} + +fn make_tree(repo: &Path, files: &[(&str, &str)]) -> String { + let mut items = Vec::new(); + let mut directories: BTreeMap<&str, Vec<(&str, &str)>> = BTreeMap::new(); + for (path, bytes) in files { + if let Some((head, tail)) = path.split_once('/') { + directories.entry(head).or_default().push((tail, bytes)); + } else { + let oid = git( + repo, + &["hash-object", "-w", "--stdin"], + Some(bytes.as_bytes()), + ); + items.push(format!("100644 blob {}\t{path}\n", oid.trim())); + } + } + for (name, children) in directories { + let oid = make_tree(repo, &children); + items.push(format!("040000 tree {oid}\t{name}\n")); + } + git(repo, &["mktree"], Some(items.concat().as_bytes())) + .trim() + .to_string() +} + +#[derive(Debug, Clone)] +struct GitSource { + root: String, + commit: String, + scope_path: String, +} + +struct GitReader<'a> { + repo: &'a Path, +} + +impl FixedSourceReader for GitReader<'_> { + fn resolve(&mut self, source: &GitSource, path: &str) -> Result { + if path.is_empty() { + return Ok(SourceOutcome::Found(SourceNode { + fs_kind: FsKind::Directory, + oid: source.root.clone(), + })); + } + let mut tree = source.root.clone(); + let components: Vec<_> = path.split('/').collect(); + for (position, component) in components.iter().enumerate() { + let Some(entry) = self + .read_tree(&tree) + .into_iter() + .find(|entry| entry.name == *component) + else { + return Ok(SourceOutcome::AbsentProven); + }; + if position + 1 == components.len() { + return Ok(SourceOutcome::Found(entry.node)); + } + if entry.node.fs_kind != FsKind::Directory { + return Err(SnapshotError::new( + SnapshotErrorCode::NotDirectory, + "fixed Git path crossing", + )); + } + tree = entry.node.oid; + } + unreachable!() + } + + fn list(&mut self, source: &GitSource, path: &str) -> Result, SnapshotError> { + match self.resolve(source, path)? { + SourceOutcome::Found(node) if node.fs_kind == FsKind::Directory => { + Ok(self.read_tree(&node.oid)) + } + _ => Err(SnapshotError::new( + SnapshotErrorCode::NotDirectory, + "fixed Git listing", + )), + } + } +} + +impl GitReader<'_> { + fn read_tree(&self, oid: &str) -> Vec { + git(self.repo, &["ls-tree", "-z", oid], None) + .split('\0') + .filter(|row| !row.is_empty()) + .map(|row| { + let (metadata, name) = row.split_once('\t').unwrap(); + let fields: Vec<_> = metadata.split_whitespace().collect(); + SourceEntry { + name: name.to_string(), + node: SourceNode { + fs_kind: if fields[0] == "040000" { + FsKind::Directory + } else { + FsKind::Regular + }, + oid: fields[2].to_string(), + }, + } + }) + .collect() + } +} + +fn fixed_source(repo: &Path, files: &[(&str, &str)]) -> GitSource { + let root = make_tree(repo, files); + let commit = git( + repo, + &["commit-tree", &root], + Some(b"fixed oracle fixture\n"), + ) + .trim() + .to_string(); + GitSource { + root, + commit, + scope_path: "/".to_string(), + } +} + +#[test] +fn attested_scoped_root_is_not_prefixed_again_when_subpath_and_nested_mount_are_used() { + let repo = TempDir::new().unwrap(); + git( + repo.path(), + &["init", "--bare", "--object-format=sha1", "--quiet"], + None, + ); + let native = fixed_source(repo.path(), &[("native.txt", "native")]); + let full = fixed_source( + repo.path(), + &[ + ("scope/src/deep/leaf.txt", "scoped"), + ("other/outside.txt", "outside"), + ], + ); + let scoped = GitSource { + root: git( + repo.path(), + &["rev-parse", &format!("{}:scope", full.commit)], + None, + ) + .trim() + .to_string(), + commit: full.commit.clone(), + scope_path: "/scope".to_string(), + }; + let nested = fixed_source(repo.path(), &[("inside/nested.txt", "nested")]); + let mut reader = GitReader { repo: repo.path() }; + let index = FixedNamespaceIndex::new( + native, + vec![ + git_binding("/lib", &scoped, "src"), + git_binding("/lib/deep/nested", &nested, "inside"), + ], + BTreeSet::new(), + &mut reader, + ) + .unwrap(); + let route = index.route("/lib/deep/leaf.txt").unwrap(); + assert_eq!(route.source.scope_path, "/scope"); + assert_eq!(route.source_path, "src/deep/leaf.txt"); + let route = index.route("/lib/deep/nested/nested.txt").unwrap(); + assert_eq!(route.source_path, "inside/nested.txt"); + let mut files = BTreeMap::new(); + traverse(&index, &mut reader, "/", &mut files); + assert_eq!( + files, + BTreeMap::from([ + ("/native.txt".to_string(), b"native".to_vec()), + ("/lib/deep/leaf.txt".to_string(), b"scoped".to_vec()), + ( + "/lib/deep/nested/nested.txt".to_string(), + b"nested".to_vec() + ), + ]) + ); +} + +fn git_binding(path: &str, source: &GitSource, subpath: &str) -> FixedBinding { + FixedBinding { + mount_path: path.to_string(), + source: source.clone(), + source_kind: SourceKind::Import, + source_subpath: subpath.to_string(), + policy: BindingPolicy::Mutable, + } +} + +/// Independently flatten commits using Git, then compose a handwritten fixture. +/// This oracle does not call the production trie or source reader to select paths. +fn flatten_commit(repo: &Path, source: &GitSource) -> BTreeMap> { + git( + repo, + &["ls-tree", "-rz", "--full-tree", &source.commit], + None, + ) + .split('\0') + .filter(|row| !row.is_empty()) + .map(|row| { + let (metadata, path) = row.split_once('\t').unwrap(); + let oid = metadata.split_whitespace().nth(2).unwrap(); + ( + path.to_string(), + git(repo, &["cat-file", "blob", oid], None).into_bytes(), + ) + }) + .collect() +} + +fn traverse( + index: &FixedNamespaceIndex, + reader: &mut GitReader<'_>, + path: &str, + files: &mut BTreeMap>, +) { + for entry in index.list(path, reader).unwrap().unwrap().entries { + let child = if path == "/" { + format!("/{}", entry.name) + } else { + format!("{path}/{}", entry.name) + }; + if entry.fs_kind == FsKind::Directory { + traverse(index, reader, &child, files); + } else { + files.insert( + child, + git( + reader.repo, + &["cat-file", "blob", entry.oid.as_deref().unwrap()], + None, + ) + .into_bytes(), + ); + } + } +} + +#[test] +fn composed_listing_matches_independent_fixed_git_oracle_after_ref_and_path_reuse() { + let repo = TempDir::new().unwrap(); + git( + repo.path(), + &["init", "--bare", "--object-format=sha1", "--quiet"], + None, + ); + let native = fixed_source( + repo.path(), + &[ + ("project/app/native.txt", "native"), + ("lib/native-hidden.txt", "hidden"), + ("library/keep.txt", "neighbor"), + ], + ); + let a20 = fixed_source( + repo.path(), + &[ + ("src/deep/old.txt", "A20"), + ("src/vendor/b/hidden.txt", "outer-hidden"), + ("elsewhere/not-visible.txt", "outside-subpath"), + ], + ); + let a21 = fixed_source(repo.path(), &[("src/deep/new.txt", "A21")]); + let b30 = fixed_source(repo.path(), &[("nested.txt", "B30")]); + let bindings = vec![ + git_binding("/lib", &a20, "src"), + git_binding("/lib/vendor/b", &b30, ""), + git_binding("/missing/deep/import", &b30, ""), + ]; + let mut reader = GitReader { repo: repo.path() }; + let old = + FixedNamespaceIndex::new(native.clone(), bindings, BTreeSet::new(), &mut reader).unwrap(); + // Simulate advancing current refs and reassigning a registry path. The + // reader has only the captured fixed handle; no registry/ref read is possible. + git( + repo.path(), + &["update-ref", "refs/heads/import-current", &a21.commit], + None, + ); + let current_registry = BTreeMap::from([("/lib", a21.clone())]); + assert_ne!( + current_registry["/lib"].commit, + old.route("/lib/deep/old.txt").unwrap().source.commit + ); + let mut expected = BTreeMap::new(); + for (path, bytes) in flatten_commit(repo.path(), &native) { + if !path.starts_with("lib/") { + expected.insert(format!("/{path}"), bytes); + } + } + for (path, bytes) in flatten_commit(repo.path(), &a20) { + if let Some(suffix) = path.strip_prefix("src/") + && !suffix.starts_with("vendor/b/") + { + expected.insert(format!("/lib/{suffix}"), bytes); + } + } + for (path, bytes) in flatten_commit(repo.path(), &b30) { + expected.insert(format!("/lib/vendor/b/{path}"), bytes.clone()); + expected.insert(format!("/missing/deep/import/{path}"), bytes); + } + let mut actual = BTreeMap::new(); + traverse(&old, &mut reader, "/", &mut actual); + assert_eq!(actual, expected); + assert!(actual.contains_key("/lib/deep/old.txt")); + assert!(!actual.contains_key("/lib/deep/new.txt")); + let unbound = + FixedNamespaceIndex::new(native.clone(), Vec::new(), BTreeSet::new(), &mut reader).unwrap(); + let mut restored = BTreeMap::new(); + traverse(&unbound, &mut reader, "/", &mut restored); + let expected_native = flatten_commit(repo.path(), &native) + .into_iter() + .map(|(path, bytes)| (format!("/{path}"), bytes)) + .collect::>(); + assert_eq!(restored, expected_native); + let mut old_again = BTreeMap::new(); + traverse(&old, &mut reader, "/", &mut old_again); + assert_eq!(old_again, expected); +} diff --git a/src/ceres/snapshot/pages.rs b/src/ceres/snapshot/pages.rs index 74824a38..555ed913 100644 --- a/src/ceres/snapshot/pages.rs +++ b/src/ceres/snapshot/pages.rs @@ -15,13 +15,16 @@ use mst2_codec::metapage::{Entry, EntryKind, Page, page_id}; use sea_orm::ActiveValue::Set; use sha2::{Digest, Sha256}; -use crate::ceres::{ - api_service::ApiHandler, - snapshot::{ - error::{SnapshotError, SnapshotErrorCode}, - resolver::FsKind, - view::hex, +use crate::{ + ceres::{ + api_service::ApiHandler, + snapshot::{ + error::{SnapshotError, SnapshotErrorCode}, + resolver::FsKind, + view::hex, + }, }, + jupiter::storage::mono_storage::MST2_VERIFICATION_VERSION, }; /// A directory page plus everything the JSON layer needs. @@ -99,37 +102,43 @@ async fn build_directory_page_uncached( let dirents = crate::ceres::snapshot::resolver::direct_entries(&tree)?; // T03 write-through verification: consult verified records first, then - // fetch + hash the misses and persist them. A verification-record read - // failure degrades to recomputation (safe: strictly more verification). + // fetch + hash the misses and persist them. Invalid records and lookup + // failures are errors, never a missing verification fact. let blob_oids: Vec = dirents .iter() .filter(|(_, k, _)| *k != FsKind::Directory) .map(|(_, _, oid)| oid.clone()) .collect(); - let verified = match handler + let verified = handler .get_context() .mono_storage() .get_verified_blobs(blob_oids) .await - { - Ok(map) => map, - Err(e) => { - tracing::warn!(error = %e, "verified blob lookup failed; recomputing digests"); - HashMap::new() - } - }; + .map_err(|e| { + let code = if matches!( + e, + crate::common::errors::MegaError::ObjStorageInconsistent(_) + ) { + SnapshotErrorCode::IntegrityError + } else { + SnapshotErrorCode::Internal + }; + tracing::warn!(error = %e, "verified blob lookup failed"); + SnapshotError::new(code, "verified blob lookup failed") + })?; - let mut new_verified: Vec = Vec::new(); + let mut new_verified = HashMap::new(); let mut entries = Vec::with_capacity(dirents.len()); let mut codec_entries = Vec::with_capacity(dirents.len()); for (name, fs_kind, oid) in dirents { + let child_rel = if rel_path == "/" { + format!("/{name}") + } else { + format!("{rel_path}/{name}") + }; + crate::ceres::snapshot::view::validate_scope_relative_path(&child_rel)?; match fs_kind { FsKind::Directory => { - let child_rel = if rel_path == "/" { - format!("/{name}") - } else { - format!("{rel_path}/{name}") - }; let child_page = Box::pin(build_directory_page(handler, root_tree, &child_rel)).await?; codec_entries.push(Entry::dir(name.as_bytes(), child_page.page_id)); @@ -145,24 +154,37 @@ async fn build_directory_page_uncached( FsKind::Regular | FsKind::Executable | FsKind::Symlink => { let (size, digest) = if let Some(v) = verified.get(&oid) { // Verified record: 64-bit size + raw digest, no content read. - let d: [u8; 32] = v.raw_sha256.clone().try_into().unwrap_or([0u8; 32]); - (v.size as u64, d) + let d: [u8; 32] = v.raw_sha256.as_slice().try_into().map_err(|_| { + SnapshotError::new( + SnapshotErrorCode::IntegrityError, + "invalid verified content digest", + ) + })?; + let size = u64::try_from(v.size).map_err(|_| { + SnapshotError::new( + SnapshotErrorCode::IntegrityError, + "invalid verified content size", + ) + })?; + (size, d) } else { let raw = fetch_raw_blob(handler, &oid).await?; let mut h = Sha256::new(); h.update(&raw); let digest: [u8; 32] = h.finalize().into(); - new_verified.push(crate::callisto::mst2_verified_object::ActiveModel { - id: sea_orm::ActiveValue::NotSet, - storage_domain: Set("git".to_string()), - git_oid: Set(oid.clone()), - object_kind: Set("blob".to_string()), - raw_sha256: Set(digest.to_vec()), - size: Set(raw.len() as i64), - verification_version: Set(1), - state: Set("VERIFIED".to_string()), - created_at: Set(chrono_now()), - }); + new_verified.entry(oid.clone()).or_insert( + crate::callisto::mst2_verified_object::ActiveModel { + id: sea_orm::ActiveValue::NotSet, + storage_domain: Set("git".to_string()), + git_oid: Set(oid.clone()), + object_kind: Set("blob".to_string()), + raw_sha256: Set(digest.to_vec()), + size: Set(raw.len() as i64), + verification_version: Set(MST2_VERIFICATION_VERSION), + state: Set("VERIFIED".to_string()), + created_at: Set(chrono_now()), + }, + ); (raw.len() as u64, digest) }; let kind = match fs_kind { @@ -189,7 +211,7 @@ async fn build_directory_page_uncached( if let Err(e) = handler .get_context() .mono_storage() - .insert_verified_blobs(new_verified) + .insert_verified_blobs(new_verified.into_values().collect()) .await { tracing::warn!(error = %e, "verified blob persistence failed"); @@ -242,6 +264,7 @@ async fn fetch_tree( root_tree: &git_internal::internal::object::tree::Tree, rel_path: &str, ) -> Result { + crate::ceres::snapshot::view::validate_scope_relative_path(rel_path)?; if rel_path == "/" { return Ok(root_tree.clone()); } @@ -329,6 +352,7 @@ pub async fn resolve_abs( root_tree: &git_internal::internal::object::tree::Tree, abs_path: &str, ) -> Result { + crate::ceres::snapshot::view::validate_scope_relative_path(abs_path)?; if abs_path == "/" { return Ok(WalkOutcome::FoundDir); } @@ -379,7 +403,7 @@ pub async fn resolve_abs( unreachable!("loop returns on the last component") } -/// Raw blob bytes with the `blob \0` git header stripped. +/// GitService stores and returns raw file content, without an object header. pub async fn fetch_raw_blob( handler: &T, oid: &str, @@ -387,24 +411,19 @@ pub async fn fetch_raw_blob( let data = handler.get_raw_blob_by_hash(oid).await.map_err(|e| { // Missing/corrupt content must be an error, never an empty file // (spec 00 SYS-04). - SnapshotError::new( - SnapshotErrorCode::Internal, - format!("blob fetch failed for {oid}: {e}"), - ) + let code = match e { + crate::common::errors::MegaError::ObjStorageNotFound(_) => { + SnapshotErrorCode::ObjectUnavailable + } + crate::common::errors::MegaError::ObjStorageInconsistent(_) => { + SnapshotErrorCode::IntegrityError + } + _ => SnapshotErrorCode::Internal, + }; + tracing::warn!(error = %e, "fixed-view blob fetch failed"); + SnapshotError::new(code, "fixed-view content could not be read") })?; - Ok(strip_git_blob_header(&data)) -} - -fn strip_git_blob_header(data: &[u8]) -> Vec { - if let Some(pos) = data.iter().position(|&b| b == 0) - && data.starts_with(b"blob ") - && let Ok(len) = std::str::from_utf8(&data[5..pos]).map(|s| s.parse::()) - && let Ok(len) = len - && data.len() == pos + 1 + len - { - return data[pos + 1..].to_vec(); - } - data.to_vec() + Ok(data) } pub fn sha256_hex(data: &[u8]) -> String { @@ -426,3 +445,486 @@ fn chrono_now() -> chrono::DateTime { pub fn hex_of(id: &[u8; 32]) -> String { hex(id) } + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use axum::{body::Body, http::Request}; + use bytes::Bytes; + use git_internal::internal::{ + metadata::EntryMeta, + object::{ + blob::Blob, + tree::{Tree, TreeItem, TreeItemMode}, + }, + }; + use sea_orm::{ActiveValue::Set, EntityTrait, IntoActiveModel}; + use tower::ServiceExt; + + use super::*; + use crate::{ + api::{ + MonoApiServiceState, oauth::api_store::BrowserSessionStore, router::snapshot_router, + }, + callisto::{mega_refs, mst2_verified_object}, + ceres::api_service::cache::GitObjectCache, + config::testing::isolated_config, + jupiter::{ + storage::base_storage::StorageConnector, + tests::{test_redis_manager, test_storage_with_config}, + utils::converter::IntoMegaModel, + }, + }; + + async fn state(temp: &std::path::Path) -> MonoApiServiceState { + let mut config = isolated_config(temp.join("config")); + config.mst2.enabled = true; + config.mst2.instance_uuid = Some(uuid::Uuid::new_v4().to_string()); + config.mst2.auth_token = Some("mst2-projection-test".to_string()); + let storage = test_storage_with_config(temp, config).await; + MonoApiServiceState { + entity_store: storage.entity_store.clone(), + storage, + session_store: BrowserSessionStore::Anonymous, + git_object_cache: Arc::new(GitObjectCache { + connection: test_redis_manager().await, + prefix: String::new(), + }), + listen_addr: "127.0.0.1:0".to_string(), + } + } + + async fn seed(state: &MonoApiServiceState, filename: &str, raw: &[u8]) -> String { + seed_path(state, &format!("/{filename}"), raw).await + } + + async fn seed_path(state: &MonoApiServiceState, path: &str, raw: &[u8]) -> String { + let blob = Blob::from_content_bytes(raw.to_vec()); + state + .storage + .git_service + .save_object_from_raw(Bytes::copy_from_slice(raw)) + .await + .unwrap(); + let mut child_id = blob.id; + let mut child_mode = TreeItemMode::Blob; + let mut trees = Vec::new(); + for name in path[1..].split('/').rev() { + let tree = + Tree::from_tree_items(vec![TreeItem::new(child_mode, child_id, name.to_string())]) + .unwrap(); + child_id = tree.id; + child_mode = TreeItemMode::Tree; + trees.push(tree.into_mega_model(EntryMeta::new()).into_active_model()); + } + let mono = state.storage.mono_storage(); + mono.batch_save_model(trees).await.unwrap(); + mono.save_refs( + mega_refs::Model::new( + "/", + "refs/heads/main".to_string(), + blob.id.to_string(), + child_id.to_string(), + false, + ), + None, + ) + .await + .unwrap(); + blob.id.to_string() + } + + async fn seed_legacy(state: &MonoApiServiceState, oid: &str, raw: &[u8]) { + mst2_verified_object::Entity::insert(mst2_verified_object::ActiveModel { + id: sea_orm::ActiveValue::NotSet, + storage_domain: Set("git".to_string()), + git_oid: Set(oid.to_string()), + object_kind: Set("blob".to_string()), + raw_sha256: Set(Sha256::digest(raw).to_vec()), + size: Set(raw.len() as i64), + verification_version: Set(1), + state: Set("VERIFIED".to_string()), + created_at: Set(chrono::Utc::now().fixed_offset()), + }) + .exec(state.storage.mono_storage().get_connection()) + .await + .unwrap(); + } + + async fn snapshot_id(state: &MonoApiServiceState, scope: &str) -> String { + let main = state + .storage + .mono_storage() + .get_main_ref("/") + .await + .unwrap() + .unwrap(); + let handler = crate::ceres::api_service::mono_api_service::MonoApiService::from(state); + let tree = handler.get_tree_by_hash(&main.ref_tree_hash).await.unwrap(); + let built = build_directory_page(&handler, &tree, scope).await.unwrap(); + crate::ceres::snapshot::descriptor::build( + &state.storage.config().mst2, + &crate::ceres::snapshot::view::SnapshotView::from_commit( + &main.ref_commit_hash, + &main.ref_tree_hash, + ), + scope, + built.page_id, + ) + .unwrap() + .snapshot_id + } + + fn request(method: &str, uri: &str, lease: Option<&str>, body: Body) -> Request { + let mut request = Request::builder() + .method(method) + .uri(uri) + .header("authorization", "Bearer mst2-projection-test") + .header("content-type", "application/json"); + if let Some(lease) = lease { + request = request.header("x-mega-snapshot-lease", lease); + } + request.body(body).unwrap() + } + + #[tokio::test] + async fn mst2_raw_header_like_bytes_survive_projection_and_http_reads() { + assert_header_like_blob_http(false).await; + } + + #[tokio::test] + async fn mst2_legacy_header_like_fact_is_reverified_before_http_projection() { + assert_header_like_blob_http(true).await; + } + + async fn assert_header_like_blob_http(legacy: bool) { + let temp = tempfile::TempDir::new().unwrap(); + let state = state(temp.path()).await; + let raw = b"blob 3\0abc"; + let filename = format!("binary-{}", uuid::Uuid::new_v4()); + let oid = seed(&state, &filename, raw).await; + if legacy { + seed_legacy(&state, &oid, b"abc").await; + } + let app = snapshot_router::routers(state.clone()).with_state(state.clone()); + let resolve = app + .clone() + .oneshot(request( + "POST", + "/snapshots/resolve", + None, + Body::from(r#"{"target":{"kind":"latest"}}"#), + )) + .await + .unwrap(); + assert_eq!(resolve.status(), 200); + let resolved: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(resolve.into_body(), 1_048_576) + .await + .unwrap(), + ) + .unwrap(); + let snapshot = snapshot_id(&state, "/").await; + let lease = resolved["lease_id"].as_str().unwrap(); + let digest = sha256_hex(raw); + let rows = state + .storage + .mono_storage() + .get_verified_blobs(vec![oid.clone()]) + .await + .unwrap(); + assert_eq!(rows[&oid].size, raw.len() as i64); + assert_eq!(rows[&oid].raw_sha256, Sha256::digest(raw).to_vec()); + assert_eq!(rows[&oid].verification_version, MST2_VERIFICATION_VERSION); + + let directory = app + .clone() + .oneshot(request( + "GET", + &format!("/snapshots/{snapshot}/directory"), + Some(lease), + Body::empty(), + )) + .await + .unwrap(); + assert_eq!(directory.status(), 200); + let directory: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(directory.into_body(), 1_048_576) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(directory["entries"][0]["size"], raw.len().to_string()); + assert_eq!( + directory["entries"][0]["content_digest"], + format!("sha256:{digest}") + ); + let uri = + format!("/snapshots/{snapshot}/blob?path=/{filename}&expected_digest=sha256:{digest}"); + let get = app + .clone() + .oneshot(request("GET", &uri, Some(lease), Body::empty())) + .await + .unwrap(); + assert_eq!(get.status(), 200); + assert_eq!( + axum::body::to_bytes(get.into_body(), 128) + .await + .unwrap() + .as_ref(), + raw + ); + let head = app + .clone() + .oneshot(request("HEAD", &uri, Some(lease), Body::empty())) + .await + .unwrap(); + assert_eq!(head.status(), 200); + assert_eq!(head.headers()["content-length"], raw.len().to_string()); + assert!( + axum::body::to_bytes(head.into_body(), 128) + .await + .unwrap() + .is_empty() + ); + let again = app + .oneshot(request( + "POST", + "/snapshots/resolve", + None, + Body::from(r#"{"target":{"kind":"latest"}}"#), + )) + .await + .unwrap(); + assert_eq!(again.status(), 200); + let again: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(again.into_body(), 1_048_576) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(again["descriptor"], resolved["descriptor"]); + crate::ceres::snapshot::runtime::runtime() + .release_lease(again["lease_id"].as_str().unwrap()); + crate::ceres::snapshot::runtime::runtime().release_lease(lease); + } + + #[tokio::test] + async fn mst2_invalid_verified_record_is_not_recomputed_into_a_successful_snapshot() { + let temp = tempfile::TempDir::new().unwrap(); + let state = state(temp.path()).await; + let filename = format!("corrupt-{}", uuid::Uuid::new_v4()); + let raw = b"valid stored content"; + let oid = seed(&state, &filename, raw).await; + mst2_verified_object::Entity::insert(mst2_verified_object::ActiveModel { + id: sea_orm::ActiveValue::NotSet, + storage_domain: Set("git".to_string()), + git_oid: Set(oid), + object_kind: Set("blob".to_string()), + raw_sha256: Set(vec![7; 31]), + size: Set(raw.len() as i64), + verification_version: Set(MST2_VERIFICATION_VERSION), + state: Set("VERIFIED".to_string()), + created_at: Set(chrono::Utc::now().fixed_offset()), + }) + .exec(state.storage.mono_storage().get_connection()) + .await + .unwrap(); + let response = snapshot_router::routers(state.clone()) + .with_state(state) + .oneshot(request( + "POST", + "/snapshots/resolve", + None, + Body::from(r#"{"target":{"kind":"latest"}}"#), + )) + .await + .unwrap(); + assert_eq!(response.status(), 502); + let body: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(response.into_body(), 1_048_576) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(body["error"]["code"], "INTEGRITY_ERROR"); + } + + #[tokio::test] + async fn mst2_legacy_fact_cannot_upgrade_when_raw_storage_is_unavailable() { + use sea_orm::{ColumnTrait, QueryFilter}; + + use crate::orbit_api::object_storage::{ObjectKey, ObjectNamespace}; + + let temp = tempfile::TempDir::new().unwrap(); + let state = state(temp.path()).await; + let filename = format!("missing-{}", uuid::Uuid::new_v4()); + let oid = seed(&state, &filename, b"blob 3\0abc").await; + seed_legacy(&state, &oid, b"abc").await; + state + .storage + .git_service + .obj_storage + .inner + .delete(&ObjectKey { + namespace: ObjectNamespace::Git, + key: oid.clone(), + }) + .await + .unwrap(); + let response = snapshot_router::routers(state.clone()) + .with_state(state.clone()) + .oneshot(request( + "POST", + "/snapshots/resolve", + None, + Body::from(r#"{"target":{"kind":"latest"}}"#), + )) + .await + .unwrap(); + assert_eq!(response.status(), 503); + let body: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(response.into_body(), 1_048_576) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(body["error"]["code"], "OBJECT_UNAVAILABLE"); + let saved = mst2_verified_object::Entity::find() + .filter(mst2_verified_object::Column::GitOid.eq(oid)) + .one(state.storage.mono_storage().get_connection()) + .await + .unwrap() + .unwrap(); + assert_eq!(saved.verification_version, 1); + assert_eq!(saved.size, 3); + assert_eq!(saved.raw_sha256, Sha256::digest(b"abc").to_vec()); + } + + #[tokio::test] + async fn mst2_shared_blob_paths_upgrade_one_verification_record() { + let temp = tempfile::TempDir::new().unwrap(); + let state = state(temp.path()).await; + let raw = b"blob 3\0abc"; + let oid = state + .storage + .git_service + .save_object_from_raw(Bytes::copy_from_slice(raw)) + .await + .unwrap(); + seed_legacy(&state, &oid, b"abc").await; + let blob = Blob::from_content_bytes(raw.to_vec()); + let tree = Tree::from_tree_items(vec![ + TreeItem::new( + TreeItemMode::Blob, + blob.id, + format!("a-{}", uuid::Uuid::new_v4()), + ), + TreeItem::new( + TreeItemMode::Blob, + blob.id, + format!("b-{}", uuid::Uuid::new_v4()), + ), + ]) + .unwrap(); + let handler = crate::ceres::api_service::mono_api_service::MonoApiService::from(&state); + let built = build_directory_page_uncached(&handler, &tree, "/") + .await + .unwrap(); + assert_eq!(built.entries.len(), 2); + assert!(built.entries.iter().all(|entry| entry.size == Some(10) + && entry.content_digest == Some(Sha256::digest(raw).into()))); + let rows = state + .storage + .mono_storage() + .get_verified_blobs(vec![oid.clone()]) + .await + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[&oid].size, 10); + assert_eq!(rows[&oid].verification_version, MST2_VERIFICATION_VERSION); + } + + #[tokio::test] + async fn mst2_projection_checks_full_leaf_path_before_resolve_and_http_reads() { + for (component_boundary, valid) in + [(true, true), (true, false), (false, true), (false, false)] + { + let temp = tempfile::TempDir::new().unwrap(); + let state = state(temp.path()).await; + let marker = uuid::Uuid::new_v4().to_string(); + let mut components = if component_boundary { + let mut parts = vec!["d".to_string(); if valid { 255 } else { 256 }]; + parts[0] = marker; + parts.push("file".to_string()); + parts + } else { + let mut parts = vec!["d".repeat(255); 15]; + parts[0].replace_range(..marker.len(), &marker); + parts.push("d".repeat(127)); + parts.push("f".repeat(if valid { 127 } else { 128 })); + parts + }; + let filename = components.pop().unwrap(); + let scope = format!("/{}", components.join("/")); + let full_path = format!("{scope}/{filename}"); + if component_boundary { + assert_eq!( + full_path[1..].split('/').count(), + if valid { 256 } else { 257 } + ); + } else { + assert_eq!(full_path.len(), if valid { 4096 } else { 4097 }); + } + let raw = b"boundary file bytes"; + seed_path(&state, &full_path, raw).await; + let app = snapshot_router::routers(state.clone()).with_state(state.clone()); + let resolve = app + .clone() + .oneshot(request( + "POST", + "/snapshots/resolve", + None, + Body::from( + serde_json::json!({"target":{"kind":"latest"}, "scope":scope}).to_string(), + ), + )) + .await + .unwrap(); + let status = resolve.status(); + let resolved: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(resolve.into_body(), 1_048_576) + .await + .unwrap(), + ) + .unwrap(); + if !valid { + assert_eq!(status, 400); + assert_eq!(resolved["error"]["code"], "SCOPE_INVALID"); + assert!(resolved.get("lease_id").is_none()); + continue; + } + assert_eq!(status, 200); + let snapshot = snapshot_id(&state, &scope).await; + let lease = resolved["lease_id"].as_str().unwrap(); + let uri = format!( + "/snapshots/{snapshot}/blob?path=/{filename}&expected_digest=sha256:{}", + sha256_hex(raw) + ); + let get = app + .oneshot(request("GET", &uri, Some(lease), Body::empty())) + .await + .unwrap(); + assert_eq!(get.status(), 200); + assert_eq!( + axum::body::to_bytes(get.into_body(), 128) + .await + .unwrap() + .as_ref(), + raw + ); + crate::ceres::snapshot::runtime::runtime().release_lease(lease); + } + } +} diff --git a/src/ceres/snapshot/runtime.rs b/src/ceres/snapshot/runtime.rs index 03cafef6..770c4725 100644 --- a/src/ceres/snapshot/runtime.rs +++ b/src/ceres/snapshot/runtime.rs @@ -349,7 +349,15 @@ mod tests { use crate::ceres::snapshot::retention::{ NodeState, RetainedKind, RetentionRoot, RetentionStore as _, }; - let r = runtime(); + let r = Mst2Runtime { + hmac_key: blake3_key(), + contexts: Mutex::new(HashMap::new()), + leases: Mutex::new(HashMap::new()), + retention: crate::ceres::snapshot::retention::RetentionCoordinator::new( + crate::ceres::snapshot::retention::mem::InMemoryRetentionStore::default(), + ), + tip_state: Mutex::new((String::new(), 0u64)), + }; // Retain the snapshot's derived nodes directly, covered by the lease // root that insert_context created. let lease_root = RetentionRoot::Lease("gc-lease-1".to_string()); diff --git a/src/ceres/snapshot/view.rs b/src/ceres/snapshot/view.rs index 5f81cba5..88d1352c 100644 --- a/src/ceres/snapshot/view.rs +++ b/src/ceres/snapshot/view.rs @@ -62,15 +62,20 @@ pub fn validate_scope_relative_path(path: &str) -> Result<(), SnapshotError> { if path == "/" { return Ok(()); } + let mut component_count = 0; for comp in path[1..].split('/') { + component_count += 1; + if component_count > 256 { + return Err(invalid("path has more than 256 components")); + } if comp.is_empty() { return Err(invalid("empty path component")); } if comp == "." || comp == ".." { return Err(invalid("dot component")); } - if comp.len() > 256 { - return Err(invalid("path component over 256 bytes")); + if comp.len() > 255 { + return Err(invalid("path component over 255 bytes")); } } Ok(()) @@ -99,4 +104,14 @@ mod tests { assert!(validate_scope_relative_path("/a/").is_err()); assert!(validate_scope_relative_path("/a\0").is_err()); } + + #[test] + fn scope_relative_path_enforces_byte_and_component_boundaries() { + assert!(validate_scope_relative_path(&format!("/{}", "a".repeat(255))).is_ok()); + assert!(validate_scope_relative_path(&format!("/{}", "a".repeat(256))).is_err()); + assert!(validate_scope_relative_path(&format!("/{}a", "é".repeat(127))).is_ok()); + assert!(validate_scope_relative_path(&format!("/{}", "é".repeat(128))).is_err()); + assert!(validate_scope_relative_path(&format!("/{}", vec!["a"; 256].join("/"))).is_ok()); + assert!(validate_scope_relative_path(&format!("/{}", vec!["a"; 257].join("/"))).is_err()); + } } diff --git a/src/jupiter/storage/mono_storage.rs b/src/jupiter/storage/mono_storage.rs index 550d8ced..dc140b2e 100644 --- a/src/jupiter/storage/mono_storage.rs +++ b/src/jupiter/storage/mono_storage.rs @@ -47,6 +47,10 @@ pub struct MonoStorage { pub base: BaseStorage, } +/// Generation 2 hashes the raw file body without guessing a Git envelope. +pub const MST2_VERIFICATION_VERSION: i32 = 2; +const MST2_MAX_FILE_SIZE: i64 = 8_796_093_022_208; + impl Deref for MonoStorage { type Target = BaseStorage; fn deref(&self) -> &Self::Target { @@ -1730,12 +1734,29 @@ impl MonoStorage { .filter(mst2_verified_object::Column::GitOid.is_in(oids)) .all(self.get_connection()) .await?; - Ok(rows.into_iter().map(|m| (m.git_oid.clone(), m)).collect()) + for row in &rows { + if row.state != "VERIFIED" + || ![1, MST2_VERIFICATION_VERSION].contains(&row.verification_version) + || !(0..=MST2_MAX_FILE_SIZE).contains(&row.size) + || row.raw_sha256.len() != 32 + { + return Err(MegaError::ObjStorageInconsistent( + "invalid MST/2 verified blob record".to_string(), + )); + } + } + // Generation 1 may have stripped header-like file bytes. Reverify + // valid legacy rows from object storage before trusting their facts. + Ok(rows + .into_iter() + .filter(|m| m.verification_version == MST2_VERIFICATION_VERSION) + .map(|m| (m.git_oid.clone(), m)) + .collect()) } /// Write-through verification: rows are only written after the content /// was fetched and hashed. Conflicts (concurrent verification of the same - /// oid) resolve to the existing row. + /// oid) keep current facts. Only valid generation-1 facts may be upgraded. pub async fn insert_verified_blobs( &self, rows: Vec, @@ -1743,6 +1764,23 @@ impl MonoStorage { if rows.is_empty() { return Ok(()); } + for row in &rows { + if row.state.try_as_ref().map(String::as_str) != Some("VERIFIED") + || row.verification_version.try_as_ref() != Some(&MST2_VERIFICATION_VERSION) + || !row + .size + .try_as_ref() + .is_some_and(|size| (0..=MST2_MAX_FILE_SIZE).contains(size)) + || !row + .raw_sha256 + .try_as_ref() + .is_some_and(|digest| digest.len() == 32) + { + return Err(MegaError::ObjStorageInconsistent( + "invalid MST/2 verified blob write".to_string(), + )); + } + } match mst2_verified_object::Entity::insert_many(rows) .on_conflict( OnConflict::columns([ @@ -1750,15 +1788,43 @@ impl MonoStorage { mst2_verified_object::Column::GitOid, mst2_verified_object::Column::ObjectKind, ]) - .do_nothing() + .update_columns([ + mst2_verified_object::Column::RawSha256, + mst2_verified_object::Column::Size, + mst2_verified_object::Column::VerificationVersion, + mst2_verified_object::Column::CreatedAt, + ]) + .action_and_where( + Expr::col(( + mst2_verified_object::Entity, + mst2_verified_object::Column::VerificationVersion, + )) + .eq(1), + ) + .action_and_where( + Expr::col(( + mst2_verified_object::Entity, + mst2_verified_object::Column::State, + )) + .eq("VERIFIED"), + ) + .action_and_where( + Expr::col(( + mst2_verified_object::Entity, + mst2_verified_object::Column::Size, + )) + .between(0, MST2_MAX_FILE_SIZE), + ) + .action_and_where(Expr::cust( + "length(\"mst2_verified_object\".\"raw_sha256\") = 32", + )) .to_owned(), ) .exec(self.get_connection()) .await { - // ON CONFLICT DO NOTHING returns RecordNotInserted when every - // row was a duplicate; that is the intended first-write-wins - // outcome (same convention as next_insert_retry). + // A skipped conditional conflict returns RecordNotInserted; + // current-generation facts retain first-write-wins semantics. Ok(_) | Err(DbErr::RecordNotInserted) => Ok(()), Err(e) => Err(e.into()), } @@ -3339,7 +3405,7 @@ mod tests { object_kind: Set("blob".to_string()), raw_sha256: Set(digest.clone()), size: Set(4_294_967_296), // > i32::MAX: the size64 gap this table closes - verification_version: Set(1), + verification_version: Set(MST2_VERIFICATION_VERSION), state: Set("VERIFIED".to_string()), created_at: Set(chrono::Utc::now().fixed_offset()), }]) @@ -3364,7 +3430,7 @@ mod tests { object_kind: Set("blob".to_string()), raw_sha256: Set(vec![9u8; 32]), size: Set(1), - verification_version: Set(1), + verification_version: Set(MST2_VERIFICATION_VERSION), state: Set("VERIFIED".to_string()), created_at: Set(chrono::Utc::now().fixed_offset()), }]) @@ -3381,8 +3447,7 @@ mod tests { #[tokio::test] async fn mst2_verified_blob_insert_real_error_is_not_swallowed() { - // The ON CONFLICT DO NOTHING sentinel (RecordNotInserted on an - // all-duplicate batch) is treated as idempotent success; a genuine + // A skipped conditional conflict is idempotent success; a genuine // database error must still surface, never be masked as Ok. let temp = tempfile::TempDir::new().unwrap(); let storage = test_storage(temp.path()).await; @@ -3402,7 +3467,7 @@ mod tests { object_kind: Set("blob".to_string()), raw_sha256: Set(vec![2u8; 32]), size: Set(2), - verification_version: Set(1), + verification_version: Set(MST2_VERIFICATION_VERSION), state: Set("VERIFIED".to_string()), created_at: Set(chrono::Utc::now().fixed_offset()), }]) @@ -3412,4 +3477,169 @@ mod tests { "a real DB error must propagate, not be swallowed by the ON CONFLICT sentinel match" ); } + + #[tokio::test] + async fn mst2_verified_blob_invalid_records_reject_the_entire_lookup() { + let temp = tempfile::TempDir::new().unwrap(); + let storage = test_storage(temp.path()).await; + let mono = storage.mono_storage(); + let valid_oid = "f".repeat(40); + let row = |oid: String, state: &str, version, size, digest: Vec| { + mst2_verified_object::ActiveModel { + id: sea_orm::ActiveValue::NotSet, + storage_domain: Set("git".to_string()), + git_oid: Set(oid), + object_kind: Set("blob".to_string()), + raw_sha256: Set(digest), + size: Set(size), + verification_version: Set(version), + state: Set(state.to_string()), + created_at: Set(chrono::Utc::now().fixed_offset()), + } + }; + mono.insert_verified_blobs(vec![row( + valid_oid.clone(), + "VERIFIED", + MST2_VERIFICATION_VERSION, + 0, + vec![7; 32], + )]) + .await + .unwrap(); + for (index, (state, version, size, digest_len)) in [ + ("PENDING", 1, 1, 32), + ("VERIFIED", 0, 1, 32), + ("VERIFIED", 3, 1, 32), + ("VERIFIED", 1, -1, 32), + ("VERIFIED", 1, 8_796_093_022_209, 32), + ("VERIFIED", 1, 1, 31), + ("VERIFIED", 1, 1, 33), + ("VERIFIED", MST2_VERIFICATION_VERSION, -1, 32), + ("VERIFIED", MST2_VERIFICATION_VERSION, 8_796_093_022_209, 32), + ("VERIFIED", MST2_VERIFICATION_VERSION, 1, 31), + ("VERIFIED", MST2_VERIFICATION_VERSION, 1, 33), + ] + .into_iter() + .enumerate() + { + let bad_oid = format!("{index:040x}"); + mst2_verified_object::Entity::insert(row( + bad_oid.clone(), + state, + version, + size, + vec![8; digest_len], + )) + .exec(mono.get_connection()) + .await + .unwrap(); + assert!(matches!( + mono.get_verified_blobs(vec![valid_oid.clone(), bad_oid]) + .await, + Err(MegaError::ObjStorageInconsistent(_)) + )); + assert_eq!( + mono.get_verified_blobs(vec![valid_oid.clone()]) + .await + .unwrap() + .len(), + 1 + ); + } + } + + #[tokio::test] + async fn mst2_verified_blob_accepts_the_eight_tib_boundary() { + let temp = tempfile::TempDir::new().unwrap(); + let storage = test_storage(temp.path()).await; + let mono = storage.mono_storage(); + let oid = "e".repeat(40); + mono.insert_verified_blobs(vec![mst2_verified_object::ActiveModel { + id: sea_orm::ActiveValue::NotSet, + storage_domain: Set("git".to_string()), + git_oid: Set(oid.clone()), + object_kind: Set("blob".to_string()), + raw_sha256: Set(vec![9; 32]), + size: Set(8_796_093_022_208), + verification_version: Set(MST2_VERIFICATION_VERSION), + state: Set("VERIFIED".to_string()), + created_at: Set(chrono::Utc::now().fixed_offset()), + }]) + .await + .unwrap(); + let rows = mono.get_verified_blobs(vec![oid.clone()]).await.unwrap(); + assert_eq!(rows[&oid].size, 8_796_093_022_208); + } + + #[tokio::test] + async fn mst2_verified_blob_upgrade_only_replaces_valid_legacy_facts() { + let temp = tempfile::TempDir::new().unwrap(); + let storage = test_storage(temp.path()).await; + let mono = storage.mono_storage(); + for (index, (version, state, size, digest_len)) in [ + (1, "VERIFIED", 3, 32), + (1, "PENDING", 3, 32), + (1, "VERIFIED", -1, 32), + (1, "VERIFIED", MST2_MAX_FILE_SIZE + 1, 32), + (1, "VERIFIED", 3, 31), + (MST2_VERIFICATION_VERSION, "VERIFIED", 3, 32), + (3, "VERIFIED", 3, 32), + ] + .into_iter() + .enumerate() + { + let oid = format!("{index:040x}"); + let legacy = mst2_verified_object::ActiveModel { + id: sea_orm::ActiveValue::NotSet, + storage_domain: Set("git".to_string()), + git_oid: Set(oid.clone()), + object_kind: Set("blob".to_string()), + raw_sha256: Set(vec![7; digest_len]), + size: Set(size), + verification_version: Set(version), + state: Set(state.to_string()), + created_at: Set(chrono::Utc::now().fixed_offset()), + }; + let old = mst2_verified_object::Entity::insert(legacy) + .exec_with_returning(mono.get_connection()) + .await + .unwrap(); + if index == 0 { + assert!( + mono.get_verified_blobs(vec![oid.clone()]) + .await + .unwrap() + .is_empty() + ); + } + mono.insert_verified_blobs(vec![mst2_verified_object::ActiveModel { + id: sea_orm::ActiveValue::NotSet, + storage_domain: Set("git".to_string()), + git_oid: Set(oid), + object_kind: Set("blob".to_string()), + raw_sha256: Set(vec![9; 32]), + size: Set(10), + verification_version: Set(MST2_VERIFICATION_VERSION), + state: Set("VERIFIED".to_string()), + created_at: Set(chrono::Utc::now().fixed_offset()), + }]) + .await + .unwrap(); + let saved = mst2_verified_object::Entity::find_by_id(old.id) + .one(mono.get_connection()) + .await + .unwrap() + .unwrap(); + if index == 0 { + assert_eq!(saved.verification_version, MST2_VERIFICATION_VERSION); + assert_eq!(saved.size, 10); + assert_eq!(saved.raw_sha256, vec![9; 32]); + } else { + assert_eq!( + saved, old, + "invalid, unknown and current rows are never replaced" + ); + } + } + } } diff --git a/tests/integration_api_write_trunk.rs b/tests/integration_api_write_trunk.rs index 938f5a6e..569340ce 100644 --- a/tests/integration_api_write_trunk.rs +++ b/tests/integration_api_write_trunk.rs @@ -18,7 +18,7 @@ mod git_cli; use std::{ fs, io::Read, - net::{TcpListener, TcpStream}, + net::TcpStream, path::{Path, PathBuf}, process::{Child, Command, ExitStatus, Stdio}, sync::atomic::{AtomicUsize, Ordering}, @@ -235,6 +235,7 @@ struct ServiceProcess { impl ServiceProcess { fn spawn(mut command: Command) -> Self { let child = command.spawn().expect("spawn mega2 service"); + git_cli::record_service_pid(child.id()); Self { child, reaped: false, @@ -711,11 +712,9 @@ fn with_runtime(future: impl std::future::Future) -> T { } fn reserve_free_port() -> u16 { - TcpListener::bind("127.0.0.1:0") - .expect("bind ephemeral port") - .local_addr() - .expect("local addr") - .port() + let port = git_cli::reserve_ephemeral_port(); + git_cli::record_allocated_port(port); + port } fn create_log_file(path: &Path) -> fs::File {