diff --git a/yarn-project/end-to-end/bootstrap.sh b/yarn-project/end-to-end/bootstrap.sh index f89af5c66285..a53083745132 100755 --- a/yarn-project/end-to-end/bootstrap.sh +++ b/yarn-project/end-to-end/bootstrap.sh @@ -33,10 +33,12 @@ function test_cmds { echo "$prefix:NAME=e2e_prover_full_fake FAKE_PROOFS=1 $run_test_script simple e2e_prover/full" fi echo "$prefix:TIMEOUT=15m:NAME=e2e_block_building $(set_dump_avm e2e_block_building) $run_test_script simple e2e_block_building" + echo "$prefix:TIMEOUT=15m:NAME=e2e_epochs/epochs_long_proving_time $run_test_script simple src/e2e_epochs/epochs_long_proving_time.test.ts" local tests=( # List all standalone and nested tests, except for the ones listed above. - src/e2e_!(prover)/*.test.ts + src/e2e_!(prover|epochs)/*.test.ts + src/e2e_epochs/!(epochs_long_proving_time).test.ts src/e2e_p2p/reqresp/*.test.ts src/e2e_!(block_building).test.ts ) diff --git a/yarn-project/prover-client/src/mocks/test_context.ts b/yarn-project/prover-client/src/mocks/test_context.ts index 750237c6d1b5..b06fe3322d85 100644 --- a/yarn-project/prover-client/src/mocks/test_context.ts +++ b/yarn-project/prover-client/src/mocks/test_context.ts @@ -159,6 +159,29 @@ export class TestContext { this.epochNumber++; } + /** Removes the last checkpoint from the test context, rolling back state changes from makeCheckpoint. */ + public async removeLastCheckpoint() { + const removed = this.checkpoints.pop(); + this.checkpointOutHashes.pop(); + if (!removed) { + return; + } + + const numBlocks = removed.blocks.length; + this.nextCheckpointIndex--; + this.nextCheckpointNumber = CheckpointNumber(Number(this.nextCheckpointNumber) - 1); + this.nextBlockNumber -= numBlocks; + + // Remove block headers. + for (const block of removed.blocks) { + this.headers.delete(block.number); + } + + // Unwind world state to before the removed checkpoint's blocks. + const firstBlockNumber = removed.blocks[0].number; + await this.worldState.unwindBlocks(BlockNumber(firstBlockNumber - 1)); + } + // Return blob fields of all checkpoints in the epoch. public getBlobFields() { return this.checkpoints.map(checkpoint => checkpoint.toBlobFields()); diff --git a/yarn-project/prover-client/src/orchestrator/checkpoint-proving-state.ts b/yarn-project/prover-client/src/orchestrator/checkpoint-proving-state.ts index cebf6465e257..a204fd5046b0 100644 --- a/yarn-project/prover-client/src/orchestrator/checkpoint-proving-state.ts +++ b/yarn-project/prover-client/src/orchestrator/checkpoint-proving-state.ts @@ -68,7 +68,7 @@ export class CheckpointProvingState { public readonly index: number, public readonly constants: CheckpointConstantData, public readonly totalNumBlocks: number, - private readonly finalBlobBatchingChallenges: FinalBlobBatchingChallenges, + private finalBlobBatchingChallenges: FinalBlobBatchingChallenges | undefined, private readonly headerOfLastBlockInPreviousCheckpoint: BlockHeader, private readonly lastArchiveSiblingPath: Tuple, private readonly l1ToL2Messages: Fr[], @@ -91,6 +91,22 @@ export class CheckpointProvingState { this.firstBlockNumber = BlockNumber(headerOfLastBlockInPreviousCheckpoint.globalVariables.blockNumber + 1); } + /** Sets the final blob batching challenges. Called from EpochProvingState.finalizeEpochStructure(). */ + public setFinalBlobBatchingChallenges(challenges: FinalBlobBatchingChallenges) { + this.finalBlobBatchingChallenges = challenges; + } + + /** Returns true if the block merge tree is fully resolved (all block root/merge proofs are ready). */ + /** Returns true when all block-level proofs that feed into the checkpoint root are complete. */ + public isBlockMergeTreeComplete(): boolean { + if (this.isAcceptingBlocks()) { + return false; + } + // Use the same check as isReadyForCheckpointRoot for the proof tree, + // but without requiring blob/out-hash data (which comes from epoch finalization). + return this.#getChildProofsForRoot().every(p => !!p); + } + public get epochNumber(): number { return this.parentEpoch.epochNumber; } @@ -283,6 +299,9 @@ export class CheckpointProvingState { if (!this.startBlobAccumulator) { throw new Error('Start blob accumulator is not set.'); } + if (!this.finalBlobBatchingChallenges) { + throw new Error('Final blob batching challenges are not set. Call finalizeEpochStructure first.'); + } // `blobFields` must've been set if `startBlobAccumulator` is set (in `accumulateBlobs`). const blobFields = this.blobFields!; diff --git a/yarn-project/prover-client/src/orchestrator/epoch-proving-state.ts b/yarn-project/prover-client/src/orchestrator/epoch-proving-state.ts index a551082873c6..7e67e1dd4289 100644 --- a/yarn-project/prover-client/src/orchestrator/epoch-proving-state.ts +++ b/yarn-project/prover-client/src/orchestrator/epoch-proving-state.ts @@ -8,6 +8,7 @@ import { } from '@aztec/constants'; import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types'; import { Fr } from '@aztec/foundation/curves/bn254'; +import { createLogger } from '@aztec/foundation/log'; import type { Tuple } from '@aztec/foundation/serialize'; import { MerkleTreeCalculator, @@ -51,19 +52,27 @@ export type ProvingResult = { status: 'success' } | { status: 'failure'; reason: * Captures resolve and reject callbacks to provide a promise base interface to the consumer of our proving. */ export class EpochProvingState { - private checkpointProofs: UnbalancedTreeStore< - ProofState - >; + private checkpointProofs: + | UnbalancedTreeStore> + | undefined; private checkpointPaddingProof: | ProofState | undefined; private rootRollupProof: ProofState | undefined; private checkpoints: (CheckpointProvingState | undefined)[] = []; - private startBlobAccumulator: BatchedBlobAccumulator; + private startBlobAccumulator: BatchedBlobAccumulator | undefined; private endBlobAccumulator: BatchedBlobAccumulator | undefined; private finalBatchedBlob: BatchedBlob | undefined; private provingStateLifecycle = PROVING_STATE_LIFECYCLE.PROVING_STATE_CREATED; + /** Set after `finalizeEpochStructure` is called. */ + private _totalNumCheckpoints: number | undefined; + /** Set after `finalizeEpochStructure` is called. */ + private _finalBlobBatchingChallenges: FinalBlobBatchingChallenges | undefined; + + /** All callbacks waiting for checkpoints to be block-level ready. */ + private checkpointsReadyCallbacks: Array<{ resolve: () => void; reject: (reason: string) => void }> = []; + // Map from tx hash to chonk verifier proof promise. Used when kickstarting chonk verifier proofs before tx processing. public readonly cachedChonkVerifierProofs = new Map< string, @@ -72,20 +81,150 @@ export class EpochProvingState { > >(); + private log = createLogger('prover-client:epoch-proving-state'); + constructor( public readonly epochNumber: EpochNumber, - public readonly totalNumCheckpoints: number, - private readonly finalBlobBatchingChallenges: FinalBlobBatchingChallenges, private onCheckpointBlobAccumulatorSet: (checkpoint: CheckpointProvingState) => Promise, private completionCallback: (result: ProvingResult) => void, private rejectionCallback: (reason: string) => void, + ) {} + + /** Returns the total number of checkpoints, or undefined if not yet finalized. */ + public get totalNumCheckpoints(): number | undefined { + return this._totalNumCheckpoints; + } + + /** Returns whether the epoch structure has been finalized. */ + public get isEpochStructureFinalized(): boolean { + return this.totalNumCheckpoints != undefined; + } + + /** + * Finalizes the epoch structure after all checkpoints have been added. + * Sets the final checkpoint count and blob batching challenges, creates the + * UnbalancedTreeStore for checkpoint merges, and triggers checkpoint root + * enqueue for any checkpoints whose block merge proofs are already complete. + */ + public async finalizeEpochStructure( + totalNumCheckpoints: number, + finalBlobBatchingChallenges: FinalBlobBatchingChallenges, ) { + if (this.isEpochStructureFinalized) { + throw new Error('Epoch structure has already been finalized.'); + } + + this._totalNumCheckpoints = totalNumCheckpoints; + this._finalBlobBatchingChallenges = finalBlobBatchingChallenges; this.checkpointProofs = new UnbalancedTreeStore(totalNumCheckpoints); this.startBlobAccumulator = BatchedBlobAccumulator.newWithChallenges(finalBlobBatchingChallenges); + + // Transition to FULL if all checkpoints are added. + if (this.checkpoints.filter(c => !!c).length === totalNumCheckpoints) { + this.provingStateLifecycle = PROVING_STATE_LIFECYCLE.PROVING_STATE_FULL; + } + + // Set blob batching challenges on all existing checkpoints. + for (const checkpoint of this.checkpoints) { + if (checkpoint) { + checkpoint.setFinalBlobBatchingChallenges(finalBlobBatchingChallenges); + } + } + + // Accumulate out hashes and blob data now that structure is known. + await this.accumulateCheckpointOutHashes(); + await this.setBlobAccumulators(); + + // For any checkpoints whose block merge proofs are already complete, trigger checkpoint root enqueue. + for (const checkpoint of this.checkpoints) { + if (checkpoint && checkpoint.isReadyForCheckpointRoot()) { + await this.onCheckpointBlobAccumulatorSet(checkpoint); + } + } + } + + /** + * Removes the last checkpoint from the epoch. Only valid before `finalizeEpochStructure` has been called. + * Returns the removed checkpoint, or undefined if there are no checkpoints. + */ + public removeLastCheckpoint(): CheckpointProvingState | undefined { + if (this.isEpochStructureFinalized) { + throw new Error('Cannot remove checkpoints after epoch structure has been finalized.'); + } + + // Find the last non-undefined checkpoint. + let lastIndex = -1; + for (let i = this.checkpoints.length - 1; i >= 0; i--) { + if (this.checkpoints[i]) { + lastIndex = i; + break; + } + } + + if (lastIndex === -1) { + return undefined; + } + + const removed = this.checkpoints[lastIndex]!; + this.checkpoints[lastIndex] = undefined; + // Trim trailing undefined entries. + while (this.checkpoints.length > 0 && !this.checkpoints[this.checkpoints.length - 1]) { + this.checkpoints.pop(); + } + + // Re-evaluate: removing a checkpoint may mean all remaining are now ready. + this.notifyCheckpointBlockLevelComplete(); + + return removed; } - // Adds a block to the proving state, returns its index - // Will update the proving life cycle if this is the last block + /** + * Returns a promise that resolves when all current checkpoints have completed block-level proving. + * Block-level proving complete means the checkpoint's block merge tree is fully resolved. + */ + public waitForAllCheckpointsReady(): Promise { + if (!this.verifyState()) { + return Promise.reject(new Error('Epoch proving state is no longer valid')); + } + if (this.areAllCheckpointsBlockLevelReady()) { + this.log.debug(`All checkpoints already block-level ready`); + return Promise.resolve(); + } + this.log.debug(`Waiting for all checkpoints to complete block-level proving`); + return new Promise((resolve, reject) => { + this.checkpointsReadyCallbacks.push({ resolve, reject }); + }); + } + + /** Called when a checkpoint completes block-level proving. Re-evaluates readiness and notifies waiters. */ + public notifyCheckpointBlockLevelComplete() { + if (this.areAllCheckpointsBlockLevelReady()) { + this.log.info(`All checkpoints block-level ready, notifying ${this.checkpointsReadyCallbacks.length} waiters`); + for (const { resolve } of this.checkpointsReadyCallbacks) { + resolve(); + } + this.checkpointsReadyCallbacks = []; + } + } + + /** Rejects all checkpoint-ready waiters. Called when the epoch is cancelled. */ + private rejectCheckpointsReadyWaiters(reason: string) { + for (const { reject } of this.checkpointsReadyCallbacks) { + reject(reason); + } + this.checkpointsReadyCallbacks = []; + } + + private areAllCheckpointsBlockLevelReady(): boolean { + const activeCheckpoints = this.checkpoints.filter(c => !!c); + if (activeCheckpoints.length === 0) { + return false; + } + return activeCheckpoints.every(c => c!.isBlockMergeTreeComplete()); + } + + // Adds a checkpoint to the proving state. + // Will update the proving life cycle if this is the last checkpoint (only when epoch structure is finalized). public startNewCheckpoint( checkpointIndex: number, constants: CheckpointConstantData, @@ -98,9 +237,9 @@ export class EpochProvingState { newL1ToL2MessageTreeSnapshot: AppendOnlyTreeSnapshot, newL1ToL2MessageSubtreeRootSiblingPath: Tuple, ): CheckpointProvingState { - if (checkpointIndex >= this.totalNumCheckpoints) { + if (this._totalNumCheckpoints !== undefined && checkpointIndex >= this._totalNumCheckpoints) { throw new Error( - `Unable to start a new checkpoint at index ${checkpointIndex}. Expected at most ${this.totalNumCheckpoints} checkpoints.`, + `Unable to start a new checkpoint at index ${checkpointIndex}. Expected at most ${this._totalNumCheckpoints} checkpoints.`, ); } @@ -108,7 +247,7 @@ export class EpochProvingState { checkpointIndex, constants, totalNumBlocks, - this.finalBlobBatchingChallenges, + this._finalBlobBatchingChallenges, previousBlockHeader, lastArchiveSiblingPath, l1ToL2Messages, @@ -121,7 +260,10 @@ export class EpochProvingState { ); this.checkpoints[checkpointIndex] = checkpoint; - if (this.checkpoints.filter(c => !!c).length === this.totalNumCheckpoints) { + if ( + this._totalNumCheckpoints !== undefined && + this.checkpoints.filter(c => !!c).length === this._totalNumCheckpoints + ) { this.provingStateLifecycle = PROVING_STATE_LIFECYCLE.PROVING_STATE_FULL; } @@ -155,7 +297,11 @@ export class EpochProvingState { // Returns true if we are still able to accept checkpoints, false otherwise. public isAcceptingCheckpoints() { - return this.checkpoints.filter(c => !!c).length < this.totalNumCheckpoints; + // Before finalization, always accept checkpoints. + if (this._totalNumCheckpoints === undefined) { + return true; + } + return this.checkpoints.filter(c => !!c).length < this._totalNumCheckpoints; } public setCheckpointRootRollupProof( @@ -165,10 +311,16 @@ export class EpochProvingState { typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH >, ): TreeNodeLocation { + if (!this.checkpointProofs) { + throw new Error('Checkpoint proofs store not initialized. Call finalizeEpochStructure first.'); + } return this.checkpointProofs.setLeaf(checkpointIndex, { provingOutput }); } public tryStartProvingCheckpointMerge(location: TreeNodeLocation) { + if (!this.checkpointProofs) { + throw new Error('Checkpoint proofs store not initialized. Call finalizeEpochStructure first.'); + } if (this.checkpointProofs.getNode(location)?.isProving) { return false; } else { @@ -184,6 +336,9 @@ export class EpochProvingState { typeof NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH >, ) { + if (!this.checkpointProofs) { + throw new Error('Checkpoint proofs store not initialized. Call finalizeEpochStructure first.'); + } this.checkpointProofs.setNode(location, { provingOutput }); } @@ -219,6 +374,10 @@ export class EpochProvingState { } public async accumulateCheckpointOutHashes() { + if (this._totalNumCheckpoints === undefined) { + return; + } + const treeCalculator = await MerkleTreeCalculator.create(OUT_HASH_TREE_HEIGHT, undefined, (left, right) => Promise.resolve(shaMerkleHash(left, right)), ); @@ -237,11 +396,9 @@ export class EpochProvingState { let hint = this.checkpoints[0]?.getOutHashHint(); const outHashes = []; - for (let i = 0; i < this.totalNumCheckpoints; i++) { - const checkpoint = this.checkpoints[i]; - if (!checkpoint) { - break; - } + const activeCheckpoints = this.checkpoints.filter(c => !!c) as CheckpointProvingState[]; + for (let i = 0; i < activeCheckpoints.length; i++) { + const checkpoint = activeCheckpoints[i]; // If hints are not set yet, it must be the first checkpoint. Compute the hints with an empty tree. hint ??= await computeOutHashHint([]); @@ -255,7 +412,7 @@ export class EpochProvingState { outHashes.push(outHash); // If this is NOT the last checkpoint, get or create the hint for the next checkpoint. - if (i !== this.totalNumCheckpoints - 1) { + if (i !== activeCheckpoints.length - 1) { hint = checkpoint.getOutHashHintForNextCheckpoint() ?? (await computeOutHashHint(outHashes)); checkpoint.setOutHashHintForNextCheckpoint(hint); } @@ -263,13 +420,15 @@ export class EpochProvingState { } public async setBlobAccumulators() { + if (!this.startBlobAccumulator || this._totalNumCheckpoints === undefined) { + return; + } + let previousAccumulator = this.startBlobAccumulator; // Accumulate blobs as far as we can for this epoch. - for (let i = 0; i < this.totalNumCheckpoints; i++) { - const checkpoint = this.checkpoints[i]; - if (!checkpoint) { - break; - } + const activeCheckpoints = this.checkpoints.filter(c => !!c) as CheckpointProvingState[]; + for (let i = 0; i < activeCheckpoints.length; i++) { + const checkpoint = activeCheckpoints[i]; const endAccumulator = checkpoint.getEndBlobAccumulator() || (await checkpoint.accumulateBlobs(previousAccumulator)); @@ -280,7 +439,7 @@ export class EpochProvingState { previousAccumulator = endAccumulator; // If this is the last checkpoint, set the end blob accumulator. - if (i === this.totalNumCheckpoints - 1) { + if (i === activeCheckpoints.length - 1) { this.endBlobAccumulator = endAccumulator; } } @@ -294,10 +453,16 @@ export class EpochProvingState { } public getParentLocation(location: TreeNodeLocation) { + if (!this.checkpointProofs) { + throw new Error('Checkpoint proofs store not initialized. Call finalizeEpochStructure first.'); + } return this.checkpointProofs.getParentLocation(location); } public getCheckpointMergeRollupInputs(mergeLocation: TreeNodeLocation) { + if (!this.checkpointProofs) { + throw new Error('Checkpoint proofs store not initialized. Call finalizeEpochStructure first.'); + } const [left, right] = this.checkpointProofs.getChildren(mergeLocation).map(c => c?.provingOutput); if (!left || !right) { throw new Error('At least one child is not ready for the checkpoint merge rollup.'); @@ -336,11 +501,17 @@ export class EpochProvingState { } public isReadyForCheckpointMerge(location: TreeNodeLocation) { + if (!this.checkpointProofs) { + return false; + } return !!this.checkpointProofs.getSibling(location)?.provingOutput; } // Returns true if we have sufficient inputs to execute the block root rollup public isReadyForRootRollup() { + if (!this.checkpointProofs || !this.isEpochStructureFinalized) { + return false; + } const childProofs = this.#getChildProofsForRoot(); return childProofs.every(p => !!p); } @@ -357,6 +528,7 @@ export class EpochProvingState { return; } this.provingStateLifecycle = PROVING_STATE_LIFECYCLE.PROVING_STATE_REJECTED; + this.rejectCheckpointsReadyWaiters(reason); this.rejectionCallback(reason); } @@ -371,9 +543,12 @@ export class EpochProvingState { } #getChildProofsForRoot() { + if (!this.checkpointProofs || this._totalNumCheckpoints === undefined) { + return [undefined, undefined]; + } const rootLocation = { level: 0, index: 0 }; - // If there's only 1 block, its block root proof will be stored at the root. - return this.totalNumCheckpoints === 1 + // If there's only 1 checkpoint, its checkpoint root proof will be stored at the root. + return this._totalNumCheckpoints === 1 ? [this.checkpointProofs.getNode(rootLocation)?.provingOutput, this.checkpointPaddingProof?.provingOutput] : this.checkpointProofs.getChildren(rootLocation).map(c => c?.provingOutput); } diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator.ts b/yarn-project/prover-client/src/orchestrator/orchestrator.ts index 1dd893fe6af6..cbf606a6c7d6 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator.ts @@ -124,6 +124,11 @@ export class ProvingOrchestrator implements EpochProver { return this.dbs.size; } + /** Returns the number of proving jobs that are still in-flight. */ + public getNumPendingProvingJobs() { + return this.pendingProvingJobs.length; + } + public async stop(): Promise { // Grab the old queue before cancel() replaces it, so we can await its draining. const oldQueue = this.deferredJobQueue; @@ -131,11 +136,7 @@ export class ProvingOrchestrator implements EpochProver { await oldQueue.cancel(); } - public startNewEpoch( - epochNumber: EpochNumber, - totalNumCheckpoints: number, - finalBlobBatchingChallenges: FinalBlobBatchingChallenges, - ) { + public startNewEpoch(epochNumber: EpochNumber) { if (this.provingState?.verifyState()) { throw new Error( `Cannot start epoch ${epochNumber} when epoch ${this.provingState.epochNumber} is still being processed.`, @@ -144,11 +145,9 @@ export class ProvingOrchestrator implements EpochProver { const { promise: _promise, resolve, reject } = promiseWithResolvers(); const promise = _promise.catch((reason): ProvingResult => ({ status: 'failure', reason })); - this.logger.info(`Starting epoch ${epochNumber} with ${totalNumCheckpoints} checkpoints.`); + this.logger.info(`Starting epoch ${epochNumber}.`); this.provingState = new EpochProvingState( epochNumber, - totalNumCheckpoints, - finalBlobBatchingChallenges, provingState => this.checkAndEnqueueCheckpointRootRollup(provingState), resolve, reject, @@ -156,6 +155,80 @@ export class ProvingOrchestrator implements EpochProver { this.provingPromise = promise; } + /** + * Finalizes the epoch structure after all checkpoints have been processed. + * Sets the final checkpoint count and blob batching challenges, creates the merge tree + * for checkpoints, and triggers checkpoint root enqueue for any checkpoints already ready. + */ + public async finalizeEpochStructure( + totalNumCheckpoints: number, + finalBlobBatchingChallenges: FinalBlobBatchingChallenges, + ) { + if (!this.provingState) { + throw new Error('Empty epoch proving state. Call startNewEpoch before finalizing epoch structure.'); + } + + this.logger.info( + `Finalizing epoch ${this.provingState.epochNumber} structure with ${totalNumCheckpoints} checkpoints.`, + ); + await this.provingState.finalizeEpochStructure(totalNumCheckpoints, finalBlobBatchingChallenges); + } + + /** + * Removes the last checkpoint from the epoch. Only valid before `finalizeEpochStructure` has been called. + * Closes world state forks and cleans up cached chonk verifier proofs for the removed checkpoint. + * + * In-flight proving jobs for the removed checkpoint may continue to run and their callbacks will fire, + * but they are safe to ignore: + * - checkAndEnqueueCheckpointRootRollup gates on isEpochStructureFinalized (false before finalization) + * and isReadyForCheckpointRoot (requires blob data which is never set on removed checkpoints) + * - notifyCheckpointBlockLevelComplete re-evaluates using the epoch's checkpoints array, which no longer + * includes the removed checkpoint + * - Block merge jobs may be enqueued wastefully but will hit the same safe gates + */ + public removeLastCheckpoint() { + if (!this.provingState) { + throw new Error('Empty epoch proving state. Call startNewEpoch before removing checkpoints.'); + } + + const removed = this.provingState.removeLastCheckpoint(); + if (!removed) { + this.logger.warn('No checkpoint to remove.'); + return; + } + + this.logger.info(`Removed checkpoint ${removed.index} from epoch ${this.provingState.epochNumber}.`); + + // Close world state forks for all blocks in the removed checkpoint. + for (let i = 0; i < removed.totalNumBlocks; i++) { + const blockNumber = BlockNumber(Number(removed.firstBlockNumber) + i); + const db = this.dbs.get(blockNumber); + if (db) { + void db.close().catch(err => this.logger.error(`Error closing db for block ${blockNumber}`, err)); + this.dbs.delete(blockNumber); + } + } + + // Clean up cached chonk verifier proofs for txs in the removed checkpoint. + for (let i = 0; i < removed.totalNumBlocks; i++) { + const blockNumber = BlockNumber(Number(removed.firstBlockNumber) + i); + const block = removed.getBlockProvingStateByBlockNumber(blockNumber); + if (block) { + for (const tx of block.getProcessedTxs()) { + this.provingState.cachedChonkVerifierProofs.delete(tx.hash.toString()); + } + } + } + } + + /** Returns a promise that resolves when all current checkpoints have completed block-level proving. */ + public waitForAllCheckpointsReady(): Promise { + if (!this.provingState) { + throw new Error('Empty epoch proving state. Call startNewEpoch before waiting for checkpoints.'); + } + return this.provingState.waitForAllCheckpointsReady(); + } + /** * Starts a new checkpoint. * @param checkpointIndex - The index of the checkpoint in the epoch. @@ -1214,6 +1287,16 @@ export class ProvingOrchestrator implements EpochProver { } private async checkAndEnqueueCheckpointRootRollup(provingState: CheckpointProvingState) { + // Notify the epoch that this checkpoint's block-level proving may be complete. + if (provingState.isBlockMergeTreeComplete()) { + this.provingState?.notifyCheckpointBlockLevelComplete(); + } + + // Two-input gate: only enqueue if BOTH block merge proofs are ready AND epoch structure is finalized. + if (!this.provingState?.isEpochStructureFinalized) { + return; + } + if (!provingState.isReadyForCheckpointRoot()) { return; } diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_deferred_finalization.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_deferred_finalization.test.ts new file mode 100644 index 000000000000..0cb377dd4969 --- /dev/null +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_deferred_finalization.test.ts @@ -0,0 +1,929 @@ +import { FinalBlobBatchingChallenges } from '@aztec/blob-lib/types'; +import { MAX_CHECKPOINTS_PER_EPOCH } from '@aztec/constants'; +import { EpochNumber } from '@aztec/foundation/branded-types'; +import { padArrayEnd, timesAsync } from '@aztec/foundation/collection'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { createLogger } from '@aztec/foundation/log'; +import { promiseWithResolvers } from '@aztec/foundation/promise'; +import { sleep } from '@aztec/foundation/sleep'; + +import { TestContext } from '../mocks/test_context.js'; +import type { CheckpointProvingState } from './checkpoint-proving-state.js'; +import { EpochProvingState, type ProvingResult } from './epoch-proving-state.js'; + +const logger = createLogger('prover-client:test:orchestrator-deferred-finalization'); + +const LONG_TIMEOUT = 600_000; + +/** + * Helper to create a minimal EpochProvingState for unit testing. + * Provides direct access to the state without going through the orchestrator. + */ +function createTestEpochState(onCheckpointReady?: (checkpoint: CheckpointProvingState) => Promise): { + state: EpochProvingState; + completionPromise: Promise; +} { + const { promise, resolve, reject } = promiseWithResolvers(); + const completionPromise = promise.catch((reason): ProvingResult => ({ status: 'failure', reason })); + + const state = new EpochProvingState(EpochNumber(1), onCheckpointReady ?? (() => Promise.resolve()), resolve, reject); + + return { state, completionPromise }; +} + +// ============================================================================ +// Unit tests for EpochProvingState deferred finalization mechanics +// ============================================================================ +describe('prover/orchestrator/deferred-finalization', () => { + describe('EpochProvingState unit tests', () => { + describe('startNewEpoch and isAcceptingCheckpoints', () => { + it('accepts checkpoints before finalization', () => { + const { state } = createTestEpochState(); + expect(state.isAcceptingCheckpoints()).toBe(true); + }); + + it('totalNumCheckpoints is undefined before finalization', () => { + const { state } = createTestEpochState(); + expect(state.totalNumCheckpoints).toBeUndefined(); + }); + + it('isEpochStructureFinalized is false before finalization', () => { + const { state } = createTestEpochState(); + expect(state.isEpochStructureFinalized).toBe(false); + }); + }); + + describe('finalizeEpochStructure', () => { + it('sets totalNumCheckpoints and marks structure as finalized', async () => { + const { state } = createTestEpochState(); + const challenges = FinalBlobBatchingChallenges.empty(); + + await state.finalizeEpochStructure(3, challenges); + + expect(state.totalNumCheckpoints).toBe(3); + expect(state.isEpochStructureFinalized).toBe(true); + }); + + it('throws when called twice', async () => { + const { state } = createTestEpochState(); + const challenges = FinalBlobBatchingChallenges.empty(); + + await state.finalizeEpochStructure(2, challenges); + + await expect(state.finalizeEpochStructure(2, challenges)).rejects.toThrow( + 'Epoch structure has already been finalized.', + ); + }); + + it('transitions to FULL when all checkpoints are already added', async () => { + const { state } = createTestEpochState(); + const challenges = FinalBlobBatchingChallenges.empty(); + + // We cannot easily add a real checkpoint in unit tests without all the tree data, + // but we can verify the state is CREATED before finalization and still valid. + expect(state.verifyState()).toBe(true); + + // Finalizing with 0 checkpoints is a bit unusual, but the count matching empty array + // should be fine. Let's use totalNumCheckpoints=0 to force it to FULL. + // Actually with 0 active checkpoints filter(c => !!c).length === 0 which equals totalNumCheckpoints (0). + // This should transition to FULL. + await state.finalizeEpochStructure(0, challenges); + expect(state.verifyState()).toBe(true); + }); + + it('handles finalization when no checkpoints have been added', async () => { + const { state } = createTestEpochState(); + const challenges = FinalBlobBatchingChallenges.empty(); + + // Finalize with 2 expected checkpoints when none are added yet. + // The epoch is not FULL since 0 < 2. + await state.finalizeEpochStructure(2, challenges); + expect(state.isEpochStructureFinalized).toBe(true); + expect(state.isAcceptingCheckpoints()).toBe(true); + expect(state.verifyState()).toBe(true); + }); + + it('triggers checkpoint root enqueue for checkpoints with block merge proofs ready', async () => { + const triggeredCheckpoints: CheckpointProvingState[] = []; + const { state } = createTestEpochState(checkpoint => { + triggeredCheckpoints.push(checkpoint); + return Promise.resolve(); + }); + const challenges = FinalBlobBatchingChallenges.empty(); + + // No checkpoints added, so no triggers expected. + await state.finalizeEpochStructure(0, challenges); + expect(triggeredCheckpoints).toHaveLength(0); + }); + }); + + describe('removeLastCheckpoint', () => { + it('returns undefined on empty epoch', () => { + const { state } = createTestEpochState(); + const removed = state.removeLastCheckpoint(); + expect(removed).toBeUndefined(); + }); + + it('throws if epoch structure has been finalized', async () => { + const { state } = createTestEpochState(); + const challenges = FinalBlobBatchingChallenges.empty(); + await state.finalizeEpochStructure(0, challenges); + + expect(() => state.removeLastCheckpoint()).toThrow( + 'Cannot remove checkpoints after epoch structure has been finalized.', + ); + }); + + it('isAcceptingCheckpoints still returns true after removal', () => { + const { state } = createTestEpochState(); + // Even on empty epoch, removing returns undefined but accepting should remain true. + state.removeLastCheckpoint(); + expect(state.isAcceptingCheckpoints()).toBe(true); + }); + }); + + describe('checkpoints-ready promise', () => { + it('does not resolve when no checkpoints are present', async () => { + const { state } = createTestEpochState(); + + let resolved = false; + void state.waitForAllCheckpointsReady().then(() => { + resolved = true; + }); + + await sleep(50); + expect(resolved).toBe(false); + }); + }); + + describe('cancellation', () => { + it('cancel discards all state properly', () => { + const { state } = createTestEpochState(); + state.cancel(); + // After cancellation, verifyState should return false. + expect(state.verifyState()).toBe(false); + }); + + it('reject after cancel is a no-op', () => { + const { state } = createTestEpochState(); + state.cancel(); + // Should not throw, just be ignored. + state.reject('some reason'); + expect(state.verifyState()).toBe(false); + }); + }); + }); + + // ============================================================================ + // Integration tests through the orchestrator + // ============================================================================ + describe('orchestrator integration tests', () => { + let context: TestContext; + + beforeEach(async () => { + context = await TestContext.new(logger); + }); + + afterEach(async () => { + await context.cleanup(); + }); + + describe('happy path', () => { + it( + 'single-checkpoint epoch: start epoch -> add checkpoint -> process txs -> finalize -> epoch completes', + async () => { + const numCheckpoints = 1; + const numBlocks = 1; + const numTxsPerBlock = 1; + + const { constants, blocks, previousBlockHeader, header } = await context.makeCheckpoint(numBlocks, { + numTxsPerBlock, + }); + const finalBlobChallenges = await context.getFinalBlobChallenges(); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + + await context.orchestrator.startNewCheckpoint(0, constants, [], numBlocks, previousBlockHeader); + + for (const block of blocks) { + const { blockNumber, timestamp } = block.header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, block.txs.length); + await context.orchestrator.addTxs(block.txs); + await context.orchestrator.setBlockCompleted(blockNumber, block.header); + } + + await context.orchestrator.finalizeEpochStructure(numCheckpoints, finalBlobChallenges); + const epoch = await context.orchestrator.finalizeEpoch(); + expect(epoch.proof).toBeDefined(); + expect(epoch.publicInputs.checkpointHeaderHashes).toEqual( + padArrayEnd([header.hash()], Fr.ZERO, MAX_CHECKPOINTS_PER_EPOCH), + ); + }, + LONG_TIMEOUT, + ); + + it( + 'multi-checkpoint epoch: checkpoints added incrementally -> finalize -> epoch completes', + async () => { + const numCheckpoints = 3; + const numBlocksPerCheckpoint = 1; + const numTxsPerBlock = 1; + + const checkpoints = await timesAsync(numCheckpoints, () => + context.makeCheckpoint(numBlocksPerCheckpoint, { numTxsPerBlock }), + ); + const finalBlobChallenges = await context.getFinalBlobChallenges(); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + + for (let i = 0; i < checkpoints.length; i++) { + const { + constants, + blocks: [block], + previousBlockHeader, + } = checkpoints[i]; + + await context.orchestrator.startNewCheckpoint( + i, + constants, + [], + numBlocksPerCheckpoint, + previousBlockHeader, + ); + + const { blockNumber, timestamp } = block.header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, block.txs.length); + await context.orchestrator.addTxs(block.txs); + await context.orchestrator.setBlockCompleted(blockNumber, block.header); + } + + await context.orchestrator.finalizeEpochStructure(numCheckpoints, finalBlobChallenges); + const epoch = await context.orchestrator.finalizeEpoch(); + expect(epoch.proof).toBeDefined(); + + const headerHashes = checkpoints.map(c => c.header.hash()); + expect(epoch.publicInputs.checkpointHeaderHashes).toEqual( + padArrayEnd(headerHashes, Fr.ZERO, MAX_CHECKPOINTS_PER_EPOCH), + ); + }, + LONG_TIMEOUT, + ); + }); + + describe('deferred finalization', () => { + it('startNewEpoch succeeds with only epochNumber', () => { + // startNewEpoch takes only an EpochNumber, no totalNumCheckpoints or finalBlobBatchingChallenges. + expect(() => context.orchestrator.startNewEpoch(EpochNumber(1))).not.toThrow(); + }); + + it('checkpoints can be added incrementally after startNewEpoch', async () => { + const checkpoints = await timesAsync(2, () => context.makeCheckpoint(1, { numTxsPerBlock: 0 })); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + + // Add first checkpoint. + const { constants: c1, previousBlockHeader: h1 } = checkpoints[0]; + await context.orchestrator.startNewCheckpoint(0, c1, [], 1, h1); + + // Add second checkpoint (incrementally, no need to know total count upfront). + const { constants: c2, previousBlockHeader: h2 } = checkpoints[1]; + await expect(context.orchestrator.startNewCheckpoint(1, c2, [], 1, h2)).resolves.not.toThrow(); + }); + + it('finalizeEpochStructure throws when called twice', async () => { + const { constants, previousBlockHeader } = await context.makeCheckpoint(1, { numTxsPerBlock: 0 }); + const finalBlobChallenges = await context.getFinalBlobChallenges(); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + await context.orchestrator.startNewCheckpoint(0, constants, [], 1, previousBlockHeader); + + await context.makeCheckpoint(1, { numTxsPerBlock: 0 }); + // We need to finalize once first, which requires blocks to be set up. + // Use the simpler approach: just call finalize twice. + // But we already started a checkpoint at index 0, so we need to not confuse state. + // Let's just call finalize twice and check the second throws. + + await context.orchestrator.finalizeEpochStructure(1, finalBlobChallenges); + await expect(context.orchestrator.finalizeEpochStructure(1, finalBlobChallenges)).rejects.toThrow( + 'Epoch structure has already been finalized.', + ); + }); + + it('finalizeEpochStructure without starting epoch throws', async () => { + const challenges = FinalBlobBatchingChallenges.empty(); + await expect(context.orchestrator.finalizeEpochStructure(1, challenges)).rejects.toThrow( + 'Empty epoch proving state.', + ); + }); + }); + + describe('two-input gate', () => { + it( + 'proofs-first-then-finalize: checkpoint root is enqueued when finalize is called after block merge proofs complete', + async () => { + const numCheckpoints = 1; + const numBlocks = 1; + const numTxsPerBlock = 1; + + const { constants, blocks, previousBlockHeader } = await context.makeCheckpoint(numBlocks, { + numTxsPerBlock, + }); + const finalBlobChallenges = await context.getFinalBlobChallenges(); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + await context.orchestrator.startNewCheckpoint(0, constants, [], numBlocks, previousBlockHeader); + + // Process all blocks (block merge proofs will complete before finalize). + for (const block of blocks) { + const { blockNumber, timestamp } = block.header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, block.txs.length); + await context.orchestrator.addTxs(block.txs); + await context.orchestrator.setBlockCompleted(blockNumber, block.header); + } + + // Wait for block-level proving to complete before finalizing. + await context.orchestrator.waitForAllCheckpointsReady(); + + // Now finalize -- this should trigger checkpoint root enqueue since block merge proofs are ready. + await context.orchestrator.finalizeEpochStructure(numCheckpoints, finalBlobChallenges); + + // Epoch should complete successfully. + const epoch = await context.orchestrator.finalizeEpoch(); + expect(epoch.proof).toBeDefined(); + }, + LONG_TIMEOUT, + ); + + it( + 'finalize-first-then-proofs: checkpoint root is enqueued when block merge proofs complete after finalize', + async () => { + const numCheckpoints = 1; + const numBlocks = 1; + const numTxsPerBlock = 1; + + const { constants, blocks, previousBlockHeader } = await context.makeCheckpoint(numBlocks, { + numTxsPerBlock, + }); + const finalBlobChallenges = await context.getFinalBlobChallenges(); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + await context.orchestrator.startNewCheckpoint(0, constants, [], numBlocks, previousBlockHeader); + + // Finalize BEFORE processing blocks. + await context.orchestrator.finalizeEpochStructure(numCheckpoints, finalBlobChallenges); + + // Now process blocks -- when block merge proofs complete, checkpoint root should be enqueued + // because finalize has already been called. + for (const block of blocks) { + const { blockNumber, timestamp } = block.header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, block.txs.length); + await context.orchestrator.addTxs(block.txs); + await context.orchestrator.setBlockCompleted(blockNumber, block.header); + } + + const epoch = await context.orchestrator.finalizeEpoch(); + expect(epoch.proof).toBeDefined(); + }, + LONG_TIMEOUT, + ); + }); + + describe('waitForAllCheckpointsReady', () => { + it('throws if called before starting epoch', () => { + expect(() => context.orchestrator.waitForAllCheckpointsReady()).toThrow('Empty epoch proving state.'); + }); + + it( + 'resolves when all checkpoints complete block-level proving', + async () => { + const numCheckpoints = 2; + const numBlocksPerCheckpoint = 1; + const numTxsPerBlock = 1; + + const checkpoints = await timesAsync(numCheckpoints, () => + context.makeCheckpoint(numBlocksPerCheckpoint, { numTxsPerBlock }), + ); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + + // Add and process all checkpoints. + for (let i = 0; i < checkpoints.length; i++) { + const { + constants, + blocks: [block], + previousBlockHeader, + } = checkpoints[i]; + await context.orchestrator.startNewCheckpoint( + i, + constants, + [], + numBlocksPerCheckpoint, + previousBlockHeader, + ); + + const { blockNumber, timestamp } = block.header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, block.txs.length); + await context.orchestrator.addTxs(block.txs); + await context.orchestrator.setBlockCompleted(blockNumber, block.header); + } + + // The promise should resolve since all checkpoints have completed block-level proving. + await expect(context.orchestrator.waitForAllCheckpointsReady()).resolves.toBeUndefined(); + }, + LONG_TIMEOUT, + ); + + it( + 'does not resolve prematurely while a checkpoint is still proving', + async () => { + // Create one checkpoint but do not process its blocks. + const { constants, blocks, previousBlockHeader } = await context.makeCheckpoint(1, { numTxsPerBlock: 1 }); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + await context.orchestrator.startNewCheckpoint(0, constants, [], 1, previousBlockHeader); + + // Start the block but do NOT add txs or complete it. + const { blockNumber, timestamp } = blocks[0].header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, blocks[0].txs.length); + + let resolved = false; + void context.orchestrator.waitForAllCheckpointsReady().then( + () => { + resolved = true; + }, + () => { + // Expected: cancel() rejects the promise. + }, + ); + + // Wait a bit and verify the promise has not resolved. + await sleep(100); + expect(resolved).toBe(false); + + // Clean up by cancelling to avoid hanging test. + context.orchestrator.cancel(); + }, + LONG_TIMEOUT, + ); + + it( + 'resolves immediately if called after all checkpoints are already complete', + async () => { + const { constants, blocks, previousBlockHeader } = await context.makeCheckpoint(1, { numTxsPerBlock: 1 }); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + await context.orchestrator.startNewCheckpoint(0, constants, [], 1, previousBlockHeader); + + for (const block of blocks) { + const { blockNumber, timestamp } = block.header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, block.txs.length); + await context.orchestrator.addTxs(block.txs); + await context.orchestrator.setBlockCompleted(blockNumber, block.header); + } + + // Wait for block-level proving to be complete first. + await context.orchestrator.waitForAllCheckpointsReady(); + + // Calling again should resolve immediately (no pending work). + const startTime = Date.now(); + await context.orchestrator.waitForAllCheckpointsReady(); + const elapsed = Date.now() - startTime; + // Should resolve almost immediately (< 50ms). + expect(elapsed).toBeLessThan(200); + }, + LONG_TIMEOUT, + ); + }); + + describe('reorg safety (removeLastCheckpoint)', () => { + it('removeLastCheckpoint removes the last checkpoint', async () => { + const checkpoints = await timesAsync(2, () => context.makeCheckpoint(1, { numTxsPerBlock: 0 })); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + + // Add two checkpoints. + for (let i = 0; i < 2; i++) { + const { constants, blocks, previousBlockHeader } = checkpoints[i]; + await context.orchestrator.startNewCheckpoint(i, constants, [], 1, previousBlockHeader); + const { blockNumber, timestamp } = blocks[0].header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, 0); + } + + // Remove the last checkpoint -- should not throw. + expect(() => context.orchestrator.removeLastCheckpoint()).not.toThrow(); + }); + + it('after removal, isAcceptingCheckpoints still returns true', async () => { + const { constants, blocks, previousBlockHeader } = await context.makeCheckpoint(1, { numTxsPerBlock: 0 }); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + await context.orchestrator.startNewCheckpoint(0, constants, [], 1, previousBlockHeader); + const { blockNumber, timestamp } = blocks[0].header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, 0); + + context.orchestrator.removeLastCheckpoint(); + + // The EpochProvingState has no totalNumCheckpoints set, so it should always accept. + // We verify this indirectly by checking that we can start a new checkpoint. + // Note: We cannot re-add at the same index without new world state data, + // but the state should be accepting. + // Access the internal state to check. + const internalState = (context.orchestrator as any).provingState as EpochProvingState; + expect(internalState.isAcceptingCheckpoints()).toBe(true); + }); + + it( + 'removeLastCheckpoint throws if finalizeEpochStructure has already been called', + async () => { + const { constants, blocks, previousBlockHeader } = await context.makeCheckpoint(1, { numTxsPerBlock: 0 }); + const finalBlobChallenges = await context.getFinalBlobChallenges(); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + await context.orchestrator.startNewCheckpoint(0, constants, [], 1, previousBlockHeader); + const { blockNumber, timestamp } = blocks[0].header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, 0); + + await context.orchestrator.finalizeEpochStructure(1, finalBlobChallenges); + + expect(() => context.orchestrator.removeLastCheckpoint()).toThrow( + 'Cannot remove checkpoints after epoch structure has been finalized.', + ); + }, + LONG_TIMEOUT, + ); + + it('removeLastCheckpoint on empty epoch logs a warning but does not throw', () => { + context.orchestrator.startNewEpoch(EpochNumber(1)); + // Should not throw, just logs a warning about no checkpoint to remove. + expect(() => context.orchestrator.removeLastCheckpoint()).not.toThrow(); + }); + + it('removeLastCheckpoint without starting epoch throws', () => { + expect(() => context.orchestrator.removeLastCheckpoint()).toThrow('Empty epoch proving state.'); + }); + + it( + 'multiple sequential removes work (remove last, then remove new last)', + async () => { + const checkpoints = await timesAsync(3, () => context.makeCheckpoint(1, { numTxsPerBlock: 0 })); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + + // Add three checkpoints. + for (let i = 0; i < 3; i++) { + const { constants, blocks, previousBlockHeader } = checkpoints[i]; + await context.orchestrator.startNewCheckpoint(i, constants, [], 1, previousBlockHeader); + const { blockNumber, timestamp } = blocks[0].header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, 0); + } + + // Remove last (index 2). + context.orchestrator.removeLastCheckpoint(); + // Remove new last (index 1). + context.orchestrator.removeLastCheckpoint(); + + // One checkpoint remains. Should still be able to finalize with it. + const internalState = (context.orchestrator as any).provingState as EpochProvingState; + expect(internalState.isAcceptingCheckpoints()).toBe(true); + expect(internalState.getCheckpointProvingState(0)).toBeDefined(); + expect(internalState.getCheckpointProvingState(1)).toBeUndefined(); + expect(internalState.getCheckpointProvingState(2)).toBeUndefined(); + }, + LONG_TIMEOUT, + ); + + it( + 'after removing last checkpoint, remaining checkpoint state is consistent', + async () => { + const checkpoints = await timesAsync(3, () => context.makeCheckpoint(1, { numTxsPerBlock: 0 })); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + + // Add three checkpoints. + for (let i = 0; i < 3; i++) { + const { constants, blocks, previousBlockHeader } = checkpoints[i]; + await context.orchestrator.startNewCheckpoint(i, constants, [], 1, previousBlockHeader); + const { blockNumber, timestamp } = blocks[0].header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, 0); + } + + // Remove last two checkpoints, simulating a reorg. + context.orchestrator.removeLastCheckpoint(); + context.orchestrator.removeLastCheckpoint(); + + // The internal state should have only the first checkpoint remaining. + const internalState = (context.orchestrator as any).provingState as EpochProvingState; + expect(internalState.getCheckpointProvingState(0)).toBeDefined(); + expect(internalState.getCheckpointProvingState(1)).toBeUndefined(); + expect(internalState.getCheckpointProvingState(2)).toBeUndefined(); + + // Still accepting checkpoints (epoch structure not finalized). + expect(internalState.isAcceptingCheckpoints()).toBe(true); + expect(internalState.isEpochStructureFinalized).toBe(false); + }, + LONG_TIMEOUT, + ); + }); + + describe('reorg with replacement and full proving', () => { + it( + 'remove last checkpoint with txs, add replacement, finalize, and prove epoch', + async () => { + const numTxsPerBlock = 1; + + // All checkpoints have real txs that modify world state. + const checkpoint1 = await context.makeCheckpoint(1, { numTxsPerBlock }); + const checkpoint2 = await context.makeCheckpoint(1, { numTxsPerBlock }); + const checkpointToRemove = await context.makeCheckpoint(1, { numTxsPerBlock }); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + + // Add all 3 checkpoints and process their txs. + const allCheckpoints = [checkpoint1, checkpoint2, checkpointToRemove]; + for (let i = 0; i < allCheckpoints.length; i++) { + const { constants, blocks, previousBlockHeader } = allCheckpoints[i]; + await context.orchestrator.startNewCheckpoint(i, constants, [], 1, previousBlockHeader); + const { blockNumber, timestamp } = blocks[0].header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, blocks[0].txs.length); + await context.orchestrator.addTxs(blocks[0].txs); + await context.orchestrator.setBlockCompleted(blockNumber, blocks[0].header); + } + + // Simulate reorg: remove the third checkpoint (which had real txs and world state changes). + context.orchestrator.removeLastCheckpoint(); + await context.removeLastCheckpoint(); + + // Create a replacement checkpoint with txs and add it at the same index. + const replacement = await context.makeCheckpoint(1, { numTxsPerBlock }); + const finalBlobChallenges = await context.getFinalBlobChallenges(); + + await context.orchestrator.startNewCheckpoint( + 2, + replacement.constants, + [], + 1, + replacement.previousBlockHeader, + ); + const { blockNumber, timestamp } = replacement.blocks[0].header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, replacement.blocks[0].txs.length); + await context.orchestrator.addTxs(replacement.blocks[0].txs); + await context.orchestrator.setBlockCompleted(blockNumber, replacement.blocks[0].header); + + // Finalize and prove — world state must be consistent with the replacement, not the removed. + await context.orchestrator.finalizeEpochStructure(3, finalBlobChallenges); + const epoch = await context.orchestrator.finalizeEpoch(); + expect(epoch.proof).toBeDefined(); + + const headerHashes = [checkpoint1, checkpoint2, replacement].map(c => c.header.hash()); + expect(epoch.publicInputs.checkpointHeaderHashes).toEqual( + padArrayEnd(headerHashes, Fr.ZERO, MAX_CHECKPOINTS_PER_EPOCH), + ); + }, + LONG_TIMEOUT, + ); + + it( + 'remove last checkpoint with txs, finalize with fewer checkpoints, and prove epoch', + async () => { + const numTxsPerBlock = 1; + + // All checkpoints have real txs. + const checkpoint1 = await context.makeCheckpoint(1, { numTxsPerBlock }); + const checkpoint2 = await context.makeCheckpoint(1, { numTxsPerBlock }); + const checkpointToRemove = await context.makeCheckpoint(1, { numTxsPerBlock }); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + + // Add and process all 3 checkpoints. + const allCheckpoints = [checkpoint1, checkpoint2, checkpointToRemove]; + for (let i = 0; i < allCheckpoints.length; i++) { + const { constants, blocks, previousBlockHeader } = allCheckpoints[i]; + await context.orchestrator.startNewCheckpoint(i, constants, [], 1, previousBlockHeader); + const { blockNumber, timestamp } = blocks[0].header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, blocks[0].txs.length); + await context.orchestrator.addTxs(blocks[0].txs); + await context.orchestrator.setBlockCompleted(blockNumber, blocks[0].header); + } + + // Reorg removes the last checkpoint — world state is rolled back. + context.orchestrator.removeLastCheckpoint(); + await context.removeLastCheckpoint(); + + // Finalize with only 2 checkpoints. + const finalBlobChallenges = await context.getFinalBlobChallenges(); + await context.orchestrator.finalizeEpochStructure(2, finalBlobChallenges); + const epoch = await context.orchestrator.finalizeEpoch(); + expect(epoch.proof).toBeDefined(); + + const headerHashes = [checkpoint1, checkpoint2].map(c => c.header.hash()); + expect(epoch.publicInputs.checkpointHeaderHashes).toEqual( + padArrayEnd(headerHashes, Fr.ZERO, MAX_CHECKPOINTS_PER_EPOCH), + ); + }, + LONG_TIMEOUT, + ); + + it( + 'in-flight proving jobs for removed checkpoint (with txs) do not cause errors', + async () => { + const numTxsPerBlock = 1; + + // Both checkpoints have real txs that kick off proving. + const checkpoint1 = await context.makeCheckpoint(1, { numTxsPerBlock }); + const checkpoint2 = await context.makeCheckpoint(1, { numTxsPerBlock }); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + + // Add both checkpoints and process their blocks (kicks off proving). + for (let i = 0; i < 2; i++) { + const { constants, blocks, previousBlockHeader } = [checkpoint1, checkpoint2][i]; + await context.orchestrator.startNewCheckpoint(i, constants, [], 1, previousBlockHeader); + const { blockNumber, timestamp } = blocks[0].header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, blocks[0].txs.length); + await context.orchestrator.addTxs(blocks[0].txs); + await context.orchestrator.setBlockCompleted(blockNumber, blocks[0].header); + } + + // Remove last checkpoint while proving jobs are in-flight — world state rolled back. + context.orchestrator.removeLastCheckpoint(); + await context.removeLastCheckpoint(); + + // Wait deterministically for ALL proving jobs to settle (including orphaned ones + // from the removed checkpoint). This is stronger than waitForAllCheckpointsReady + // which only tracks the surviving checkpoint. + while (context.orchestrator.getNumPendingProvingJobs() > 0) { + await sleep(50); + } + + // At this point every callback from the removed checkpoint has fired. + // Verify no world state forks leaked — removeLastCheckpoint closed the removed + // checkpoint's forks, and block completion closed the surviving checkpoint's fork. + expect(context.orchestrator.getNumActiveForks()).toBe(0); + + // The epoch should still be in a valid state — no errors from orphaned callbacks. + // Finalize with just the first checkpoint and prove successfully. + const finalBlobChallenges = await context.getFinalBlobChallenges(); + await context.orchestrator.finalizeEpochStructure(1, finalBlobChallenges); + const epoch = await context.orchestrator.finalizeEpoch(); + expect(epoch.proof).toBeDefined(); + }, + LONG_TIMEOUT, + ); + }); + + describe('edge cases', () => { + it( + 'epoch cancellation discards all state properly', + async () => { + const { constants, blocks, previousBlockHeader } = await context.makeCheckpoint(1, { numTxsPerBlock: 1 }); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + await context.orchestrator.startNewCheckpoint(0, constants, [], 1, previousBlockHeader); + + const { blockNumber, timestamp } = blocks[0].header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, blocks[0].txs.length); + await context.orchestrator.addTxs(blocks[0].txs); + + // Cancel mid-proving. + context.orchestrator.cancel(); + + // Starting a new epoch should work after cancellation. + expect(() => context.orchestrator.startNewEpoch(EpochNumber(2))).not.toThrow(); + }, + LONG_TIMEOUT, + ); + + it( + 'cancel rejects waitForAllCheckpointsReady so epoch proving job can exit', + async () => { + const { constants, blocks, previousBlockHeader } = await context.makeCheckpoint(1, { numTxsPerBlock: 1 }); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + await context.orchestrator.startNewCheckpoint(0, constants, [], 1, previousBlockHeader); + + const { blockNumber, timestamp } = blocks[0].header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, blocks[0].txs.length); + await context.orchestrator.addTxs(blocks[0].txs); + + // Start waiting for checkpoints — this will hang until cancel rejects it. + const waitPromise = context.orchestrator.waitForAllCheckpointsReady(); + + // Cancel the epoch — should reject the wait promise. + context.orchestrator.cancel(); + + await expect(waitPromise).rejects.toEqual('Proving cancelled'); + }, + LONG_TIMEOUT, + ); + + it( + 'cancel while awaiting checkpoints does not deadlock prover node shutdown', + async () => { + const numTxsPerBlock = 1; + const checkpoint1 = await context.makeCheckpoint(1, { numTxsPerBlock }); + const checkpoint2 = await context.makeCheckpoint(1, { numTxsPerBlock }); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + + // Register and process both checkpoints. + for (let i = 0; i < 2; i++) { + const { constants, blocks, previousBlockHeader } = [checkpoint1, checkpoint2][i]; + await context.orchestrator.startNewCheckpoint(i, constants, [], 1, previousBlockHeader); + const { blockNumber, timestamp } = blocks[0].header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, blocks[0].txs.length); + await context.orchestrator.addTxs(blocks[0].txs); + await context.orchestrator.setBlockCompleted(blockNumber, blocks[0].header); + } + + // Simulate: epoch proving job is awaiting checkpoints-ready. + const waitPromise = context.orchestrator.waitForAllCheckpointsReady().catch(() => { + // Expected: cancel rejects this. + }); + + // Simulate: prover node shuts down while proving is in progress. + // This should not deadlock — cancel rejects the wait, allowing the job to exit. + context.orchestrator.cancel(); + await waitPromise; + + // After cancel, a new epoch can start. + expect(() => context.orchestrator.startNewEpoch(EpochNumber(2))).not.toThrow(); + }, + LONG_TIMEOUT, + ); + + it( + 'waitForAllCheckpointsReady rejects immediately if epoch is already cancelled', + async () => { + const { constants, previousBlockHeader } = await context.makeCheckpoint(1, { numTxsPerBlock: 0 }); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + await context.orchestrator.startNewCheckpoint(0, constants, [], 1, previousBlockHeader); + + // Cancel first. + context.orchestrator.cancel(); + + // Calling waitForAllCheckpointsReady after cancel should reject immediately. + await expect(context.orchestrator.waitForAllCheckpointsReady()).rejects.toThrow(); + }, + LONG_TIMEOUT, + ); + + it('starting a new epoch while previous is active throws', async () => { + const { constants, previousBlockHeader } = await context.makeCheckpoint(1, { numTxsPerBlock: 0 }); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + await context.orchestrator.startNewCheckpoint(0, constants, [], 1, previousBlockHeader); + + expect(() => context.orchestrator.startNewEpoch(EpochNumber(2))).toThrow( + 'Cannot start epoch 2 when epoch 1 is still being processed.', + ); + }); + + it( + 'finalize-first with multiple checkpoints: all complete after finalize', + async () => { + const numCheckpoints = 2; + const numBlocksPerCheckpoint = 1; + const numTxsPerBlock = 1; + + const checkpoints = await timesAsync(numCheckpoints, () => + context.makeCheckpoint(numBlocksPerCheckpoint, { numTxsPerBlock }), + ); + const finalBlobChallenges = await context.getFinalBlobChallenges(); + + context.orchestrator.startNewEpoch(EpochNumber(1)); + + // Add all checkpoints. + for (let i = 0; i < checkpoints.length; i++) { + const { constants, previousBlockHeader } = checkpoints[i]; + await context.orchestrator.startNewCheckpoint( + i, + constants, + [], + numBlocksPerCheckpoint, + previousBlockHeader, + ); + } + + // Finalize before processing any blocks. + await context.orchestrator.finalizeEpochStructure(numCheckpoints, finalBlobChallenges); + + // Now process all blocks. + for (let i = 0; i < checkpoints.length; i++) { + const { + blocks: [block], + } = checkpoints[i]; + const { blockNumber, timestamp } = block.header.globalVariables; + await context.orchestrator.startNewBlock(blockNumber, timestamp, block.txs.length); + await context.orchestrator.addTxs(block.txs); + await context.orchestrator.setBlockCompleted(blockNumber, block.header); + } + + const epoch = await context.orchestrator.finalizeEpoch(); + expect(epoch.proof).toBeDefined(); + }, + LONG_TIMEOUT, + ); + }); + }); +}); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_errors.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_errors.test.ts index 3d148f6beca7..5849e337602b 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_errors.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_errors.test.ts @@ -1,4 +1,3 @@ -import type { FinalBlobBatchingChallenges } from '@aztec/blob-lib/types'; import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants'; import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types'; import { Fr } from '@aztec/foundation/curves/bn254'; @@ -17,7 +16,6 @@ describe('prover/orchestrator/errors', () => { let constants: CheckpointConstantData; let block: { header: BlockHeader; txs: ProcessedTx[] }; let previousBlockHeader: BlockHeader; - let finalBlobChallenges: FinalBlobBatchingChallenges; const numBlocks = 1; beforeEach(async () => { @@ -28,7 +26,6 @@ describe('prover/orchestrator/errors', () => { blocks: [block], previousBlockHeader, } = await context.makeCheckpoint(numBlocks, { numTxsPerBlock: 1 })); - finalBlobChallenges = await context.getFinalBlobChallenges(); }); afterEach(async () => { @@ -39,7 +36,7 @@ describe('prover/orchestrator/errors', () => { describe('errors', () => { it('throws if adding too many transactions', async () => { - orchestrator.startNewEpoch(EpochNumber(1), 1 /* numCheckpoints */, finalBlobChallenges); + orchestrator.startNewEpoch(EpochNumber(1)); await orchestrator.startNewCheckpoint( 0, // checkpointIndex @@ -58,7 +55,7 @@ describe('prover/orchestrator/errors', () => { }); it('throws if adding too many blocks', async () => { - orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + orchestrator.startNewEpoch(EpochNumber(1)); await orchestrator.startNewCheckpoint( 0, // checkpointIndex @@ -79,7 +76,7 @@ describe('prover/orchestrator/errors', () => { }); it('throws if adding empty block as non-first block', async () => { - orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + orchestrator.startNewEpoch(EpochNumber(1)); await orchestrator.startNewCheckpoint( 0, // checkpointIndex @@ -103,7 +100,7 @@ describe('prover/orchestrator/errors', () => { }); it('throws if adding a transaction before starting checkpoint', async () => { - orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + orchestrator.startNewEpoch(EpochNumber(1)); await expect(async () => await orchestrator.addTxs(block.txs)).rejects.toThrow( /Proving state for block 1 not found/, @@ -111,7 +108,7 @@ describe('prover/orchestrator/errors', () => { }); it('throws if adding a transaction before starting block', async () => { - orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + orchestrator.startNewEpoch(EpochNumber(1)); await orchestrator.startNewCheckpoint( 0, // checkpointIndex constants, @@ -125,7 +122,7 @@ describe('prover/orchestrator/errors', () => { }); it('throws if completing a block before start', async () => { - orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + orchestrator.startNewEpoch(EpochNumber(1)); await orchestrator.startNewCheckpoint( 0, // checkpointIndex constants, @@ -139,7 +136,7 @@ describe('prover/orchestrator/errors', () => { }); it('throws if adding to a cancelled block', async () => { - orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + orchestrator.startNewEpoch(EpochNumber(1)); await orchestrator.startNewCheckpoint( 0, // checkpointIndex constants, @@ -156,7 +153,7 @@ describe('prover/orchestrator/errors', () => { it('rejects if too many l1 to l2 messages are provided', async () => { const l1ToL2Messages = new Array(NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP + 1).fill(new Fr(0n)); - orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + orchestrator.startNewEpoch(EpochNumber(1)); await expect( async () => await orchestrator.startNewCheckpoint( diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts index 0c127406b133..5ac4558f7835 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts @@ -54,7 +54,7 @@ describe('prover/orchestrator/failures', () => { ); const finalBlobChallenges = await context.getFinalBlobChallenges(); - orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + orchestrator.startNewEpoch(EpochNumber(1)); for (let checkpointIndex = 0; checkpointIndex < checkpoints.length; checkpointIndex++) { const { constants, blocks, l1ToL2Messages, previousBlockHeader } = checkpoints[checkpointIndex]; @@ -92,6 +92,12 @@ describe('prover/orchestrator/failures', () => { break; } } + + try { + await orchestrator.finalizeEpochStructure(checkpoints.length, finalBlobChallenges); + } catch { + // Epoch structure finalization may fail if the proving state was already rejected. + } }; it( diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts index 687f294711ed..c5f1222ac495 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts @@ -47,8 +47,7 @@ describe('prover/orchestrator/lifecycle', () => { numTxsPerBlock: 0, }); - const finalBlobChallenges = await context.getFinalBlobChallenges(); - orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + orchestrator.startNewEpoch(EpochNumber(1)); await orchestrator.startNewCheckpoint( 0, // checkpointIndex @@ -91,8 +90,7 @@ describe('prover/orchestrator/lifecycle', () => { numTxsPerBlock: 0, }); - const finalBlobChallenges = await context.getFinalBlobChallenges(); - orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + orchestrator.startNewEpoch(EpochNumber(1)); await orchestrator.startNewCheckpoint(0, constants, [], 1, previousBlockHeader); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_multiple_checkpoints.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_multiple_checkpoints.test.ts index 7b013bf0a62a..da580a8ce767 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_multiple_checkpoints.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_multiple_checkpoints.test.ts @@ -35,7 +35,7 @@ describe('prover/orchestrator/multi-checkpoints', () => { logger.info(`Starting new epoch with ${numCheckpoints} checkpoints`); const finalBlobChallenges = await context.getFinalBlobChallenges(); - context.orchestrator.startNewEpoch(EpochNumber(1), numCheckpoints, finalBlobChallenges); + context.orchestrator.startNewEpoch(EpochNumber(1)); for (let i = 0; i < checkpoints.length; i++) { const { @@ -58,6 +58,7 @@ describe('prover/orchestrator/multi-checkpoints', () => { } logger.info('Finalizing epoch'); + await context.orchestrator.finalizeEpochStructure(numCheckpoints, finalBlobChallenges); const epoch = await context.orchestrator.finalizeEpoch(); expect(epoch.proof).toBeDefined(); @@ -90,7 +91,7 @@ describe('prover/orchestrator/multi-checkpoints', () => { const epochNumber = epochIndex + 1; const { checkpoints, finalBlobChallenges } = epochs[epochIndex]; logger.info(`Starting epoch ${epochNumber} with ${checkpoints.length} checkpoints`); - context.orchestrator.startNewEpoch(EpochNumber(epochNumber), checkpoints.length, finalBlobChallenges); + context.orchestrator.startNewEpoch(EpochNumber(epochNumber)); for (let i = 0; i < checkpoints.length; i++) { const { @@ -120,6 +121,7 @@ describe('prover/orchestrator/multi-checkpoints', () => { ); logger.info('Finalizing epoch'); + await context.orchestrator.finalizeEpochStructure(checkpoints.length, finalBlobChallenges); const epoch = await context.orchestrator.finalizeEpoch(); expect(epoch.proof).toBeDefined(); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_rollup_structure.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_rollup_structure.test.ts index 3ceb5715adc2..222eb566577a 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_rollup_structure.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_rollup_structure.test.ts @@ -122,7 +122,7 @@ describe('prover/orchestrator/rollup-structure', () => { }); const finalBlobChallenges = await context.getFinalBlobChallenges(); - context.orchestrator.startNewEpoch(EpochNumber(1) /* epochNumber */, numCheckpoints, finalBlobChallenges); + context.orchestrator.startNewEpoch(EpochNumber(1)); for (let checkpointIndex = 0; checkpointIndex < checkpoints.length; checkpointIndex++) { const { constants, blocks, l1ToL2Messages, previousBlockHeader } = checkpoints[checkpointIndex]; @@ -143,6 +143,7 @@ describe('prover/orchestrator/rollup-structure', () => { } } + await context.orchestrator.finalizeEpochStructure(numCheckpoints, finalBlobChallenges); const result = await context.orchestrator.finalizeEpoch(); expect(result.publicInputs.previousArchiveRoot).toEqual(epochStartArchive.root); @@ -194,7 +195,7 @@ describe('prover/orchestrator/rollup-structure', () => { }); const finalBlobChallenges = await context.getFinalBlobChallenges(); - context.orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + context.orchestrator.startNewEpoch(EpochNumber(1)); await context.orchestrator.startNewCheckpoint( 0, // checkpointIndex @@ -208,6 +209,7 @@ describe('prover/orchestrator/rollup-structure', () => { await context.orchestrator.startNewBlock(blockNumber, timestamp, block.txs.length); await context.orchestrator.setBlockCompleted(blockNumber, block.header); + await context.orchestrator.finalizeEpochStructure(1, finalBlobChallenges); const result = await context.orchestrator.finalizeEpoch(); expect(result.publicInputs.previousArchiveRoot).toEqual(epochStartArchive.root); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts index 79fd868b090e..2c49c35f8ef5 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_single_blocks.test.ts @@ -26,7 +26,7 @@ describe('prover/orchestrator/blocks', () => { } = await context.makeCheckpoint(1, { numTxsPerBlock: 0 }); const finalBlobChallenges = await context.getFinalBlobChallenges(); - context.orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + context.orchestrator.startNewEpoch(EpochNumber(1)); await context.orchestrator.startNewCheckpoint( 0, // checkpointIndex @@ -40,6 +40,7 @@ describe('prover/orchestrator/blocks', () => { await context.orchestrator.startNewBlock(blockNumber, timestamp, 0 /* numTxs */); const header = await context.orchestrator.setBlockCompleted(blockNumber, emptyBlock.header); + await context.orchestrator.finalizeEpochStructure(1, finalBlobChallenges); await context.orchestrator.finalizeEpoch(); expect(header).toEqual(emptyBlock.header); }); @@ -52,7 +53,7 @@ describe('prover/orchestrator/blocks', () => { } = await context.makeCheckpoint(1, { numTxsPerBlock: 1 }); const finalBlobChallenges = await context.getFinalBlobChallenges(); - context.orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + context.orchestrator.startNewEpoch(EpochNumber(1)); await context.orchestrator.startNewCheckpoint( 0, // checkpointIndex @@ -67,6 +68,7 @@ describe('prover/orchestrator/blocks', () => { await context.orchestrator.addTxs(block.txs); const header = await context.orchestrator.setBlockCompleted(blockNumber, block.header); + await context.orchestrator.finalizeEpochStructure(1, finalBlobChallenges); await context.orchestrator.finalizeEpoch(); expect(header).toEqual(block.header); }); @@ -84,7 +86,7 @@ describe('prover/orchestrator/blocks', () => { }); const finalBlobChallenges = await context.getFinalBlobChallenges(); - context.orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + context.orchestrator.startNewEpoch(EpochNumber(1)); await context.orchestrator.startNewCheckpoint( 0, // checkpointIndex @@ -100,6 +102,7 @@ describe('prover/orchestrator/blocks', () => { await context.orchestrator.addTxs(block.txs); const header = await context.orchestrator.setBlockCompleted(blockNumber, block.header); + await context.orchestrator.finalizeEpochStructure(1, finalBlobChallenges); await context.orchestrator.finalizeEpoch(); expect(header).toEqual(block.header); }); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_single_checkpoint.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_single_checkpoint.test.ts index be7a9b017843..44673da338aa 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_single_checkpoint.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_single_checkpoint.test.ts @@ -34,7 +34,7 @@ describe('prover/orchestrator/single-checkpoint', () => { }); const finalBlobChallenges = await context.getFinalBlobChallenges(); - context.orchestrator.startNewEpoch(EpochNumber(1), numCheckpoints, finalBlobChallenges); + context.orchestrator.startNewEpoch(EpochNumber(1)); await context.orchestrator.startNewCheckpoint( 0, // checkpointIndex @@ -53,6 +53,7 @@ describe('prover/orchestrator/single-checkpoint', () => { await context.orchestrator.setBlockCompleted(blockNumber, block.header); } + await context.orchestrator.finalizeEpochStructure(numCheckpoints, finalBlobChallenges); const epoch = await context.orchestrator.finalizeEpoch(); expect(epoch.proof).toBeDefined(); @@ -80,7 +81,7 @@ describe('prover/orchestrator/single-checkpoint', () => { }); const finalBlobChallenges = await context.getFinalBlobChallenges(); - context.orchestrator.startNewEpoch(EpochNumber(1), numCheckpoints, finalBlobChallenges); + context.orchestrator.startNewEpoch(EpochNumber(1)); await context.orchestrator.startNewCheckpoint( 0, // checkpointIndex @@ -97,6 +98,7 @@ describe('prover/orchestrator/single-checkpoint', () => { await context.orchestrator.setBlockCompleted(blockNumber, block.header); } + await context.orchestrator.finalizeEpochStructure(numCheckpoints, finalBlobChallenges); const epoch = await context.orchestrator.finalizeEpoch(); expect(epoch.proof).toBeDefined(); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts index f62184a6dbfd..c22370dd28b0 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_workflow.test.ts @@ -87,8 +87,7 @@ describe('prover/orchestrator', () => { } }); - const finalBlobChallenges = await context.getFinalBlobChallenges(); - orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + orchestrator.startNewEpoch(EpochNumber(1)); await orchestrator.startNewCheckpoint( 0, // checkpointIndex @@ -140,7 +139,7 @@ describe('prover/orchestrator', () => { } = await context.makeCheckpoint(numBlocks); const finalBlobChallenges = await context.getFinalBlobChallenges(); - orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + orchestrator.startNewEpoch(EpochNumber(1)); await orchestrator.startNewCheckpoint( 0, // checkpointIndex @@ -156,6 +155,7 @@ describe('prover/orchestrator', () => { // now finish the block await orchestrator.setBlockCompleted(blockNumber); + await orchestrator.finalizeEpochStructure(1, finalBlobChallenges); const result = await orchestrator.finalizeEpoch(); expect(result.proof).toBeDefined(); }); @@ -169,7 +169,7 @@ describe('prover/orchestrator', () => { } = await context.makeCheckpoint(numBlocks); const finalBlobChallenges = await context.getFinalBlobChallenges(); - orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + orchestrator.startNewEpoch(EpochNumber(1)); await orchestrator.startNewCheckpoint( 0, // checkpointIndex @@ -185,6 +185,7 @@ describe('prover/orchestrator', () => { // now finish the block await orchestrator.setBlockCompleted(blockNumber); + await orchestrator.finalizeEpochStructure(1, finalBlobChallenges); const result = await orchestrator.finalizeEpoch(); expect(result.proof).toBeDefined(); // Forks are closed deterministically in setBlockCompleted, so no cancel() needed. @@ -205,7 +206,7 @@ describe('prover/orchestrator', () => { }); const finalBlobChallenges = await context.getFinalBlobChallenges(); - orchestrator.startNewEpoch(EpochNumber(1), 1, finalBlobChallenges); + orchestrator.startNewEpoch(EpochNumber(1)); await orchestrator.startNewCheckpoint( 0, // checkpointIndex @@ -236,6 +237,7 @@ describe('prover/orchestrator', () => { await orchestrator.addTxs(txs); await orchestrator.setBlockCompleted(blockNumber); + await orchestrator.finalizeEpochStructure(1, finalBlobChallenges); const result = await orchestrator.finalizeEpoch(); expect(result.proof).toBeDefined(); expect(getChonkVerifierSpy).toHaveBeenCalledTimes(0); @@ -252,9 +254,9 @@ describe('prover/orchestrator', () => { ); const finalBlobChallenges = await context.getFinalBlobChallenges(); - context.orchestrator.startNewEpoch(EpochNumber(1), numCheckpoints, finalBlobChallenges); + context.orchestrator.startNewEpoch(EpochNumber(1)); - // Start checkpoint in reverse order. + // Start checkpoints in reverse order. for (let checkpointIndex = numCheckpoints - 1; checkpointIndex >= 0; checkpointIndex--) { const { constants, blocks, l1ToL2Messages, previousBlockHeader } = checkpoints[checkpointIndex]; await context.orchestrator.startNewCheckpoint( @@ -276,6 +278,7 @@ describe('prover/orchestrator', () => { } logger.info('Finalizing epoch'); + await context.orchestrator.finalizeEpochStructure(numCheckpoints, finalBlobChallenges); const epoch = await context.orchestrator.finalizeEpoch(); expect(epoch.proof).toBeDefined(); }); @@ -289,7 +292,7 @@ describe('prover/orchestrator', () => { ); const finalBlobChallenges = await context.getFinalBlobChallenges(); - context.orchestrator.startNewEpoch(EpochNumber(1), numCheckpoints, finalBlobChallenges); + context.orchestrator.startNewEpoch(EpochNumber(1)); await Promise.all( checkpoints.map(async (checkpoint, checkpointIndex) => { @@ -312,6 +315,10 @@ describe('prover/orchestrator', () => { } }), ); + + await context.orchestrator.finalizeEpochStructure(numCheckpoints, finalBlobChallenges); + const epoch = await context.orchestrator.finalizeEpoch(); + expect(epoch.proof).toBeDefined(); }); }); }); diff --git a/yarn-project/prover-client/src/prover-client/server-epoch-prover.ts b/yarn-project/prover-client/src/prover-client/server-epoch-prover.ts index dd1715757d6c..78df78a6a347 100644 --- a/yarn-project/prover-client/src/prover-client/server-epoch-prover.ts +++ b/yarn-project/prover-client/src/prover-client/server-epoch-prover.ts @@ -18,13 +18,19 @@ export class ServerEpochProver implements EpochProver { private orchestrator: ProvingOrchestrator, ) {} - startNewEpoch( - epochNumber: EpochNumber, + startNewEpoch(epochNumber: EpochNumber): void { + this.orchestrator.startNewEpoch(epochNumber); + this.facade.start(); + } + + finalizeEpochStructure( totalNumCheckpoints: number, finalBlobBatchingChallenges: FinalBlobBatchingChallenges, - ): void { - this.orchestrator.startNewEpoch(epochNumber, totalNumCheckpoints, finalBlobBatchingChallenges); - this.facade.start(); + ): Promise { + return this.orchestrator.finalizeEpochStructure(totalNumCheckpoints, finalBlobBatchingChallenges); + } + waitForAllCheckpointsReady(): Promise { + return this.orchestrator.waitForAllCheckpointsReady(); } startNewCheckpoint( checkpointIndex: number, diff --git a/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts b/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts index bc66a883cc0c..9f5fb006d91d 100644 --- a/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts +++ b/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts @@ -53,7 +53,7 @@ describe('prover/bb_prover/full-rollup', () => { ); const finalBlobChallenges = await context.getFinalBlobChallenges(); - context.orchestrator.startNewEpoch(EpochNumber(1), numCheckpoints, finalBlobChallenges); + context.orchestrator.startNewEpoch(EpochNumber(1)); for (let checkpointIndex = 0; checkpointIndex < numCheckpoints; checkpointIndex++) { const { constants, blocks, l1ToL2Messages, previousBlockHeader } = checkpoints[checkpointIndex]; @@ -81,6 +81,7 @@ describe('prover/bb_prover/full-rollup', () => { } log.info(`Awaiting proofs`); + await context.orchestrator.finalizeEpochStructure(numCheckpoints, finalBlobChallenges); const epochResult = await context.orchestrator.finalizeEpoch(); if (prover) { diff --git a/yarn-project/prover-node/src/job/epoch-proving-job.ts b/yarn-project/prover-node/src/job/epoch-proving-job.ts index 1349cf517df6..1162f00c3536 100644 --- a/yarn-project/prover-node/src/job/epoch-proving-job.ts +++ b/yarn-project/prover-node/src/job/epoch-proving-job.ts @@ -149,12 +149,7 @@ export class EpochProvingJob implements Traceable { this.runPromise = promise; try { - const blobTimer = new Timer(); - const blobFieldsPerCheckpoint = this.checkpoints.map(checkpoint => checkpoint.toBlobFields()); - const finalBlobBatchingChallenges = await buildFinalBlobChallenges(blobFieldsPerCheckpoint); - this.metrics.recordBlobProcessing(blobTimer.ms()); - - this.prover.startNewEpoch(epochNumber, epochSizeCheckpoints, finalBlobBatchingChallenges); + this.prover.startNewEpoch(epochNumber); const chonkTimer = new Timer(); await this.prover.startChonkVerifierCircuits(Array.from(this.txs.values())); this.metrics.recordChonkVerifier(chonkTimer.ms()); @@ -260,6 +255,18 @@ export class EpochProvingJob implements Traceable { }); this.metrics.recordAllCheckpointsProcessing(allCheckpointsTimer.ms()); + // Wait for all checkpoints to complete block-level proving before finalizing the epoch structure. + // In the current non-optimistic flow this will resolve once all block merge proofs complete. + // In the future optimistic flow, this will be awaited alongside the epoch-end signal. + await this.prover.waitForAllCheckpointsReady(); + + const blobTimer = new Timer(); + const blobFieldsPerCheckpoint = this.checkpoints.map(checkpoint => checkpoint.toBlobFields()); + const finalBlobBatchingChallenges = await buildFinalBlobChallenges(blobFieldsPerCheckpoint); + this.metrics.recordBlobProcessing(blobTimer.ms()); + + await this.prover.finalizeEpochStructure(epochSizeCheckpoints, finalBlobBatchingChallenges); + const executionTime = timer.ms(); this.progressState('awaiting-prover'); diff --git a/yarn-project/stdlib/src/interfaces/epoch-prover.ts b/yarn-project/stdlib/src/interfaces/epoch-prover.ts index 4a0103ea66b8..bc92983ebbca 100644 --- a/yarn-project/stdlib/src/interfaces/epoch-prover.ts +++ b/yarn-project/stdlib/src/interfaces/epoch-prover.ts @@ -16,14 +16,26 @@ export interface EpochProver extends Omit; + + /** + * Returns a promise that resolves when all current checkpoints have completed block-level proving + * (everything up to but not including checkpoint roots). + */ + waitForAllCheckpointsReady(): Promise; /** * Starts a new checkpoint.