diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 50dc0765..05bedfc9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,3 +75,11 @@ jobs: - name: Run fixture-based tests uses: ./.github/actions/run-fixture-tests + + # Reuses the release build from the test step; validates the benchmark + # harness end-to-end and its JSON output contract in a few seconds. + - name: Benchmark smoke (mock crypto) + run: | + cargo run --release --bin ethlambda -- benchmark synthetic --mock-crypto \ + --num-validators 4 --warmup-slots 4 --iterations 3 --format json \ + | jq -e '.schema_version == 1 and (.samples | length == 3)' diff --git a/Cargo.lock b/Cargo.lock index 00958308..cd10cf6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2012,6 +2012,7 @@ dependencies = [ "clap", "ethlambda-blockchain", "ethlambda-crypto", + "ethlambda-metrics", "ethlambda-network-api", "ethlambda-p2p", "ethlambda-rpc", @@ -2024,6 +2025,7 @@ dependencies = [ "libssz-types", "reqwest", "serde", + "serde_json", "serde_yaml_ng", "thiserror 2.0.18", "tikv-jemallocator", diff --git a/Makefile b/Makefile index d404100a..ee6c28dc 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help fmt lint docker-build shadow-build shadow-docker-build run-devnet test docs docs-deps docs-serve +.PHONY: help fmt lint bench docker-build shadow-build shadow-docker-build run-devnet test docs docs-deps docs-serve help: ## πŸ“š Show help for each of the Makefile recipes @grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' @@ -14,6 +14,11 @@ test: leanSpec/fixtures ## πŸ§ͺ Run all tests # signature verification/aggregation, without paying for LTO on every rebuild cargo test --workspace --profile release-fast +BENCH_ARGS ?= synthetic --mock-crypto + +bench: ## 🏁 Benchmark block building offline (override BENCH_ARGS to customize) + cargo run --release --bin ethlambda -- benchmark $(BENCH_ARGS) + GIT_COMMIT=$(shell git rev-parse HEAD) GIT_BRANCH=$(shell git rev-parse --abbrev-ref HEAD) DOCKER_TAG?=local diff --git a/bin/ethlambda/Cargo.toml b/bin/ethlambda/Cargo.toml index 94913342..591490ca 100644 --- a/bin/ethlambda/Cargo.toml +++ b/bin/ethlambda/Cargo.toml @@ -21,6 +21,7 @@ shadow-integration = ["ethlambda-crypto/shadow-integration"] [dependencies] ethlambda-blockchain.workspace = true ethlambda-crypto.workspace = true +ethlambda-metrics.workspace = true ethlambda-network-api.workspace = true ethlambda-p2p.workspace = true ethlambda-types.workspace = true @@ -37,6 +38,7 @@ tracing.workspace = true tracing-subscriber = "0.3" serde.workspace = true +serde_json.workspace = true serde_yaml_ng.workspace = true hex.workspace = true diff --git a/bin/ethlambda/build.rs b/bin/ethlambda/build.rs index ad4184ed..aea5f542 100644 --- a/bin/ethlambda/build.rs +++ b/bin/ethlambda/build.rs @@ -1,5 +1,13 @@ +use std::path::PathBuf; + use vergen_git2::{Emitter, Git2Builder, RustcBuilder}; +/// Crate names whose resolved git revision is embedded in the binary, one per +/// upstream crypto repository: `leansig` for leanSig, `lean-multisig` for +/// leanVM (the direct dependency `ethlambda-crypto` builds against). +const LEANSIG_PACKAGE: &str = "leansig"; +const LEANVM_PACKAGE: &str = "lean-multisig"; + fn main() -> Result<(), Box> { let git2 = Git2Builder::default().branch(true).sha(true).build()?; let rustc = RustcBuilder::default() @@ -12,5 +20,70 @@ fn main() -> Result<(), Box> { .add_instructions(&git2)? .emit()?; + emit_crypto_revs(); + Ok(()) } + +/// Embed the resolved leanSig and leanVM git revisions from the workspace +/// Cargo.lock. +/// +/// The crypto dependencies are pinned upstream (leansig to a moving branch, +/// leanVM to a rev), so a `cargo update` or a rev bump changes the measured +/// crypto with little or no ethlambda diff; benchmark reports embed these +/// revisions to keep results interpretable across lock bumps. +fn emit_crypto_revs() { + let revs = lockfile_git_revs(); + for (package, env_var) in [ + (LEANSIG_PACKAGE, "ETHLAMBDA_LEANSIG_REV"), + (LEANVM_PACKAGE, "ETHLAMBDA_LEANVM_REV"), + ] { + let rev = revs + .as_ref() + .and_then(|revs| revs.get(package).cloned()) + .unwrap_or_else(|| "unknown".to_string()); + println!("cargo:rustc-env={env_var}={rev}"); + } + if let Some(lockfile) = workspace_lockfile() { + println!("cargo:rerun-if-changed={}", lockfile.display()); + } +} + +fn workspace_lockfile() -> Option { + let manifest_dir = std::env::var("CARGO_MANIFEST_DIR").ok()?; + Some(PathBuf::from(manifest_dir).join("../../Cargo.lock")) +} + +/// Map each git-sourced package in the lockfile to its resolved revision. +/// +/// Both fields of a `[[package]]` block are collected before the revision is +/// extracted, so the result does not depend on TOML field order within the +/// table (a lock-file reformatter emitting `source` before `name` would +/// otherwise silently yield "unknown"). +fn lockfile_git_revs() -> Option> { + let lockfile = std::fs::read_to_string(workspace_lockfile()?).ok()?; + let mut revs = std::collections::HashMap::new(); + // A lockfile is a flat sequence of `[[package]]` blocks; splitting on the + // header gives one chunk per package (the first chunk is the file preamble, + // which has no `name` and is skipped). + for block in lockfile.split("[[package]]") { + let mut name = None; + let mut source = None; + for line in block.lines() { + let line = line.trim(); + if let Some(value) = line.strip_prefix("name = ") { + name = Some(value.trim_matches('"').to_string()); + } else if let Some(value) = line.strip_prefix("source = ") { + source = Some(value.trim_matches('"').to_string()); + } + } + // source = "git+https://github.com/leanEthereum/leanSig?branch=devnet4#" + let (Some(name), Some(source)) = (name, source) else { + continue; + }; + if let Some(rev) = source.strip_prefix("git+").and_then(|s| s.rsplit_once('#')) { + revs.insert(name, rev.1.to_string()); + } + } + Some(revs) +} diff --git a/bin/ethlambda/src/benchmark/corpus.rs b/bin/ethlambda/src/benchmark/corpus.rs new file mode 100644 index 00000000..330c7cc6 --- /dev/null +++ b/bin/ethlambda/src/benchmark/corpus.rs @@ -0,0 +1,145 @@ +//! Synthetic benchmark corpus: deterministic validators, a genesis store, and +//! per-slot attestation-pool seeding. + +use std::sync::Arc; + +use ethlambda_blockchain::store::produce_attestation_data; +use ethlambda_storage::{Store, backend::InMemoryBackend}; +use ethlambda_types::{ + attestation::{AggregationBits, HashedAttestationData}, + block::SingleMessageAggregate, + state::{State, Validator, ValidatorPubkeyBytes}, +}; + +/// Fixed genesis time for synthetic runs. The harness derives every tick +/// timestamp from slot numbers relative to this value and never reads the wall +/// clock, so runs are reproducible at any time of day. +const GENESIS_TIME: u64 = 1_700_000_000; + +pub(crate) struct SyntheticCorpus { + num_validators: u64, + proofs_per_data: u64, +} + +impl SyntheticCorpus { + pub(crate) fn new(num_validators: u64, proofs_per_data: u64) -> Self { + Self { + num_validators, + proofs_per_data, + } + } + + /// Build a genesis store over an in-memory backend with `num_validators` + /// seed-derived validators. + /// + /// Pubkeys are deterministic placeholder bytes: in mock-crypto mode no code + /// path decodes them (signature verification is skipped and best-proof + /// compaction never resolves pubkeys). + pub(crate) fn genesis_store(&self, seed: u64) -> Store { + let mut rng_state = seed; + let validators = (0..self.num_validators) + .map(|index| Validator { + attestation_pubkey: synthetic_pubkey(&mut rng_state), + proposal_pubkey: synthetic_pubkey(&mut rng_state), + index, + }) + .collect(); + let genesis_state = State::from_genesis(GENESIS_TIME, validators); + Store::from_anchor_state(Arc::new(InMemoryBackend::new()), genesis_state) + } + + /// Seed the pending ("new") pool with the full validator set's attestations + /// for `attestation_slot`, split into `proofs_per_data` disjoint aggregates. + /// + /// Mirrors what committee aggregators gossip during a slot: several + /// aggregates for the same `AttestationData`, each covering a validator + /// subset. The proposal tick then promotes them to the known pool, exactly + /// as on a live node. Entries are inserted in a fixed order because pool + /// insertion order pins within-entry proof choice during selection. + pub(crate) fn seed_pool(&self, store: &mut Store, attestation_slot: u64) { + let data = produce_attestation_data(store, attestation_slot); + let entries = participant_groups(self.num_validators, self.proofs_per_data) + .into_iter() + .map(|participants| { + ( + HashedAttestationData::new(data.clone()), + SingleMessageAggregate::empty(participants), + ) + }) + .collect(); + store.insert_new_aggregated_payloads_batch(entries); + } +} + +/// Partition validators 0..num_validators into `groups` disjoint bitfields, +/// assigning validator `i` to group `i % groups`. Every group is non-empty +/// (groups is capped at the validator count) and the union covers every +/// validator exactly once. +fn participant_groups(num_validators: u64, groups: u64) -> Vec { + let groups = groups.clamp(1, num_validators); + (0..groups) + .map(|group| { + let mut bits = AggregationBits::with_length(num_validators as usize) + .expect("validator count is within the bitlist limit"); + for index in (group..num_validators).step_by(groups as usize) { + bits.set(index as usize, true) + .expect("index is within the bitlist length"); + } + bits + }) + .collect() +} + +/// splitmix64: tiny deterministic generator for placeholder pubkey bytes, +/// avoiding a rand dependency. +fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) +} + +fn synthetic_pubkey(rng_state: &mut u64) -> ValidatorPubkeyBytes { + let mut bytes = [0u8; 52]; + for chunk in bytes.chunks_mut(8) { + let word = splitmix64(rng_state).to_le_bytes(); + chunk.copy_from_slice(&word[..chunk.len()]); + } + bytes +} + +#[cfg(test)] +mod tests { + use super::*; + use ethlambda_types::attestation::validator_indices; + + #[test] + fn participant_groups_partition_all_validators() { + for (validators, groups) in [(8u64, 2u64), (8, 3), (5, 8), (1, 1), (4096, 4)] { + let partition = participant_groups(validators, groups); + assert_eq!(partition.len() as u64, groups.min(validators)); + let mut seen = vec![0u32; validators as usize]; + for bits in &partition { + let indices: Vec = validator_indices(bits).collect(); + assert!(!indices.is_empty(), "every group must be non-empty"); + for index in indices { + seen[index as usize] += 1; + } + } + assert!( + seen.iter().all(|&count| count == 1), + "every validator must appear in exactly one group: {seen:?}" + ); + } + } + + #[test] + fn synthetic_pubkeys_are_deterministic() { + let mut a = 42u64; + let mut b = 42u64; + assert_eq!(synthetic_pubkey(&mut a), synthetic_pubkey(&mut b)); + let mut c = 43u64; + assert_ne!(synthetic_pubkey(&mut a), synthetic_pubkey(&mut c)); + } +} diff --git a/bin/ethlambda/src/benchmark/mod.rs b/bin/ethlambda/src/benchmark/mod.rs new file mode 100644 index 00000000..2077fb12 --- /dev/null +++ b/bin/ethlambda/src/benchmark/mod.rs @@ -0,0 +1,289 @@ +//! Offline block-building benchmark (`ethlambda benchmark`). +//! +//! Drives the exact production proposer path β€” `produce_block_with_signatures`, +//! the same entry `BlockChainServer::propose_block` uses β€” against a synthetic +//! in-memory chain, and reports per-phase timing distributions. Gossip publish +//! and the slot-alignment sleep are outside the measured span, matching the +//! node's own `lean_block_building_time_seconds` boundary. +//! +//! See docs/plans/block-building-benchmark.md for the design and roadmap +//! (real-crypto pools and replay-from-datadir land in later milestones). + +mod corpus; +mod report; + +use std::collections::{BTreeMap, HashMap}; +use std::path::PathBuf; +use std::time::Instant; + +use ethlambda_blockchain::block_builder::ProposerConfig; +use ethlambda_blockchain::metrics::BLOCK_PROPOSAL_ATTESTATION_BUILD_PHASES; +use ethlambda_blockchain::store::{on_block_without_verification, produce_block_with_signatures}; +use ethlambda_storage::NEW_PAYLOAD_CAP; +use ethlambda_types::block::{MultiMessageAggregate, SignedBlock}; +use ethlambda_types::primitives::HashTreeRoot as _; +use eyre::WrapErr as _; + +use report::{Environment, Params, Report, Sample}; + +#[derive(Debug, clap::Args)] +pub(crate) struct BenchmarkOptions { + #[command(subcommand)] + workload: Workload, +} + +#[derive(Debug, clap::Subcommand)] +enum Workload { + /// Benchmark block building on a synthetic in-memory chain. + Synthetic(SyntheticOptions), +} + +#[derive(Debug, clap::Args)] +struct SyntheticOptions { + /// Number of validators in the synthetic genesis. + #[arg(long, default_value = "8", value_parser = clap::value_parser!(u64).range(1..=4096))] + num_validators: u64, + /// Unmeasured chain-advancement slots before measuring. Builds and imports + /// one block per slot so the measured builds run on a state with + /// representative historical roots and justifications, and warms the state + /// cache. + #[arg(long, default_value = "8")] + warmup_slots: u64, + /// Aggregate proofs seeded per AttestationData, mimicking committee + /// aggregators covering disjoint validator subsets. The default of 1 (one + /// full-coverage proof per data) keeps justification/finalization + /// advancing every slot. Higher values exercise multi-proof selection and + /// same-data collapse, but without --enable-proposer-aggregation the block + /// then carries only the best partial proof (< 2/3 coverage), so + /// justification stalls β€” the real coverage cost of disabling proposer + /// aggregation. + #[arg(long, default_value = "1", value_parser = clap::value_parser!(u64).range(1..))] + proofs_per_data: u64, + /// Deterministic seed for the synthetic validator set. Two runs with the + /// same seed and parameters produce identical per-iteration block roots. + #[arg(long, default_value = "42")] + seed: u64, + #[command(flatten)] + common: CommonOptions, +} + +#[derive(Debug, clap::Args)] +struct CommonOptions { + /// Measured iterations (one built block each), after warmup. + #[arg(long, default_value = "10", value_parser = clap::value_parser!(u64).range(1..))] + iterations: u64, + /// Seed pools with empty placeholder proofs instead of real XMSS/leanVM + /// crypto. Measures selection + best-proof compaction + state transition + /// only; runs in seconds. Conflicts with --enable-proposer-aggregation, + /// whose recursive aggregation needs real proof bytes. + #[arg(long, conflicts_with = "enable_proposer_aggregation")] + mock_crypto: bool, + /// Mirrors the node flag: collapse same-data proofs via recursive leanVM + /// aggregation instead of keeping the single best-coverage proof. + #[arg(long)] + enable_proposer_aggregation: bool, + /// Mirrors the node flag: distinct AttestationData cap per built block. + #[arg(long, default_value = "3")] + max_attestations_per_block: usize, + /// Report format printed to stdout. Logs go to stderr, so JSON output can + /// be piped directly (e.g. into jq). + #[arg(long, value_enum, default_value_t = OutputFormat::Human)] + format: OutputFormat, + /// Also write the JSON report to this file. + #[arg(long)] + output: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, clap::ValueEnum)] +enum OutputFormat { + Human, + Json, +} + +pub(crate) fn run(options: BenchmarkOptions) -> eyre::Result<()> { + let Workload::Synthetic(synthetic) = options.workload; + run_synthetic(synthetic) +} + +fn run_synthetic(options: SyntheticOptions) -> eyre::Result<()> { + let common = &options.common; + eyre::ensure!( + common.mock_crypto, + "real-crypto benchmarking is not implemented yet; rerun with --mock-crypto" + ); + // The pending pool evicts whole data-root entries FIFO once its proof cap + // is exceeded, so a single slot's batch larger than the cap would silently + // seed nothing and every measured block would be empty. + eyre::ensure!( + options.proofs_per_data as usize <= NEW_PAYLOAD_CAP, + "--proofs-per-data {} exceeds the pending-pool capacity ({NEW_PAYLOAD_CAP}); \ + one slot's batch would be evicted whole and every measured block would be empty", + options.proofs_per_data + ); + + let proposer_config = ProposerConfig { + enable_proposer_aggregation: common.enable_proposer_aggregation, + max_attestations_per_block: common.max_attestations_per_block, + }; + let corpus = corpus::SyntheticCorpus::new(options.num_validators, options.proofs_per_data); + let mut store = corpus.genesis_store(options.seed); + + let total_slots = options + .warmup_slots + .checked_add(common.iterations) + .ok_or_else(|| eyre::eyre!("--warmup-slots plus --iterations overflows u64"))?; + let mut samples = Vec::with_capacity(common.iterations as usize); + for slot in 1..=total_slots { + // Seed the pending pool with the previous slot's attestations, exactly + // where gossip aggregates would sit before the proposal tick promotes + // them to the known pool. Entries from earlier slots stay in the known + // pool, as they would on a live node. + corpus.seed_pool(&mut store, slot - 1); + eyre::ensure!( + store.new_aggregated_payloads_count() > 0, + "seeded attestations were evicted from the pending pool at slot {slot}; \ + the measured workload would not match the requested parameters" + ); + let pool_entries = + store.new_aggregated_payloads_count() + store.known_aggregated_payloads_count(); + + // Round-robin proposer, matching `is_proposer`. + let proposer = slot % options.num_validators; + + let before = phase_snapshot(); + let build_start = Instant::now(); + let (block, aggregates, _checkpoints) = + produce_block_with_signatures(&mut store, slot, proposer, proposer_config) + .wrap_err_with(|| format!("block build failed at slot {slot}"))?; + let wall_seconds = build_start.elapsed().as_secs_f64(); + let phases = phase_deltas(&before, &phase_snapshot())?; + + let block_root = block.hash_tree_root(); + let attestations_packed = block.body.attestations.len(); + let aggregates_count = aggregates.len(); + + // Import the built block (outside the measured span) so the next + // iteration builds one slot ahead of head, like a live proposer; + // building repeatedly on a fixed head would make `process_slots` cost + // grow with the iteration index. + let signed_block = SignedBlock { + message: block, + proof: MultiMessageAggregate::default(), + }; + on_block_without_verification(&mut store, signed_block) + .wrap_err_with(|| format!("importing the built block failed at slot {slot}"))?; + + let measured = slot > options.warmup_slots; + let label = if measured { "measured" } else { "warmup" }; + eprintln!( + "[{slot}/{total_slots}] {label}: built block in {:.3}ms \ + (attestations={attestations_packed}, pool_entries={pool_entries})", + wall_seconds * 1e3, + ); + + if measured { + // Clamped: the unattributed preamble makes the remainder + // positive in practice, but summing many small phase values can + // round just above the wall measurement, and a negative overhead + // in the report would read as an accounting bug. + let overhead_seconds = (wall_seconds - phases.values().sum::()).max(0.0); + samples.push(Sample { + iteration: slot - options.warmup_slots, + slot, + proposer, + block_root: format!("0x{}", hex::encode(block_root.0)), + wall_seconds, + phases, + overhead_seconds, + attestations_packed, + aggregates: aggregates_count, + pool_entries, + }); + } + } + + eyre::ensure!( + samples.len() as u64 == common.iterations, + "collected {} samples but expected {}; the measured-slot accounting drifted", + samples.len(), + common.iterations + ); + + let params = Params { + mode: "synthetic", + mock_crypto: common.mock_crypto, + num_validators: options.num_validators, + warmup_slots: options.warmup_slots, + proofs_per_data: options.proofs_per_data, + seed: options.seed, + iterations: common.iterations, + enable_proposer_aggregation: common.enable_proposer_aggregation, + max_attestations_per_block: common.max_attestations_per_block, + }; + let report = Report::new(Environment::collect(), params, samples); + + match common.format { + OutputFormat::Human => println!("{}", report.human_table()), + OutputFormat::Json => println!("{}", report.to_json()?), + } + if let Some(path) = &common.output { + std::fs::write(path, report.to_json()?) + .wrap_err_with(|| format!("failed to write report to {}", path.display()))?; + eprintln!("report written to {}", path.display()); + } + + Ok(()) +} + +const PHASE_HISTOGRAM: &str = "lean_block_proposal_attestation_build_phase_seconds"; + +/// Per-phase (sample_sum, sample_count) snapshot of the block-proposal phase +/// histogram, read from the default prometheus registry. +type PhaseSnapshot = HashMap; + +fn phase_snapshot() -> PhaseSnapshot { + ethlambda_metrics::gather() + .iter() + .filter(|family| family.name() == PHASE_HISTOGRAM) + .flat_map(|family| family.get_metric()) + .filter_map(|metric| { + let phase = metric + .get_label() + .iter() + .find(|label| label.name() == "phase")? + .value() + .to_string(); + let histogram = metric.get_histogram(); + Some(( + phase, + (histogram.get_sample_sum(), histogram.get_sample_count()), + )) + }) + .collect() +} + +/// Exact per-iteration phase durations from two snapshots around one build. +/// +/// Histogram sums accumulate the raw f64 seconds of every observation, so the +/// sum delta IS the build's phase time β€” bucket boundaries play no role. The +/// count must advance by exactly 1 per phase (each phase observes once per +/// `build_block` in this single-threaded process); anything else means the +/// accounting drifted and attribution would be wrong, so it is a hard error. +fn phase_deltas( + before: &PhaseSnapshot, + after: &PhaseSnapshot, +) -> eyre::Result> { + let mut deltas = BTreeMap::new(); + for &phase in BLOCK_PROPOSAL_ATTESTATION_BUILD_PHASES { + let (sum_before, count_before) = before.get(phase).copied().unwrap_or((0.0, 0)); + let (sum_after, count_after) = after.get(phase).copied().unwrap_or((0.0, 0)); + let observations = count_after.saturating_sub(count_before); + eyre::ensure!( + observations == 1, + "phase '{phase}' was observed {observations} times during one build (expected 1); \ + phase attribution would be wrong" + ); + deltas.insert(phase.to_string(), sum_after - sum_before); + } + Ok(deltas) +} diff --git a/bin/ethlambda/src/benchmark/report.rs b/bin/ethlambda/src/benchmark/report.rs new file mode 100644 index 00000000..e3302f88 --- /dev/null +++ b/bin/ethlambda/src/benchmark/report.rs @@ -0,0 +1,312 @@ +//! Statistics and report emission for the block-building benchmark. +//! +//! Raw per-iteration samples are always included in the JSON report: outliers +//! are never discarded (XMSS signing and OTS window advancement produce +//! legitimate heavy tails worth inspecting), and per-iteration block roots let +//! a baseline-vs-optimized diff prove an optimization changed only speed, not +//! which attestations get selected. + +use std::collections::BTreeMap; +use std::fmt::Write as _; + +use serde::Serialize; + +use crate::version; + +/// Coefficient-of-variation threshold above which wall-time results are +/// flagged as too noisy to compare, per the benchmarking workflow standard. +const CV_WARN_THRESHOLD: f64 = 0.10; + +#[derive(Debug, Serialize)] +pub(crate) struct Sample { + pub iteration: u64, + pub slot: u64, + pub proposer: u64, + /// Determinism checksum: same seed + params must reproduce the same roots. + pub block_root: String, + pub wall_seconds: f64, + /// Per-phase seconds from histogram sum deltas. + pub phases: BTreeMap, + /// Wall time not attributed to any phase: the `produce_block_with_signatures` + /// preamble (tick advance, pool promotion, fork-choice head update, pool + /// deep-clone, block-roots scan) plus measurement slack. + pub overhead_seconds: f64, + pub attestations_packed: usize, + pub aggregates: usize, + /// Pool entries (new + known) visible to this build; reported so pool + /// growth across iterations is visible in the samples. + pub pool_entries: usize, +} + +#[derive(Debug, Serialize)] +pub(crate) struct Environment { + pub client_version: &'static str, + /// Resolved leansig git revision from Cargo.lock. leansig is pinned to a + /// moving branch, so results are not comparable across revisions. + pub leansig_rev: &'static str, + /// Resolved leanVM git revision from Cargo.lock. leanVM does the signature + /// aggregation, so a rev bump moves the measured crypto too. + pub leanvm_rev: &'static str, + pub os: &'static str, + pub arch: &'static str, + pub available_parallelism: usize, +} + +impl Environment { + pub(crate) fn collect() -> Self { + Self { + client_version: version::CLIENT_VERSION, + leansig_rev: env!("ETHLAMBDA_LEANSIG_REV"), + leanvm_rev: env!("ETHLAMBDA_LEANVM_REV"), + os: std::env::consts::OS, + arch: std::env::consts::ARCH, + available_parallelism: std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(0), + } + } +} + +#[derive(Debug, Serialize)] +pub(crate) struct Params { + pub mode: &'static str, + pub mock_crypto: bool, + pub num_validators: u64, + pub warmup_slots: u64, + pub proofs_per_data: u64, + pub seed: u64, + pub iterations: u64, + pub enable_proposer_aggregation: bool, + pub max_attestations_per_block: usize, +} + +#[derive(Debug, Serialize)] +pub(crate) struct Stats { + pub count: usize, + pub min_seconds: f64, + pub mean_seconds: f64, + pub p50_seconds: f64, + pub p90_seconds: f64, + pub max_seconds: f64, + /// Coefficient of variation (stddev / mean); NaN-free (0 when mean is 0). + pub cv: f64, +} + +#[derive(Debug, Serialize)] +pub(crate) struct Summary { + pub phases: BTreeMap, + pub overhead: Stats, + pub wall: Stats, +} + +#[derive(Debug, Serialize)] +pub(crate) struct Report { + pub schema_version: u32, + pub environment: Environment, + pub params: Params, + pub samples: Vec, + pub summary: Summary, +} + +impl Report { + pub(crate) fn new(environment: Environment, params: Params, samples: Vec) -> Self { + let mut phases: BTreeMap = BTreeMap::new(); + if let Some(first) = samples.first() { + for phase in first.phases.keys() { + let values: Vec = samples + .iter() + .filter_map(|sample| sample.phases.get(phase).copied()) + .collect(); + phases.insert(phase.clone(), stats(&values)); + } + } + let overhead = stats( + &samples + .iter() + .map(|sample| sample.overhead_seconds) + .collect::>(), + ); + let wall = stats( + &samples + .iter() + .map(|sample| sample.wall_seconds) + .collect::>(), + ); + + if wall.cv > CV_WARN_THRESHOLD { + eprintln!( + "warning: wall-time coefficient of variation is {:.1}% (>{:.0}%); \ + results are noisy β€” check for background load or increase --iterations", + wall.cv * 100.0, + CV_WARN_THRESHOLD * 100.0 + ); + } + + Self { + schema_version: 1, + environment, + params, + samples, + summary: Summary { + phases, + overhead, + wall, + }, + } + } + + pub(crate) fn to_json(&self) -> eyre::Result { + serde_json::to_string_pretty(self).map_err(Into::into) + } + + pub(crate) fn human_table(&self) -> String { + let mut out = String::new(); + let params = &self.params; + let env = &self.environment; + let crypto = if params.mock_crypto { "mock" } else { "real" }; + let _ = writeln!( + out, + "Block-building benchmark β€” {} workload ({crypto} crypto)", + params.mode + ); + let _ = writeln!( + out, + " validators={} warmup_slots={} iterations={} proofs_per_data={} seed={}", + params.num_validators, + params.warmup_slots, + params.iterations, + params.proofs_per_data, + params.seed + ); + let _ = writeln!( + out, + " enable_proposer_aggregation={} max_attestations_per_block={}", + params.enable_proposer_aggregation, params.max_attestations_per_block + ); + let _ = writeln!( + out, + " {} leansig={} leanvm={} os={} arch={} threads={}", + env.client_version, + env.leansig_rev, + env.leanvm_rev, + env.os, + env.arch, + env.available_parallelism + ); + let _ = writeln!(out); + let _ = writeln!( + out, + " {:<18} {:>5} {:>10} {:>10} {:>10} {:>10} {:>10}", + "phase", "count", "min", "mean", "p50", "p90", "max" + ); + for (phase, stats) in &self.summary.phases { + let _ = writeln!(out, "{}", stats_row(phase, stats)); + } + let _ = writeln!(out, "{}", stats_row("overhead", &self.summary.overhead)); + let _ = writeln!(out, "{}", stats_row("wall", &self.summary.wall)); + out + } +} + +fn stats_row(name: &str, stats: &Stats) -> String { + format!( + " {:<18} {:>5} {:>10} {:>10} {:>10} {:>10} {:>10}", + name, + stats.count, + format_ms(stats.min_seconds), + format_ms(stats.mean_seconds), + format_ms(stats.p50_seconds), + format_ms(stats.p90_seconds), + format_ms(stats.max_seconds), + ) +} + +fn format_ms(seconds: f64) -> String { + format!("{:.3}ms", seconds * 1e3) +} + +fn stats(values: &[f64]) -> Stats { + if values.is_empty() { + return Stats { + count: 0, + min_seconds: 0.0, + mean_seconds: 0.0, + p50_seconds: 0.0, + p90_seconds: 0.0, + max_seconds: 0.0, + cv: 0.0, + }; + } + let mut sorted = values.to_vec(); + sorted.sort_by(|a, b| a.total_cmp(b)); + let count = sorted.len(); + let mean = sorted.iter().sum::() / count as f64; + let variance = sorted + .iter() + .map(|value| (value - mean).powi(2)) + .sum::() + / count as f64; + let cv = if mean > 0.0 { + variance.sqrt() / mean + } else { + 0.0 + }; + Stats { + count, + min_seconds: sorted[0], + mean_seconds: mean, + p50_seconds: percentile(&sorted, 0.50), + p90_seconds: percentile(&sorted, 0.90), + max_seconds: sorted[count - 1], + cv, + } +} + +/// Nearest-rank percentile over a sorted slice (no interpolation; sample +/// counts are small so exact sample values are preferable to blends). +fn percentile(sorted: &[f64], q: f64) -> f64 { + let index = ((sorted.len() - 1) as f64 * q).round() as usize; + sorted[index] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn percentile_handles_single_sample() { + let sorted = [7.0]; + assert_eq!(percentile(&sorted, 0.0), 7.0); + assert_eq!(percentile(&sorted, 0.5), 7.0); + assert_eq!(percentile(&sorted, 1.0), 7.0); + } + + #[test] + fn percentile_odd_and_even_lengths() { + let odd = [1.0, 2.0, 3.0, 4.0, 5.0]; + assert_eq!(percentile(&odd, 0.5), 3.0); + assert_eq!(percentile(&odd, 1.0), 5.0); + let even = [1.0, 2.0, 3.0, 4.0]; + assert_eq!(percentile(&even, 0.5), 3.0); + assert_eq!(percentile(&even, 0.0), 1.0); + } + + #[test] + fn stats_on_known_values() { + let stats = stats(&[2.0, 4.0, 4.0, 4.0, 5.0, 5.0, 7.0, 9.0]); + assert_eq!(stats.count, 8); + assert_eq!(stats.min_seconds, 2.0); + assert_eq!(stats.max_seconds, 9.0); + assert_eq!(stats.mean_seconds, 5.0); + // population stddev of this classic set is 2.0 => cv = 0.4 + assert!((stats.cv - 0.4).abs() < 1e-12); + } + + #[test] + fn stats_on_empty_input_is_zeroed() { + let stats = stats(&[]); + assert_eq!(stats.count, 0); + assert_eq!(stats.mean_seconds, 0.0); + assert_eq!(stats.cv, 0.0); + } +} diff --git a/bin/ethlambda/src/command.rs b/bin/ethlambda/src/command.rs index ec806561..7a663182 100644 --- a/bin/ethlambda/src/command.rs +++ b/bin/ethlambda/src/command.rs @@ -9,41 +9,84 @@ //! same arguments it saw before this module existed. Its error messages, exit //! codes and `--version` output are therefore unchanged by construction rather //! than by test; only `--help` differs, by the [`HELP_NOTE`] it appends. +//! +//! `benchmark` parses through [`BenchmarkCommand`] instead, so the harness +//! arguments stay out of `CliOptions` entirely: nothing the benchmark needs can +//! reshape the parser the node depends on. use std::ffi::OsString; use clap::Parser; +use crate::benchmark::BenchmarkOptions; use crate::cli::CliOptions; -/// The sub-command token accepted in first position. +/// The sub-command tokens accepted in first position. /// /// `CliOptions` declares no positional arguments, so the first token after the -/// program name is either a flag or this sub-command: a flag *value* never -/// lands there and is never mistaken for it. +/// program name is either a flag or one of these: a flag *value* never lands +/// there and is never mistaken for a sub-command. const NODE: &str = "node"; +const BENCHMARK: &str = "benchmark"; + +/// Appended to `--help` by `CliOptions`. The tokens never reach clap, so +/// without this the sub-commands would be undiscoverable from the help output. +pub(crate) const HELP_NOTE: &str = "Sub-commands:\n \ + node Run the consensus node (assumed when omitted)\n \ + benchmark Benchmark block building offline \ + (see `ethlambda benchmark --help`)"; + +/// What the command line asked the binary to do. +/// +/// `Node` is ~312 bytes against `Benchmark`'s ~80, but exactly one of these is +/// built per process and consumed immediately by `main`, so boxing would buy an +/// allocation and nothing else. +#[derive(Debug)] +#[allow(clippy::large_enum_variant)] +pub(crate) enum Invocation { + /// Run the consensus node. + Node(CliOptions), + /// Run the offline block-building benchmark. + Benchmark(BenchmarkOptions), +} -/// Appended to `--help` by `CliOptions`. The token never reaches clap, so -/// without this the sub-command would be undiscoverable from the help output. -pub(crate) const HELP_NOTE: &str = "Sub-commands:\n node \ - Run the consensus node (assumed when omitted)"; +/// The `benchmark` sub-command, parsed on its own so that its arguments never +/// enter `CliOptions`. +#[derive(Debug, clap::Parser)] +#[command(about = "Benchmark block building offline against a controlled workload")] +struct BenchmarkCommand { + #[command(flatten)] + options: BenchmarkOptions, +} /// Parse the process arguments, exiting the way clap does on a parse error, /// `--help` or `--version`. -pub(crate) fn parse() -> CliOptions { +pub(crate) fn parse() -> Invocation { try_parse_from(std::env::args_os()).unwrap_or_else(|err| err.exit()) } -fn try_parse_from(args: I) -> Result +fn try_parse_from(args: I) -> Result where I: IntoIterator, I::Item: Into, { let mut args: Vec = args.into_iter().map(Into::into).collect(); - if args.get(1).is_some_and(|arg| arg == NODE) { - args.remove(1); + match args.get(1).and_then(|arg| arg.to_str()) { + Some(NODE) => { + args.remove(1); + CliOptions::try_parse_from(args).map(Invocation::Node) + } + Some(BENCHMARK) => { + // clap renders usage from argv[0], so the two tokens collapse into + // one program name and its usage lines read `ethlambda benchmark + // ` rather than dropping the sub-command they belong to. + args.drain(..2); + let argv = std::iter::once(OsString::from("ethlambda benchmark")).chain(args); + BenchmarkCommand::try_parse_from(argv).map(|cmd| Invocation::Benchmark(cmd.options)) + } + // No sub-command named: the flat node form, byte for byte as before. + _ => CliOptions::try_parse_from(args).map(Invocation::Node), } - CliOptions::try_parse_from(args) } #[cfg(test)] @@ -85,7 +128,12 @@ mod tests { } fn node_options(args: &[&str]) -> CliOptions { - try_parse_from(args.iter().map(OsString::from)).expect("invocation parses") + let invocation = + try_parse_from(args.iter().map(OsString::from)).expect("invocation parses"); + match invocation { + Invocation::Node(options) => options, + other => panic!("expected a node invocation, got {other:?}"), + } } #[test] @@ -185,9 +233,53 @@ mod tests { } #[test] - fn help_documents_the_node_sub_command() { + fn help_documents_the_sub_commands() { let err = try_parse_from(["ethlambda", "--help"].iter().map(OsString::from)) .expect_err("--help short-circuits parsing"); - assert!(err.to_string().contains(NODE), "{err}"); + let help = err.to_string(); + assert!(help.contains(NODE), "{help}"); + assert!(help.contains(BENCHMARK), "{help}"); + } + + #[test] + fn benchmark_parses_without_any_node_argument() { + // The point of parsing the harness separately: none of --genesis, + // --validators, --node-key … is required, or even accepted, here. + let args = ["ethlambda", BENCHMARK, "synthetic", "--num-validators", "4"]; + let invocation = try_parse_from(args.iter().map(OsString::from)).expect("benchmark parses"); + let Invocation::Benchmark(options) = invocation else { + panic!("expected a benchmark invocation, got {invocation:?}"); + }; + assert!( + format!("{options:?}").contains("num_validators: 4"), + "{options:?}" + ); + } + + #[test] + fn benchmark_rejects_node_arguments() { + let args = [ + "ethlambda", + BENCHMARK, + "synthetic", + "--genesis", + "config.yaml", + ]; + let err = try_parse_from(args.iter().map(OsString::from)) + .expect_err("node flags are not benchmark flags"); + assert_eq!(err.kind(), ErrorKind::UnknownArgument); + } + + #[test] + fn benchmark_usage_names_the_sub_command() { + // The token never reaches clap, so without `name` the usage line would + // read `ethlambda synthetic …` and mislead. + let err = try_parse_from( + ["ethlambda", BENCHMARK, "--help"] + .iter() + .map(OsString::from), + ) + .expect_err("--help short-circuits parsing"); + assert!(err.to_string().contains("ethlambda benchmark"), "{err}"); } } diff --git a/bin/ethlambda/src/main.rs b/bin/ethlambda/src/main.rs index 6638a21d..a6a71fef 100644 --- a/bin/ethlambda/src/main.rs +++ b/bin/ethlambda/src/main.rs @@ -1,3 +1,4 @@ +mod benchmark; mod checkpoint_sync; mod cli; mod command; @@ -32,6 +33,8 @@ use std::{ }; use tokio_util::sync::CancellationToken; +use cli::CliOptions; +use command::Invocation; use ethlambda_blockchain::MILLISECONDS_PER_SLOT; use ethlambda_blockchain::block_builder::ProposerConfig; use ethlambda_blockchain::key_manager::ValidatorKeyPair; @@ -65,21 +68,54 @@ const ASCII_ART: &str = r#" \___|\__|_| |_|_|\__,_|_| |_| |_|_.__/ \__,_|\__,_| "#; -// Shadow single-steps execution in a discrete-event simulation, so the default -// multi-threaded runtime's worker threads add only scheduling noise, never -// parallelism. Use a single-threaded runtime under Shadow. This is an -// optimization, not a correctness requirement. -#[cfg_attr(not(feature = "shadow-integration"), tokio::main)] -#[cfg_attr(feature = "shadow-integration", tokio::main(flavor = "current_thread"))] -async fn main() -> eyre::Result<()> { +fn main() -> eyre::Result<()> { + match command::parse() { + Invocation::Node(options) => { + init_node_logging()?; + run_node(options) + } + // The benchmark is synchronous, CPU-bound work, so it runs on this + // thread and the tokio runtime is never started β€” rather than parking + // a worker thread for the whole run. + Invocation::Benchmark(options) => { + init_benchmark_logging()?; + benchmark::run(options) + } + } +} + +/// Node logging: INFO and above, on stdout. +fn init_node_logging() -> eyre::Result<()> { let filter = EnvFilter::builder() .with_default_directive(tracing::Level::INFO.into()) .from_env_lossy(); let subscriber = Registry::default().with(tracing_subscriber::fmt::layer().with_filter(filter)); tracing::subscriber::set_global_default(subscriber) - .wrap_err("failed to set global tracing subscriber")?; + .wrap_err("failed to set global tracing subscriber") +} + +/// Benchmark logging: WARN and above, on stderr, so that the report owns stdout +/// and stays pipe-clean for `--format json | jq`. +fn init_benchmark_logging() -> eyre::Result<()> { + let filter = EnvFilter::builder() + .with_default_directive(tracing::Level::WARN.into()) + .from_env_lossy(); + let subscriber = Registry::default().with( + tracing_subscriber::fmt::layer() + .with_writer(std::io::stderr) + .with_filter(filter), + ); + tracing::subscriber::set_global_default(subscriber) + .wrap_err("failed to set global tracing subscriber") +} - let options = command::parse(); +// Shadow single-steps execution in a discrete-event simulation, so the default +// multi-threaded runtime's worker threads add only scheduling noise, never +// parallelism. Use a single-threaded runtime under Shadow. This is an +// optimization, not a correctness requirement. +#[cfg_attr(not(feature = "shadow-integration"), tokio::main)] +#[cfg_attr(feature = "shadow-integration", tokio::main(flavor = "current_thread"))] +async fn run_node(options: CliOptions) -> eyre::Result<()> { options.validate_discovery()?; #[cfg(feature = "shadow-integration")] diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index ffd10301..95da4df6 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -8,4 +8,6 @@ pub use api::{ALL_TABLES, StorageBackend, StorageReadView, StorageWriteBatch, Ta /// Error type returned by the fallible [`Store`] operations, exported so /// callers can match on it (e.g. to distinguish [`Error::GenesisMismatch`]). pub use error::Error; -pub use store::{ForkCheckpoints, GetForkchoiceStoreError, MAX_RESUMABLE_DB_STATE_AGE, Store}; +pub use store::{ + ForkCheckpoints, GetForkchoiceStoreError, MAX_RESUMABLE_DB_STATE_AGE, NEW_PAYLOAD_CAP, Store, +}; diff --git a/crates/storage/src/store.rs b/crates/storage/src/store.rs index 05621f8a..914b2cd1 100644 --- a/crates/storage/src/store.rs +++ b/crates/storage/src/store.rs @@ -125,7 +125,9 @@ const AGGREGATED_PAYLOAD_CAP: usize = 512; /// Hard cap for the new (pending) aggregated payload buffer. /// Smaller than known since new payloads are drained every interval (~4s). -const NEW_PAYLOAD_CAP: usize = 64; +/// Public so pool-seeding callers (the block-building benchmark) can reject +/// workloads that a single insertion batch would silently evict. +pub const NEW_PAYLOAD_CAP: usize = 64; /// Hard cap for the gossip signature buffer (individual signatures, not distinct data_roots). /// With 4 validators and 4-second slots, 2048 signatures covers ~512 slots (~34 min). diff --git a/docs/plans/block-building-benchmark.md b/docs/plans/block-building-benchmark.md new file mode 100644 index 00000000..090b37b8 --- /dev/null +++ b/docs/plans/block-building-benchmark.md @@ -0,0 +1,159 @@ +# Plan: `ethlambda benchmark` β€” offline block-building benchmark sub-command + +## Context + +The README roadmap lists **"Optimize block building" (issue #465)** as the top near-term +priority, but block building is only observable today through Prometheus histograms on a +live devnet β€” there is no reproducible, offline way to measure it or to compare an +optimization against a baseline. This adds an `ethlambda benchmark` sub-command that +drives the exact production proposer code path against controlled workloads. + +Fixed scope decisions: offline harness; synthetic **and** replay-from-datadir workloads; +real XMSS/leanVM crypto by default with a mock fast mode. + +## What gets measured + +The proposer pipeline as executed at interval 4, entered through the same functions the +actor calls: + +``` +produce_block_with_signatures (crates/blockchain/src/store.rs:788) ← already public + β”œβ”€ preamble: on_tick β†’ interval 0, promote attestations, + β”‚ fork-choice head, pool deep-clone β†’ reported as derived "build_overhead" + └─ build_block: select_payloads β†’ compact β†’ stf_simulate +seal_block (extracted from crates/blockchain/src/lib.rs:504-631, see refactor) + └─ sign β†’ wrap_proposer_type1 (leanVM) β†’ merge_type_2 (leanVM) +``` + +**Excluded** (same boundary as the node's own `time_block_building` metric): gossip +publish, slot-alignment sleep, block import. + +**Phase capture with zero hot-path changes**: the existing +`lean_block_proposal_attestation_build_phase_seconds` HistogramVec accumulates exact f64 +sums, observed exactly once per phase per build β€” the harness deltas per-label sums +between iterations (prometheus 0.14 exposes `get_sample_sum()`, readable in-process). +Guards: assert per-phase count advanced by exactly 1, and warn if `wall βˆ’ Ξ£phases` +exceeds 2%. + +**Statistics**: warmup 3 + 10 iterations (defaults, configurable); min/mean/p50/p90/max + +CV>10% warning per phase; raw samples always exported; outliers never auto-discarded +(XMSS rejection-sampling and OTS window advancement produce legitimate tails). Each +iteration records `block.hash_tree_root()` β€” diffing root sequences between baseline and +optimized runs proves an optimization changed only speed, not attestation selection. + +## CLI (verified on clap 4.6.1) + +Every existing flat invocation (devnet skills, Dockerfile, lean-quickstart) parses +byte-for-byte unchanged. + +- `command.rs` (new) owns dispatch, and `cli.rs` keeps the exact shape it had: a leading + `node` or `benchmark` token is removed before parsing, and the untouched `CliOptions` + parser then sees the very same arguments as before for every other form. Its seven + required arguments stay plain `PathBuf`/`String`, so clap's own missing-argument errors + are preserved without an `Option` to unwrap anywhere. + + The rejected alternative was clap's `subcommand_negates_reqs` + + `args_conflicts_with_subcommands` with `command: Option` on `CliOptions`. It + works, but forces all seven required arguments to `Option` β€” `negates_reqs` lifts + only the *requirement check*, while the derive still fails extracting a non-`Option` + field that the command line never supplied β€” which means an unwrap helper on the node + path for an invariant clap already enforces. Reviewers pushed back on that churn in + #497, and it buys nothing the token dispatch does not. +- `benchmark` parses through its own `clap::Parser` (`BenchmarkCommand`), so harness + arguments never enter `CliOptions` at all. Its argv[0] is rewritten to + `ethlambda benchmark` so usage lines name the sub-command that owns them. +- Because the tokens never reach clap, they would be absent from `--help`; `cli.rs` + carries one `after_help` line listing both, sourced from a const `command.rs` owns. +- `main.rs`: `main` is synchronous and matches on the invocation. The node path keeps the + `#[tokio::main]` attributes on `run_node`; the benchmark runs on the main thread and + never starts the runtime. + +``` +ethlambda benchmark synthetic --num-validators 8 --warmup-slots 8 + --proofs-per-data 1 --seed 42 [--key-cache-dir ] # cache: M2 +ethlambda benchmark replay --data-dir --genesis config.yaml [--no-copy] + [--validators … --hash-sig-keys-dir … --node-id …] # enables seal +common: --iterations 10 --mock-crypto --enable-proposer-aggregation + --max-attestations-per-block 3 --format human|json --output +``` + +Implementation refinements (M1): there is no `--pool-datas` knob β€” the pool +accumulates one distinct `AttestationData` per elapsed slot naturally, exactly +as on a live node, and per-sample `pool_entries` makes the growth visible. +`--proofs-per-data` defaults to 1 (a single full-coverage aggregate per data, +what a committee aggregator emits) so justification/finalization advance every +slot; higher values exercise multi-proof selection but stall justification +without proposer aggregation β€” the real coverage cost of that node flag. +Warmup slots double as chain advancement, so there is no separate warmup- +iterations knob. + +Known pre-existing issue (unrelated): `lean-quickstart/client-cmds/ethlambda-cmd.sh` +still uses `--custom-network-config-dir`, removed in #321 β€” needs an upstream fix. + +## Harness design (`bin/ethlambda/src/benchmark/{mod,keys,corpus,report}.rs`) + +- **Iteration model**: slots advance monotonically, proposer rotates `slot % N` (matches + round-robin `is_proposer`); each built block is imported via + `on_block_without_verification` so the empty-slot gap stays constant; the pool is + re-seeded per iteration in fixed seeded order (insertion order pins proof choice). +- **Keys**: seeded in-process keygen, cached on disk keyed by (leansig rev, seed, index, + role). Minimal-window keygen costs ~1s/key in release (verified empirically; the window + floors at 131,072 epochs β€” ample for thousands of bench slots; the 2^32 lifetime is + fixed in the type and unaffected). Arbitrary N, no Docker, no fixture download. +- **Synthetic corpus**: `State::from_genesis` + `InMemoryBackend`; K warmup blocks; pool + = attestations from the last `--pool-datas` slots Γ— `--proofs-per-data` real type-1 + proofs via `aggregate_signatures` (built outside the timed span, progress on stderr). +- **`--mock-crypto`**: empty proofs, forces the `keep_best` path (clap `conflicts_with + --enable-proposer-aggregation`, since `compact` invokes the real prover), seal skipped + and reported as null-not-zero. Runs in seconds β†’ CI smoke test. +- **Replay (v1 scope)**: copies the datadir before opening (mandatory β€” `on_tick`/head + updates write Metadata per interval and RocksDB has no read-only mode; `--no-copy` + opt-out with a warning). Loads via `Store::from_db_state`, builds at head+1. Pools are + in-memory-only and unrecoverable from disk, so v1 replay measures selection + STF + + state-root realism on real deep states; supplying the node's key trio additionally + enables the seal phases. Type-2 splitting / pool recording = deferred future work. +- **Report**: human table + `--format json` (stdout pipe-clean, logs to stderr) with + `schema_version`, environment (CPU model, cores, OS, ethlambda rev via vergen, leansig + lock rev via a small `build.rs` Cargo.lock parse β€” leansig tracks the moving `devnet4` + branch), full params + seed, per-iteration raw samples. One configuration per process + invocation (global cumulative histograms, rayon/prover state). + +## The one library refactor + +Extract `crates/blockchain/src/lib.rs:504-631` (proposer sign β†’ type-1 wrap β†’ pubkey +resolution β†’ type-2 merge) into `pub fn seal_block(...) -> Result` in the blockchain crate; `propose_block` calls it. Justified: the +benchmark cannot reach these phases otherwise (a bin-side copy would drift), it collapses +six repeated error-return-with-metric blocks into one `match` (net-negative LOC), and +adding `sign`/`wrap_proposer_type1`/`merge_type_2` labels to the existing phase histogram +gives production dashboards the currently-untimed expensive steps issue #465 targets. +Verbatim move, own commit, devnet smoke before merge. `build_block` stays `pub(crate)`. + +## Milestones + +| | Deliverable | Files | +|---|---|---| +| **M1** β€” CLI + mock end-to-end | `ethlambda benchmark synthetic --mock-crypto` runs in seconds; table + JSON; flat-invocation compat tests; `make bench`; CI smoke step in the existing Test job. Includes one small library fix found by the determinism gate: `extend_proofs_greedily` kept its candidate set in a `HashSet`, so equal-coverage proof ties were broken by randomized hash order and block contents differed run to run β€” ties now break to the lowest pool index | `cli.rs`, `main.rs`, `benchmark/{mod,corpus,report}.rs`, `build.rs` (leansig rev), `Makefile`, `ci.yml`, `block_builder.rs` (tie-break) | +| **M2** β€” real crypto | `seal_block` extraction (first commit) + 3 new phase labels; seeded keygen + cache; real type-1 pools; all 7 phases measured; first baseline JSON recorded | `crates/blockchain/src/{seal.rs,lib.rs,metrics.rs}`, `benchmark/keys.rs`, `types/src/signature.rs` (keygen wrapper) | +| **M3** β€” replay + docs | replay mode against a devnet-runner datadir; `docs/benchmarking.md` + `SUMMARY.md` + README roadmap line | `benchmark/corpus.rs`, docs | + +One PR per milestone; `make fmt/lint/test` before each; M2 additionally gated by a devnet +smoke via `test-branch.sh`. + +## Verification + +- clap `try_parse_from` tests: flat invocation parses, missing-arg errors preserved, + `benchmark` parses without node args, mixed invocation rejected. +- Determinism: two same-seed runs produce identical per-iteration block-root sequences. +- Accounting: Ξ£phases β‰₯ 98% of wall per iteration, per-phase count deltas == 1. +- CI mock smoke: `benchmark synthetic --mock-crypto --num-validators 4 --iterations 3 + --format json | jq -e '.schema_version == 1'`. + +## Main risks + +- Real-mode setup cost: iterations Γ— pool proofs of leanVM proving β†’ default real run + takes minutes (mitigated: mock mode, small defaults, ETA logging, key cache). +- `seal_block` extraction touches consensus-critical `propose_block` β€” verbatim + extraction, careful review of the six error branches, devnet smoke. +- Cross-run comparability: rayon-parallel proving is machine/load-sensitive and leansig + is a moving branch β€” the env block in every report is the guard, not a fix.