diff --git a/.env.example b/.env.example index 38c9f51ba..532c66f00 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,11 @@ # Block production interval (seconds). Lower = faster blocks, higher CPU. CONSENSUS_TIME=10 +# Experimental POC: replace each block recipient's all-peer sync-status +# rebroadcast with one bounded aggregate from the block sender. Keep disabled +# outside an isolated devnet until convergence and adversarial tests pass. +BLOCK_SYNC_AGGREGATION_ENABLED=false + # Genesis-state network parameters. Governance proposals can override these # at runtime; these values are the chain's bootstrap defaults. # Total per-tx flat fee = RPC_FEE + NETWORK_FEE + BURN_FEE (default 1+1+1=3). diff --git a/.gitignore b/.gitignore index ba39b1a41..35a6afa73 100644 --- a/.gitignore +++ b/.gitignore @@ -89,6 +89,7 @@ testing/devnet/identities/ testing/devnet/.env testing/devnet/postgres-data/ testing/runs/ +.poc-results/ local_tests/ # ---- Local devnet identities (private keys / mnemonics) ---- diff --git a/docs/poc/block-sync-aggregation.md b/docs/poc/block-sync-aggregation.md new file mode 100644 index 000000000..c5ec8d78b --- /dev/null +++ b/docs/poc/block-sync-aggregation.md @@ -0,0 +1,165 @@ +# Block sync aggregation POC + +## Purpose + +This POC tests one narrow change to the existing PoRBFT network path. It does +not change transaction validation, committee voting, block construction, or +finality. + +The legacy path invokes block broadcast on every committee member, then makes +each non-signing block recipient rebroadcast its sync status to every known +peer. With `N` nodes and `S` block signers, its modeled post-block request +burst is: + +```text +S * (N - S) duplicate block deliveries ++ N * (N - S) recipient status calls ++ N * S sender status calls +``` + +The POC designates the existing committee secretary as the sole block +publisher and replaces the recipient broadcasts with one acknowledgement +aggregate: + +```text +(N - S) block deliveries + (N - 1) aggregate deliveries +``` + +## Activation + +The experiment is disabled by default. Enable it only on an isolated devnet: + +```text +BLOCK_SYNC_AGGREGATION_ENABLED=true +``` + +Every node in the experiment must use the same setting. This POC does not +define a mixed-version activation protocol. + +The repository includes a loopback-only, resource-bounded overlay. Generate +six identities and enable both POC profiles so a four-validator committee has +two real non-signing block recipients: + +```text +NODE_COUNT=6 testing/devnet/scripts/setup.sh +docker compose --profile rehearsal --profile scale-poc -p demos-sync-poc \ + -f testing/devnet/docker-compose.yml \ + -f testing/devnet/docker-compose.fixture.yml \ + -f testing/devnet/docker-compose.sync-aggregation-poc.yml up --build +``` + +Set `BLOCK_SYNC_AGGREGATION_ENABLED=false` on the compose command to run the +same resource-bounded topology through the legacy path for comparison. + +Six full processes are the safe ceiling for the current 4-CPU production VPS. +The 20/30/50-node validation must run across dedicated hosts; putting those +processes on one busy machine would benchmark CPU and memory starvation rather +than consensus networking. + +## Validation performed by recipients + +An aggregate is accepted only when: + +- it has the bounded version-1 shape; +- its block number and hash match a locally stored block; +- its RPC sender signed that block; +- each claimed identity was committed in the block peerlist or signed the + block; and +- the peer is already known locally. + +The aggregate only advances the peer's sync hint to an already verified local +block. It never marks a peer online. Existing authenticated hello calls and +peer gossip remain the anti-entropy path for missed aggregate deliveries. + +## POC limitation + +The aggregate authenticates the block-signing publisher, not each relayed +peer acknowledgement. A production protocol should either carry a detached +signature from every acknowledging peer or formally state that a quorum block +signer is trusted to relay inclusion-only liveness observations. This POC must +not be deployed until that trust decision and mixed-version activation are +reviewed. + +## Modeled request counts + +For a four-validator committee: + +| Nodes | Legacy | Aggregate | Reduction | +| ----: | ------: | --------: | --------: | +| 5 | 29 | 5 | 82.8% | +| 6 | 44 | 7 | 84.1% | +| 20 | 464 | 35 | 92.5% | +| 30 | 1,004 | 55 | 94.5% | +| 50 | 2,684 | 95 | 96.5% | +| 500 | 251,984 | 995 | 99.6% | + +With 500 realistic 66-character public-key identities, the version-1 JSON +aggregate is about 34.4 KB before its authenticated RPC envelope. A production +version should encode acknowledgements as a peerlist-indexed bitmap (or use a +bounded gossip tree) to reduce bytes as well as request count. + +The model excludes periodic anti-entropy because it is not triggered once per +block. A real multi-host run is still required to measure bytes, latency, +retries, convergence, and failure recovery. + +## Six-node VPS result + +On 2026-08-18 the legacy and aggregate paths were run on the same isolated +six-node Docker topology (four-member shard, two non-signing recipients), with +a 20-second consensus cadence. Each measurement excluded startup traffic and +covered five complete blocks. + +| Mode | Block deliveries | Sender status | Receiver status | Aggregate | Total calls/block | +| --- | ---: | ---: | ---: | ---: | ---: | +| Legacy | 8 | 24 | 12 | 0 | 44 | +| Aggregate | 2 | 0 | 0 | 5 | 7 | + +The measured reduction was 84.1%, exactly matching the model. All six nodes +started and ended each sample at the same height (height spread zero). + +A missed-update recovery check then stopped node 6 while the chain advanced, +restarted it, and observed node 6 automatically converge with node 1 at height +10. This confirms the retained fast-sync/anti-entropy path repairs missed +aggregate deliveries in this topology. + +The host had four CPUs and 15 GiB RAM. Six full nodes saturated the available +CPU during rounds and used roughly 1 GiB resident memory each, so larger real +tests must use dedicated multi-host infrastructure. Running 20–50 processes on +that host would measure resource starvation rather than network scalability. + +## Lightweight transport-scale result + +`testing/devnet/scripts/run-sync-scale-emulator.ts` exercises the real +aggregate builder, wire shape, and recipient admission code over loopback HTTP +with hundreds of virtual identities. It injects 20–100 ms baseline jitter, 5% +slow peers with another 220 ms delay, and 5% retryable first-attempt failures. +The VPS wrapper pauses (but does not remove) the six full POC nodes, enforces a +memory floor, verifies DACS health, and resumes all six nodes after the run. + +On 2026-08-18, five post-block bursts were measured at each size: + +| Peers | Legacy calls/block | Aggregate calls/block | Reduction | Mean burst | Aggregate | Total wire/block | Peak RSS | Event-loop p99 | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 100 | 10,384 | 195 | 98.1% | 504 ms | 6.8 KB | 0.72 MB | 70 MB | 3.9 ms | +| 250 | 63,484 | 495 | 99.2% | 528 ms | 17.1 KB | 4.51 MB | 104 MB | 14.7 ms | +| 500 | 251,984 | 995 | 99.6% | 810 ms | 34.4 KB | 18.10 MB | 149 MB | 55.2 ms | + +The observed logical call count matched the linear model at every size. All +deliveries were admitted after bounded retries; valid aggregates passed, while +non-signer and wrong-block aggregates failed closed. The six full nodes and +all four live DACS services were healthy after automatic resume. + +Run the guarded VPS test with: + +```text +NODE_COUNTS=100,250,500 ITERATIONS=5 \ + testing/devnet/scripts/run-sync-scale-vps.sh +``` + +This is strong evidence for the transport path, not a substitute for a real +multi-host validator soak. The emulator multiplexes virtual recipients through +one Bun process and does not reproduce hundreds of databases, consensus loops, +OmniProtocol connections, cryptographic acknowledgement signatures, or WAN +kernel scheduling. The 18.1 MB sender burst at 500 peers also motivates the +documented bitmap/tree follow-up even though the burst completed in under one +second here. diff --git a/src/config/defaults.ts b/src/config/defaults.ts index 8a599075e..926f87af1 100644 --- a/src/config/defaults.ts +++ b/src/config/defaults.ts @@ -67,6 +67,7 @@ export const DEFAULT_CONFIG: AppConfig = { restore: false, blockWatchdogEnabled: false, blockWatchdogTimeoutSeconds: 600, + blockSyncAggregationEnabled: false, }, tlsnotary: { diff --git a/src/config/envKeys.ts b/src/config/envKeys.ts index bf16ac4ac..876850622 100644 --- a/src/config/envKeys.ts +++ b/src/config/envKeys.ts @@ -53,6 +53,7 @@ export const EnvKey = { RESTORE: "RESTORE", BLOCK_WATCHDOG_ENABLED: "BLOCK_WATCHDOG_ENABLED", BLOCK_WATCHDOG_TIMEOUT_SECONDS: "BLOCK_WATCHDOG_TIMEOUT_SECONDS", + BLOCK_SYNC_AGGREGATION_ENABLED: "BLOCK_SYNC_AGGREGATION_ENABLED", // --- TLSNotary --- TLSNOTARY_ENABLED: "TLSNOTARY_ENABLED", diff --git a/src/config/loader.ts b/src/config/loader.ts index de6f840f5..96feeb74f 100644 --- a/src/config/loader.ts +++ b/src/config/loader.ts @@ -159,6 +159,10 @@ export function loadConfig(): Readonly { EnvKey.BLOCK_WATCHDOG_TIMEOUT_SECONDS, d.core.blockWatchdogTimeoutSeconds, ), + blockSyncAggregationEnabled: envBool( + EnvKey.BLOCK_SYNC_AGGREGATION_ENABLED, + d.core.blockSyncAggregationEnabled, + ), }, tlsnotary: { diff --git a/src/config/types.ts b/src/config/types.ts index 6f695fddc..f699a23c2 100644 --- a/src/config/types.ts +++ b/src/config/types.ts @@ -66,6 +66,11 @@ export interface CoreConfig { restore: boolean blockWatchdogEnabled: boolean blockWatchdogTimeoutSeconds: number + /** + * POC: replace receiver-side all-peer sync rebroadcasts with one + * block-signer aggregate. Disabled by default until multi-node validation. + */ + blockSyncAggregationEnabled: boolean rpcFee: number networkFee: number /** Per-tx burn — sat/lamport-style integer for now. diff --git a/src/features/metrics/MetricsService.ts b/src/features/metrics/MetricsService.ts index afc4adbcd..dcdf258f5 100644 --- a/src/features/metrics/MetricsService.ts +++ b/src/features/metrics/MetricsService.ts @@ -143,6 +143,11 @@ export class MetricsService { this.createCounter("messages_sent_total", "Total messages sent", [ "type", ]) + this.createCounter( + "block_sync_messages_sent_total", + "POC block synchronization messages sent", + ["kind", "source"], + ) this.createCounter( "messages_received_total", "Total messages received", diff --git a/src/libs/blockchain/routines/Sync.ts b/src/libs/blockchain/routines/Sync.ts index abc0db2cf..8dd0eb77a 100644 --- a/src/libs/blockchain/routines/Sync.ts +++ b/src/libs/blockchain/routines/Sync.ts @@ -1232,7 +1232,7 @@ async function requestBlocks(): Promise { peer = next continue } - await BroadcastManager.broadcastOurSyncData() + await BroadcastManager.broadcastOurSyncData("catchup_complete") // Trigger L2PS sync triggerL2PSSync(peer) @@ -1528,7 +1528,7 @@ export async function fastSync( if (difference >= 2) { getSharedState.syncStatus = false - await BroadcastManager.broadcastOurSyncData() + await BroadcastManager.broadcastOurSyncData("lag_signal") log.debug( "[fastSync] Network highest block is more than 2 blocks ahead of our highest block, setting sync status to false and broadcasting", ) @@ -1591,7 +1591,7 @@ export async function fastSync( log.debug("[fastSync] Fast sync routine ended ⚪️⚪️⚪️⚪️⚪️⚪️⚪️⚪️⚪️") log.debug("[fastSync] Sync status: " + synced) getSharedState.syncStatus = synced - BroadcastManager.broadcastOurSyncData() + BroadcastManager.broadcastOurSyncData("fast_sync_complete") log.debug("[fastSync] Broadcasted our sync data 📤📤📤📤📤📤📤📤📤") const lastBlockNumber = await Chain.getLastBlockNumber() diff --git a/src/libs/communications/broadcastManager.ts b/src/libs/communications/broadcastManager.ts index 9e50ac49b..b144dfc10 100644 --- a/src/libs/communications/broadcastManager.ts +++ b/src/libs/communications/broadcastManager.ts @@ -8,12 +8,23 @@ import { Waiter } from "@/utilities/waiter" import { getSharedState } from "@/utilities/sharedState" import SecretaryManager from "../consensus/v2/types/secretaryManager" import { Mutex } from "async-mutex" +import { Config } from "src/config" +import { MetricsService } from "src/features/metrics/MetricsService" +import { + BlockSyncAggregateV1, + admitSyncAggregate, + buildSyncAggregate, +} from "./syncAggregation" /** * * Manages the broadcasting of messages to the network */ export class BroadcastManager { + private static syncAggregationEnabled(): boolean { + return Config.getInstance().core.blockSyncAggregationEnabled + } + /** * Broadcasts a new block to the network * @@ -48,6 +59,17 @@ export class BroadcastManager { } }) + MetricsService.getInstance().incrementCounter( + "messages_sent_total", + { type: "syncNewBlock" }, + peers.length, + ) + MetricsService.getInstance().incrementCounter( + "block_sync_messages_sent_total", + { kind: "syncNewBlock", source: "post_block" }, + peers.length, + ) + type BroadcastResult = { pubkey: string; result: RPCResponse } const settled = await Promise.allSettled(promises) const responses = settled @@ -58,15 +80,33 @@ export class BroadcastManager { .map(r => r.value) const successful = responses.filter(res => res.result.result === 200) - for (const res of responses) { - if (res.result.result !== 200) continue - await this.handleUpdatePeerSyncData( - res.pubkey, - res.result.response.syncData, + if (this.syncAggregationEnabled()) { + const aggregate = buildSyncAggregate( + block, + getSharedState.publicKeyHex, + responses, ) - } + // Apply the same aggregate locally before publishing it so the + // block sender and recipients converge through one code path. + this.applySyncAggregate( + getSharedState.publicKeyHex, + aggregate, + block, + ) + await this.broadcastSyncAggregate(aggregate) + } else { + for (const res of responses) { + if (res.result.result !== 200) continue + const body = res.result.response + if (!body || typeof body !== "object") continue + await this.handleUpdatePeerSyncData( + res.pubkey, + (body as { syncData?: string }).syncData, + ) + } - await this.broadcastOurSyncData() + await this.broadcastOurSyncData("sender_post_block") + } if (successful.length > 0) { return true @@ -171,8 +211,13 @@ export class BroadcastManager { const peer = peerman.getPeer(sender) const res = await syncBlock(block, peer) - // REVIEW: Should we await this? - await this.broadcastOurSyncData() + // Legacy behaviour fans each recipient's status back out to every + // peer. The POC returns the same syncData in this response and lets + // the block sender publish one aggregate instead. Existing hello and + // peer-gossip routines remain the anti-entropy recovery path. + if (!this.syncAggregationEnabled()) { + await this.broadcastOurSyncData("receiver_post_block") + } return { result: res ? 200 : 400, @@ -184,7 +229,7 @@ export class BroadcastManager { /** * Broadcasts our sync data to the network */ - static async broadcastOurSyncData() { + static async broadcastOurSyncData(source = "anti_entropy") { const peerlist = PeerManager.getInstance().getPeers() const promises = peerlist.map(async peer => { const request: RPCRequest = { @@ -213,6 +258,16 @@ export class BroadcastManager { type SyncResult = { pubkey: string; result: RPCResponse } const settled = await Promise.allSettled(promises) + MetricsService.getInstance().incrementCounter( + "messages_sent_total", + { type: "updateSyncData" }, + peerlist.length, + ) + MetricsService.getInstance().incrementCounter( + "block_sync_messages_sent_total", + { kind: "updateSyncData", source }, + peerlist.length, + ) const responses = settled .filter( (r): r is PromiseFulfilledResult => @@ -235,6 +290,139 @@ export class BroadcastManager { return successful.length > 0 } + /** Publish one compact acknowledgement set for a consensus-approved block. */ + static async broadcastSyncAggregate(aggregate: BlockSyncAggregateV1) { + const peerlist = PeerManager.getInstance() + .getPeers() + .filter( + peer => + peer.identity.toLowerCase() !== + getSharedState.publicKeyHex.toLowerCase(), + ) + + const settled = await Promise.allSettled( + peerlist.map(peer => { + // Authenticated calls add their envelope to params, so each + // concurrent peer must receive an independent request object. + const request: RPCRequest = { + method: "gcr_routine", + params: [ + { + method: "updateSyncAggregate", + params: [aggregate], + }, + ], + } + return peer.longCall(request, true, { + sleepTime: 250, + retries: 2, + allowedCodes: [400], + }) + }), + ) + MetricsService.getInstance().incrementCounter( + "messages_sent_total", + { type: "updateSyncAggregate" }, + peerlist.length, + ) + MetricsService.getInstance().incrementCounter( + "block_sync_messages_sent_total", + { kind: "updateSyncAggregate", source: "sender_post_block" }, + peerlist.length, + ) + + return settled.filter( + result => + result.status === "fulfilled" && result.value.result === 200, + ).length + } + + /** + * Apply a bounded block-signer observation. This POC deliberately treats + * the aggregate as a liveness hint: it can only advance known peers to an + * already verified local block and never marks a peer online. + */ + static async handleSyncAggregate( + sender: string, + value: unknown, + ): Promise<{ + result: number + message: string + accepted: number + syncData: string + }> { + const rawBlockNumber = + value && + typeof value === "object" && + typeof (value as { blockNumber?: unknown }).blockNumber === "number" + ? (value as { blockNumber: number }).blockNumber + : -1 + const blockNumber = + Number.isSafeInteger(rawBlockNumber) && + rawBlockNumber >= 0 && + rawBlockNumber <= getSharedState.lastBlockNumber + ? rawBlockNumber + : -1 + const block = + blockNumber >= 0 ? await Chain.getBlockByNumber(blockNumber) : null + return this.applySyncAggregate(sender, value, block) + } + + private static applySyncAggregate( + sender: string, + value: unknown, + block: Block | null, + ): { + result: number + message: string + accepted: number + syncData: string + } { + const peerman = PeerManager.getInstance() + const syncData = peerman.ourSyncDataString + const admission = admitSyncAggregate( + value, + block, + sender, + getSharedState.publicKeyHex, + peerman.getPeers().map(peer => peer.identity), + ) + if ("status" in admission) { + return { + result: admission.status, + message: admission.message, + accepted: 0, + syncData, + } + } + + let accepted = 0 + for (const identity of admission.acceptedPeerIds) { + const existing = peerman + .getPeers() + .find(peer => peer.identity.toLowerCase() === identity) + if (!existing) continue + const changed = + !existing.sync.status || + existing.sync.block !== block.number || + existing.sync.block_hash !== block.hash + // The PeerManager returns live Peer objects. Mutating only the + // sync hint avoids touching connection, authentication, or online + // status while also correcting a same-height conflicting hash. + existing.sync.status = true + existing.sync.block = block.number + existing.sync.block_hash = block.hash + if (changed) accepted++ + } + + return { + result: 200, + message: "Sync aggregate applied", + accepted, + syncData: peerman.ourSyncDataString, + } + } + /** * Handles the update of the sync data from a peer * diff --git a/src/libs/communications/syncAggregation.test.ts b/src/libs/communications/syncAggregation.test.ts new file mode 100644 index 000000000..97e124835 --- /dev/null +++ b/src/libs/communications/syncAggregation.test.ts @@ -0,0 +1,271 @@ +import { describe, expect, test } from "bun:test" +import { + MAX_SYNC_AGGREGATE_IDENTITIES, + MAX_SYNC_AGGREGATE_IDENTITY_LENGTH, + admitSyncAggregate, + buildSyncAggregate, + estimatePostBlockTraffic, + isBlockSyncAggregateV1, + shouldPublishBlock, + syncDataMatchesBlock, +} from "./syncAggregation" + +describe("block sync aggregation", () => { + test("collects only exact-block acknowledgements deterministically", () => { + const aggregate = buildSyncAggregate( + { number: 42, hash: "block-42" }, + "0xSecretary", + [ + { + pubkey: "0xPeerB", + result: { + result: 200, + response: { syncData: "1:42:block-42" }, + }, + }, + { + pubkey: "0xPeerA", + result: { + result: 200, + response: { syncData: "1:42:block-42" }, + }, + }, + { + pubkey: "0xWrongBlock", + result: { + result: 200, + response: { syncData: "1:41:block-41" }, + }, + }, + { + pubkey: "0xFailed", + result: { + result: 500, + response: { syncData: "1:42:block-42" }, + }, + }, + ], + ) + + expect(aggregate).toEqual({ + version: 1, + blockNumber: 42, + blockHash: "block-42", + syncedPeerIds: ["0xpeera", "0xpeerb", "0xsecretary"], + }) + }) + + test("rejects malformed or ambiguous sync data", () => { + expect(syncDataMatchesBlock("1:42:block-42", 42, "block-42")).toBe(true) + expect(syncDataMatchesBlock("0:42:block-42", 42, "block-42")).toBe( + false, + ) + expect( + syncDataMatchesBlock("1:42:block-42:extra", 42, "block-42"), + ).toBe(false) + expect(syncDataMatchesBlock("1:42x:block-42", 42, "block-42")).toBe( + false, + ) + }) + + test("bounds aggregate work and validates the wire shape", () => { + const responses = Array.from( + { length: MAX_SYNC_AGGREGATE_IDENTITIES + 50 }, + (_, index) => ({ + pubkey: `peer-${index}`, + result: { + result: 200, + response: { syncData: "1:42:block-42" }, + }, + }), + ) + const aggregate = buildSyncAggregate( + { number: 42, hash: "block-42" }, + "secretary", + responses, + ) + + expect(aggregate.syncedPeerIds).toHaveLength( + MAX_SYNC_AGGREGATE_IDENTITIES, + ) + expect(isBlockSyncAggregateV1(aggregate)).toBe(true) + expect( + isBlockSyncAggregateV1({ + ...aggregate, + syncedPeerIds: Array.from( + { length: MAX_SYNC_AGGREGATE_IDENTITIES + 1 }, + (_, index) => `peer-${index}`, + ), + }), + ).toBe(false) + expect( + isBlockSyncAggregateV1({ + ...aggregate, + syncedPeerIds: [ + "x".repeat(MAX_SYNC_AGGREGATE_IDENTITY_LENGTH + 1), + ], + }), + ).toBe(false) + }) + + test("admits only known identities committed to the verified block", () => { + const admission = admitSyncAggregate( + { + version: 1, + blockNumber: 42, + blockHash: "block-42", + syncedPeerIds: [ + "SECRETARY", + "peer-a", + "PEER-A", + "peer-b", + "unknown-peer", + ], + }, + { + number: 42, + hash: "block-42", + validation_data: { signatures: { secretary: {} } }, + content: { + peerlist: [ + { identity: "peer-a" }, + { identity: "peer-b" }, + { identity: "unknown-peer" }, + ], + }, + }, + "SECRETARY", + "peer-b", + ["secretary", "peer-a", "peer-b"], + ) + + expect(admission).toEqual({ + ok: true, + acceptedPeerIds: ["secretary", "peer-a"], + }) + }) + + test("rejects a non-signer aggregate and a mismatched local block", () => { + const aggregate = { + version: 1, + blockNumber: 42, + blockHash: "block-42", + syncedPeerIds: ["peer-a"], + } + const block = { + number: 42, + hash: "block-42", + validation_data: { signatures: { secretary: {} } }, + content: { peerlist: [{ identity: "peer-a" }] }, + } + + expect( + admitSyncAggregate(aggregate, block, "not-a-signer", "local", [ + "peer-a", + ]), + ).toEqual({ + ok: false, + status: 403, + message: "Sync aggregate sender did not sign the block", + }) + expect( + admitSyncAggregate( + aggregate, + { ...block, hash: "different-block" }, + "secretary", + "local", + ["peer-a"], + ), + ).toEqual({ + ok: false, + status: 400, + message: "Sync aggregate does not match the local chain", + }) + }) + + test("uses every signer in legacy mode and only the secretary in aggregate mode", () => { + const committee = ["secretary", "signer-b", "signer-c", "signer-d"] + + for (const identity of committee) { + expect(shouldPublishBlock(false, identity, committee)).toBe(true) + } + expect(shouldPublishBlock(true, "SECRETARY", committee)).toBe(true) + expect(shouldPublishBlock(true, "signer-b", committee)).toBe(false) + expect(shouldPublishBlock(true, "outsider", committee)).toBe(false) + expect(shouldPublishBlock(true, "secretary", [])).toBe(false) + }) + + test.each([ + [5, 29, 5, 82.75], + [20, 464, 35, 92.45], + [30, 1004, 55, 94.52], + [50, 2684, 95, 96.46], + [500, 251984, 995, 99.6], + ])( + "%i nodes reduces the modeled request burst from %i to %i", + (nodeCount, legacyRequests, aggregateRequests, minimumReduction) => { + const legacy = estimatePostBlockTraffic(nodeCount, 4, false) + const aggregate = estimatePostBlockTraffic(nodeCount, 4, true) + const reduction = + ((legacy.totalRequests - aggregate.totalRequests) / + legacy.totalRequests) * + 100 + + expect(legacy.totalRequests).toBe(legacyRequests) + expect(aggregate.totalRequests).toBe(aggregateRequests) + expect(reduction).toBeGreaterThanOrEqual(minimumReduction) + }, + ) + + test.each([20, 30, 50, 500])( + "%i receivers converge on the same acknowledged sync view", + nodeCount => { + const identities = Array.from( + { length: nodeCount }, + (_, index) => `node-${index.toString().padStart(3, "0")}`, + ) + const signers = identities.slice(0, 4) + const block = { + number: 42, + hash: "block-42", + validation_data: { + signatures: Object.fromEntries( + signers.map(identity => [identity, {}]), + ), + }, + content: { peerlist: identities }, + } + const aggregate = buildSyncAggregate( + block, + signers[0], + identities.slice(4).map(pubkey => ({ + pubkey, + result: { + result: 200, + response: { syncData: "1:42:block-42" }, + }, + })), + ) + + for (const localIdentity of identities) { + const admission = admitSyncAggregate( + aggregate, + block, + signers[0], + localIdentity, + identities, + ) + expect(admission.ok).toBe(true) + if ("status" in admission) continue + + const reconstructedView = new Set(admission.acceptedPeerIds) + if (aggregate.syncedPeerIds.includes(localIdentity)) { + reconstructedView.add(localIdentity) + } + expect([...reconstructedView].sort()).toEqual( + aggregate.syncedPeerIds, + ) + } + }, + ) +}) diff --git a/src/libs/communications/syncAggregation.ts b/src/libs/communications/syncAggregation.ts new file mode 100644 index 000000000..b13257109 --- /dev/null +++ b/src/libs/communications/syncAggregation.ts @@ -0,0 +1,303 @@ +import type Block from "../blockchain/block" + +export const SYNC_AGGREGATE_VERSION = 1 as const +export const MAX_SYNC_AGGREGATE_IDENTITIES = 1000 +export const MAX_SYNC_AGGREGATE_IDENTITY_LENGTH = 256 + +export interface BlockSyncAggregateV1 { + version: typeof SYNC_AGGREGATE_VERSION + blockNumber: number + blockHash: string + syncedPeerIds: string[] +} + +export interface PostBlockTrafficEstimate { + nodeCount: number + signerCount: number + blockPublishers: number + blockDeliveries: number + receiverSyncBroadcasts: number + senderSyncBroadcasts: number + aggregateBroadcasts: number + totalRequests: number +} + +export interface SyncAggregateBlockView { + number: number + hash: string + validation_data?: { + signatures?: Record + } + content?: { + peerlist?: unknown + } +} + +export type SyncAggregateAdmission = + | { + ok: true + acceptedPeerIds: string[] + } + | { + ok: false + status: 400 | 403 + message: string + } + +interface SyncResponseLike { + pubkey: string + result: { + result: number + response?: unknown + } +} + +function normalizeIdentity(identity: string): string { + return identity.toLowerCase() +} + +function isBoundedIdentity(identity: string): boolean { + return ( + identity.length > 0 && + identity.length <= MAX_SYNC_AGGREGATE_IDENTITY_LENGTH + ) +} + +export function shouldPublishBlock( + aggregateEnabled: boolean, + localIdentity: string, + committeeIdentities: string[], +): boolean { + if (!aggregateEnabled) return true + const designatedPublisher = committeeIdentities[0] + return ( + typeof designatedPublisher === "string" && + normalizeIdentity(designatedPublisher) === + normalizeIdentity(localIdentity) + ) +} + +/** + * Return the compact `status:block:hash` value carried by syncNewBlock. + * The outer RPC response contains the handler response under `response`. + */ +export function extractSyncData(response: SyncResponseLike): string | null { + if (response.result.result !== 200) return null + const body = response.result.response + if (!body || typeof body !== "object") return null + const syncData = (body as { syncData?: unknown }).syncData + return typeof syncData === "string" ? syncData : null +} + +export function syncDataMatchesBlock( + syncData: string, + blockNumber: number, + blockHash: string, +): boolean { + const [status, rawBlock, claimedHash, ...extra] = syncData.split(":") + if (extra.length > 0 || !/^\d+$/.test(rawBlock)) return false + const claimedBlock = Number.parseInt(rawBlock, 10) + return ( + status === "1" && + Number.isInteger(claimedBlock) && + claimedBlock === blockNumber && + claimedHash === blockHash + ) +} + +/** + * Build a deterministic, bounded acknowledgement aggregate from the block + * delivery responses. A peer is included only when its returned sync state + * names the exact block just delivered. + */ +export function buildSyncAggregate( + block: Pick, + senderIdentity: string, + responses: SyncResponseLike[], +): BlockSyncAggregateV1 { + const identities = new Set() + if (isBoundedIdentity(senderIdentity)) { + identities.add(normalizeIdentity(senderIdentity)) + } + + for (const response of responses) { + if (identities.size >= MAX_SYNC_AGGREGATE_IDENTITIES) break + const syncData = extractSyncData(response) + if ( + isBoundedIdentity(response.pubkey) && + syncData && + syncDataMatchesBlock(syncData, block.number, block.hash) + ) { + identities.add(normalizeIdentity(response.pubkey)) + } + } + + return { + version: SYNC_AGGREGATE_VERSION, + blockNumber: block.number, + blockHash: block.hash, + syncedPeerIds: [...identities] + .sort() + .slice(0, MAX_SYNC_AGGREGATE_IDENTITIES), + } +} + +export function isBlockSyncAggregateV1( + value: unknown, +): value is BlockSyncAggregateV1 { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return false + } + + const aggregate = value as Record + return ( + aggregate.version === SYNC_AGGREGATE_VERSION && + typeof aggregate.blockNumber === "number" && + Number.isSafeInteger(aggregate.blockNumber) && + aggregate.blockNumber >= 0 && + typeof aggregate.blockHash === "string" && + aggregate.blockHash.length > 0 && + aggregate.blockHash.length <= 256 && + Array.isArray(aggregate.syncedPeerIds) && + aggregate.syncedPeerIds.length <= MAX_SYNC_AGGREGATE_IDENTITIES && + aggregate.syncedPeerIds.every( + identity => + typeof identity === "string" && isBoundedIdentity(identity), + ) + ) +} + +/** + * Validate a relayed acknowledgement set against a block already verified by + * the receiver. The aggregate cannot add an identity to the peer set, mark an + * identity online, or describe a block absent from the receiver's chain. + */ +export function admitSyncAggregate( + value: unknown, + block: SyncAggregateBlockView | null, + senderIdentity: string, + localIdentity: string, + knownPeerIdentities: string[], +): SyncAggregateAdmission { + if (!isBlockSyncAggregateV1(value)) { + return { + ok: false, + status: 400, + message: "Invalid sync aggregate", + } + } + if ( + !block || + block.number !== value.blockNumber || + block.hash !== value.blockHash + ) { + return { + ok: false, + status: 400, + message: "Sync aggregate does not match the local chain", + } + } + + const signerIds = new Set( + Object.keys(block.validation_data?.signatures ?? {}).map( + normalizeIdentity, + ), + ) + if (!signerIds.has(normalizeIdentity(senderIdentity))) { + return { + ok: false, + status: 403, + message: "Sync aggregate sender did not sign the block", + } + } + + const eligibleIds = new Set(signerIds) + const committedPeerlist = Array.isArray(block.content?.peerlist) + ? block.content.peerlist + : [] + for (const entry of committedPeerlist) { + if (typeof entry === "string") { + eligibleIds.add(normalizeIdentity(entry)) + } else if ( + entry && + typeof entry === "object" && + typeof (entry as { identity?: unknown }).identity === "string" + ) { + eligibleIds.add( + normalizeIdentity((entry as { identity: string }).identity), + ) + } + } + + const local = normalizeIdentity(localIdentity) + const knownIds = new Set(knownPeerIdentities.map(normalizeIdentity)) + const acceptedPeerIds: string[] = [] + const seen = new Set() + for (const claimedIdentity of value.syncedPeerIds) { + const identity = normalizeIdentity(claimedIdentity) + if ( + seen.has(identity) || + identity === local || + !eligibleIds.has(identity) || + !knownIds.has(identity) + ) { + continue + } + seen.add(identity) + acceptedPeerIds.push(identity) + } + + return { ok: true, acceptedPeerIds } +} + +/** + * Model the post-block request burst. This deliberately excludes periodic + * anti-entropy because it is not triggered once per block. + */ +export function estimatePostBlockTraffic( + nodeCount: number, + signerCount: number, + aggregateEnabled: boolean, +): PostBlockTrafficEstimate { + const nodes = Math.max(0, Math.floor(nodeCount)) + const signers = Math.min(nodes, Math.max(0, Math.floor(signerCount))) + const recipients = nodes - signers + + if (aggregateEnabled) { + const blockPublishers = signers > 0 ? 1 : 0 + const blockDeliveries = blockPublishers * recipients + const aggregateBroadcasts = + blockDeliveries > 0 ? Math.max(0, nodes - 1) : 0 + return { + nodeCount: nodes, + signerCount: signers, + blockPublishers, + blockDeliveries, + receiverSyncBroadcasts: 0, + senderSyncBroadcasts: 0, + aggregateBroadcasts, + totalRequests: blockDeliveries + aggregateBroadcasts, + } + } + + // Stabilisation currently invokes broadcastNewBlock on every committee + // member. Duplicate deliveries short-circuit at the receiver, so each + // non-signer fans out its status once, while every signer still performs + // its own sender-side status broadcast. + const blockPublishers = signers + const blockDeliveries = blockPublishers * recipients + const receiverSyncBroadcasts = blockDeliveries > 0 ? recipients * nodes : 0 + const senderSyncBroadcasts = + blockDeliveries > 0 ? blockPublishers * nodes : 0 + return { + nodeCount: nodes, + signerCount: signers, + blockPublishers, + blockDeliveries, + receiverSyncBroadcasts, + senderSyncBroadcasts, + aggregateBroadcasts: 0, + totalRequests: + blockDeliveries + receiverSyncBroadcasts + senderSyncBroadcasts, + } +} diff --git a/src/libs/consensus/v2/PoRBFT.ts b/src/libs/consensus/v2/PoRBFT.ts index e90f88009..31f882830 100644 --- a/src/libs/consensus/v2/PoRBFT.ts +++ b/src/libs/consensus/v2/PoRBFT.ts @@ -45,6 +45,8 @@ import { readNonces, } from "@/libs/debug/nonceTrace" import { computeMergedPeerlist } from "./routines/peerlistMerge" +import { Config } from "src/config" +import { shouldPublishBlock } from "src/libs/communications/syncAggregation" export type { FailedTranscation } from "./routines/mempoolFilters" @@ -408,7 +410,25 @@ export async function consensusRoutine(): Promise { ) } - BroadcastManager.broadcastNewBlock(block) + const aggregationEnabled = + Config.getInstance().core.blockSyncAggregationEnabled + if ( + shouldPublishBlock( + aggregationEnabled, + getSharedState.publicKeyHex, + manager.shard.members.map(member => member.identity), + ) + ) { + BroadcastManager.broadcastNewBlock(block).catch(error => { + log.error( + `[consensusRoutine] Block broadcast failed: ${ + error instanceof Error + ? error.message + : String(error) + }`, + ) + }) + } DTRManager.releaseDTRWaiter(block) // Apply pending L2PS proofs to L1 state diff --git a/src/libs/network/manageGCRRoutines.ts b/src/libs/network/manageGCRRoutines.ts index d5f976e41..4aa483806 100644 --- a/src/libs/network/manageGCRRoutines.ts +++ b/src/libs/network/manageGCRRoutines.ts @@ -285,6 +285,14 @@ export default async function manageGCRRoutines( break } + case "updateSyncAggregate": { + response.response = await BroadcastManager.handleSyncAggregate( + sender, + params[0], + ) + break + } + // case "getAccountByTelegramUsername": { // const username = params[0] diff --git a/testing/devnet/docker-compose.sync-aggregation-poc.yml b/testing/devnet/docker-compose.sync-aggregation-poc.yml new file mode 100644 index 000000000..1f3f43fd5 --- /dev/null +++ b/testing/devnet/docker-compose.sync-aggregation-poc.yml @@ -0,0 +1,128 @@ +# Isolated, resource-bounded overlay for the block-sync aggregation POC. +# All host-facing ports bind to loopback and all nodes use the same flag. +services: + postgres: + mem_limit: 768m + cpus: 0.5 + ports: !override + - "127.0.0.1:55432:5432" + + tlsnotary: + mem_limit: 512m + cpus: 0.25 + ports: !override + - "127.0.0.1:7148:7047" + + node-1: + mem_limit: 1280m + cpus: 0.55 + environment: + BLOCK_SYNC_AGGREGATION_ENABLED: ${BLOCK_SYNC_AGGREGATION_ENABLED:-true} + METRICS_ENABLED: "true" + volumes: + - ../../src:/app/src:ro + ports: !override + - "127.0.0.1:53551:53551" + - "127.0.0.1:53552:53552" + - "127.0.0.1:3105:3005" + + node-2: + mem_limit: 1280m + cpus: 0.55 + environment: + BLOCK_SYNC_AGGREGATION_ENABLED: ${BLOCK_SYNC_AGGREGATION_ENABLED:-true} + METRICS_ENABLED: "true" + volumes: + - ../../src:/app/src:ro + ports: !override + - "127.0.0.1:53553:53553" + - "127.0.0.1:53554:53554" + - "127.0.0.1:3106:3005" + + node-3: + mem_limit: 1280m + cpus: 0.55 + environment: + BLOCK_SYNC_AGGREGATION_ENABLED: ${BLOCK_SYNC_AGGREGATION_ENABLED:-true} + METRICS_ENABLED: "true" + volumes: + - ../../src:/app/src:ro + ports: !override + - "127.0.0.1:53555:53555" + - "127.0.0.1:53556:53556" + - "127.0.0.1:3107:3005" + + node-4: + mem_limit: 1280m + cpus: 0.55 + environment: + BLOCK_SYNC_AGGREGATION_ENABLED: ${BLOCK_SYNC_AGGREGATION_ENABLED:-true} + METRICS_ENABLED: "true" + volumes: + - ../../src:/app/src:ro + ports: !override + - "127.0.0.1:53557:53557" + - "127.0.0.1:53558:53558" + - "127.0.0.1:3108:3005" + + node-5: + mem_limit: 1280m + cpus: 0.55 + environment: + BLOCK_SYNC_AGGREGATION_ENABLED: ${BLOCK_SYNC_AGGREGATION_ENABLED:-true} + METRICS_ENABLED: "true" + volumes: + - ../../src:/app/src:ro + ports: !override + - "127.0.0.1:53559:53559" + - "127.0.0.1:53560:53560" + - "127.0.0.1:3109:3005" + + # Second non-committee receiver for the scale POC. It is absent from the + # canonical devnet and starts only when the scale-poc profile is selected. + node-6: + profiles: [scale-poc] + image: demos-devnet-node + container_name: demos-devnet-node-6 + depends_on: + postgres: + condition: service_healthy + tlsnotary: + condition: service_started + node-1: + condition: service_started + mem_limit: 1280m + cpus: 0.55 + environment: + NODE_ENV: development + L2PS_HASH_RELAY_NON_PROD: "true" + PG_HOST: postgres + PG_PORT: "5432" + PG_USER: ${POSTGRES_USER:-demosuser} + PG_PASSWORD: ${POSTGRES_PASSWORD:-demospass} + PG_DATABASE: node6_db + PORT: ${NODE6_PORT:-53561} + OMNI_PORT: ${NODE6_OMNI_PORT:-53562} + EXPOSED_URL: http://node-6:${NODE6_PORT:-53561} + CONSENSUS_TIME: ${CONSENSUS_TIME:-10} + SUDO_PUBKEY: ${SUDO_PUBKEY:-} + TLSNOTARY_ENABLED: "true" + TLSNOTARY_MODE: docker + TLSNOTARY_HOST: tlsnotary + TLSNOTARY_PORT: "7047" + BLOCK_SYNC_AGGREGATION_ENABLED: ${BLOCK_SYNC_AGGREGATION_ENABLED:-true} + METRICS_ENABLED: "true" + volumes: + - ./identities/node6.identity:/app/.demos_identity:ro + - ./demos_peerlist.json:/app/demos_peerlist.json:ro + - ./l2ps:/app/data/l2ps:ro + - ./genesis.devnet.json:/app/data/genesis.json:ro + - ./empty-snapshot:/app/data/snapshot:ro + - ../../src:/app/src:ro + ports: + - "127.0.0.1:53561:53561" + - "127.0.0.1:53562:53562" + - "127.0.0.1:3110:3005" + networks: + - demos-network + restart: unless-stopped diff --git a/testing/devnet/postgres-init/init-databases.sql b/testing/devnet/postgres-init/init-databases.sql index 73176432a..8f0ffe98b 100644 --- a/testing/devnet/postgres-init/init-databases.sql +++ b/testing/devnet/postgres-init/init-databases.sql @@ -8,6 +8,8 @@ CREATE DATABASE node4_db; -- the node-5 service is gated behind a docker-compose profile and only -- starts when the rehearsal harness brings it up. CREATE DATABASE node5_db; +-- node6_db is used only by the opt-in block-sync scale POC. +CREATE DATABASE node6_db; -- Grant permissions GRANT ALL PRIVILEGES ON DATABASE node1_db TO demosuser; @@ -15,3 +17,4 @@ GRANT ALL PRIVILEGES ON DATABASE node2_db TO demosuser; GRANT ALL PRIVILEGES ON DATABASE node3_db TO demosuser; GRANT ALL PRIVILEGES ON DATABASE node4_db TO demosuser; GRANT ALL PRIVILEGES ON DATABASE node5_db TO demosuser; +GRANT ALL PRIVILEGES ON DATABASE node6_db TO demosuser; diff --git a/testing/devnet/scripts/generate-genesis.sh b/testing/devnet/scripts/generate-genesis.sh index c4352c714..318bd6cc4 100755 --- a/testing/devnet/scripts/generate-genesis.sh +++ b/testing/devnet/scripts/generate-genesis.sh @@ -19,23 +19,14 @@ if [[ -f "${DEVNET_DIR}/.env" ]]; then source "${DEVNET_DIR}/.env" fi -NODE1_PORT=${NODE1_PORT:-53551} -NODE2_PORT=${NODE2_PORT:-53553} -NODE3_PORT=${NODE3_PORT:-53555} -NODE4_PORT=${NODE4_PORT:-53557} -NODE5_PORT=${NODE5_PORT:-53559} NODE_COUNT="${NODE_COUNT:-4}" STAKE="${DEVNET_VALIDATOR_STAKE:-1000000000000000000}" get_port() { - case "$1" in - 1) echo "${NODE1_PORT}" ;; - 2) echo "${NODE2_PORT}" ;; - 3) echo "${NODE3_PORT}" ;; - 4) echo "${NODE4_PORT}" ;; - 5) echo "${NODE5_PORT}" ;; - *) echo "❌ Unknown node index $1" >&2 && exit 1 ;; - esac + local index="$1" + local variable="NODE${index}_PORT" + local default_port=$((53549 + (2 * index))) + echo "${!variable:-${default_port}}" } echo "🧬 Syncing genesis validators to identities (count=${NODE_COUNT})..." diff --git a/testing/devnet/scripts/generate-peerlist.sh b/testing/devnet/scripts/generate-peerlist.sh index a791cc315..c4c10698f 100755 --- a/testing/devnet/scripts/generate-peerlist.sh +++ b/testing/devnet/scripts/generate-peerlist.sh @@ -11,27 +11,16 @@ if [[ -f "${DEVNET_DIR}/.env" ]]; then fi # Default ports if not set -NODE1_PORT=${NODE1_PORT:-53551} -NODE2_PORT=${NODE2_PORT:-53553} -NODE3_PORT=${NODE3_PORT:-53555} -NODE4_PORT=${NODE4_PORT:-53557} -NODE5_PORT=${NODE5_PORT:-53559} - -# NODE_COUNT mirrors generate-identities.sh — use 5 for the rehearsal -# fresh-joiner scenario, 4 for the default 4-node devnet. +# NODE_COUNT mirrors generate-identities.sh. Ports default to the existing +# odd-numbered sequence (53551, 53553, ...), while NODE_PORT can override +# any member. This keeps the POC topology extensible without another case arm. NODE_COUNT="${NODE_COUNT:-4}" -# Map index → exposed port for the peerlist body. Add new entries here if -# NODE_COUNT grows beyond 5. get_port() { - case "$1" in - 1) echo "${NODE1_PORT}" ;; - 2) echo "${NODE2_PORT}" ;; - 3) echo "${NODE3_PORT}" ;; - 4) echo "${NODE4_PORT}" ;; - 5) echo "${NODE5_PORT}" ;; - *) echo "❌ Unknown node index $1" >&2 && exit 1 ;; - esac + local index="$1" + local variable="NODE${index}_PORT" + local default_port=$((53549 + (2 * index))) + echo "${!variable:-${default_port}}" } echo "📋 Generating devnet peerlist (count=${NODE_COUNT})..." diff --git a/testing/devnet/scripts/measure-sync-poc.sh b/testing/devnet/scripts/measure-sync-poc.sh new file mode 100755 index 000000000..cdddcf056 --- /dev/null +++ b/testing/devnet/scripts/measure-sync-poc.sh @@ -0,0 +1,77 @@ +#!/bin/bash +set -euo pipefail + +NODE_COUNT="${NODE_COUNT:-6}" +TARGET_BLOCKS="${TARGET_BLOCKS:-5}" +MAX_WAIT_SECONDS="${MAX_WAIT_SECONDS:-240}" + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "${TMP_DIR}"' EXIT + +snapshot() { + local destination="$1" + : >"${destination}" + for i in $(seq 1 "${NODE_COUNT}"); do + docker exec "demos-devnet-node-${i}" bun -e ' +const text = await (await fetch("http://127.0.0.1:9090/metrics")).text() +const lines = text.split(String.fromCharCode(10)) +const value = prefix => { + const line = lines.find(candidate => candidate.startsWith(prefix)) + return line ? Number(line.slice(line.lastIndexOf(" ") + 1)) : 0 +} +console.log(JSON.stringify({ + height: value("demos_block_height "), + blockDeliveries: value("demos_block_sync_messages_sent_total{kind=\"syncNewBlock\",source=\"post_block\"}"), + senderStatus: value("demos_block_sync_messages_sent_total{kind=\"updateSyncData\",source=\"sender_post_block\"}"), + receiverStatus: value("demos_block_sync_messages_sent_total{kind=\"updateSyncData\",source=\"receiver_post_block\"}"), + aggregate: value("demos_block_sync_messages_sent_total{kind=\"updateSyncAggregate\",source=\"sender_post_block\"}"), +})) +' >>"${destination}" + done +} + +minimum_height() { + python3 - "$1" <<'PY' +import json, sys +rows = [json.loads(line) for line in open(sys.argv[1]) if line.strip()] +print(min(row["height"] for row in rows)) +PY +} + +snapshot "${TMP_DIR}/start.jsonl" +START_HEIGHT="$(minimum_height "${TMP_DIR}/start.jsonl")" +DEADLINE=$((SECONDS + MAX_WAIT_SECONDS)) + +while true; do + snapshot "${TMP_DIR}/current.jsonl" + CURRENT_HEIGHT="$(minimum_height "${TMP_DIR}/current.jsonl")" + if (( CURRENT_HEIGHT >= START_HEIGHT + TARGET_BLOCKS )); then + break + fi + if (( SECONDS >= DEADLINE )); then + echo "Timed out: minimum height moved ${START_HEIGHT} -> ${CURRENT_HEIGHT}" >&2 + exit 1 + fi + sleep 5 +done + +python3 - "${TMP_DIR}/start.jsonl" "${TMP_DIR}/current.jsonl" <<'PY' +import json, sys + +before = [json.loads(line) for line in open(sys.argv[1]) if line.strip()] +after = [json.loads(line) for line in open(sys.argv[2]) if line.strip()] +fields = ["blockDeliveries", "senderStatus", "receiverStatus", "aggregate"] +blocks = min(row["height"] for row in after) - min(row["height"] for row in before) +deltas = {field: sum(b[field] - a[field] for a, b in zip(before, after)) for field in fields} +deltas["totalPostBlockCalls"] = sum(deltas.values()) + +print(json.dumps({ + "nodes": len(before), + "blocksObserved": blocks, + "startHeights": [row["height"] for row in before], + "endHeights": [row["height"] for row in after], + "endHeightSpread": max(row["height"] for row in after) - min(row["height"] for row in after), + "deltas": deltas, + "callsPerBlock": {field: round(value / blocks, 3) for field, value in deltas.items()}, +}, indent=2)) +PY diff --git a/testing/devnet/scripts/run-sync-scale-emulator.ts b/testing/devnet/scripts/run-sync-scale-emulator.ts new file mode 100644 index 000000000..496c14c29 --- /dev/null +++ b/testing/devnet/scripts/run-sync-scale-emulator.ts @@ -0,0 +1,653 @@ +import { performance } from "node:perf_hooks" +import { + admitSyncAggregate, + buildSyncAggregate, + estimatePostBlockTraffic, + type BlockSyncAggregateV1, + type SyncAggregateBlockView, +} from "../../../src/libs/communications/syncAggregation" + +interface EmulatorConfig { + nodeCounts: number[] + iterations: number + signerCount: number + baseLatencyMs: number + jitterMs: number + slowPeerRate: number + slowPeerExtraMs: number + transientFailureRate: number + maxAttempts: number + requestTimeoutMs: number +} + +interface ActiveRound { + block: SyncAggregateBlockView + identities: string[] + secretary: string + config: EmulatorConfig +} + +interface AttemptResult { + ok: boolean + attempts: number + elapsedMs: number + requestBytes: number + responseBytes: number + body: unknown +} + +interface IterationResult { + nodeCount: number + blockNumber: number + aggregateIdentities: number + aggregateBytes: number + blockDeliverySuccesses: number + aggregateDeliverySuccesses: number + logicalCalls: number + httpAttempts: number + retryAttempts: number + wireBytes: number + elapsedMs: number + blockPhaseMs: number + aggregatePhaseMs: number + requestLatenciesMs: number[] +} + +interface ScenarioResult { + nodeCount: number + signerCount: number + iterations: number + legacyCallsPerBlock: number + aggregateCallsPerBlock: number + modeledReductionPercent: number + observedLogicalCallsPerBlock: number + observedHttpAttemptsPerBlock: number + observedRetryAttempts: number + aggregateBytes: number + wireBytesPerBlock: number + elapsedMs: { + mean: number + p50: number + p95: number + p99: number + max: number + } + requestLatencyMs: { + p50: number + p95: number + p99: number + max: number + } + cpuMs: number + rssPeakBytes: number + eventLoopDelayMs: { + mean: number + p95: number + p99: number + max: number + } + allDeliveriesAdmitted: boolean +} + +const encoder = new TextEncoder() +let activeRound: ActiveRound | null = null + +function numericArgument(name: string, fallback: number): number { + const prefix = `--${name}=` + const raw = process.argv + .find(value => value.startsWith(prefix)) + ?.slice(prefix.length) + if (raw === undefined) return fallback + const parsed = Number(raw) + if (!Number.isFinite(parsed) || parsed < 0) { + throw new Error(`Invalid --${name}`) + } + return parsed +} + +function parseConfig(): EmulatorConfig { + const rawCounts = + process.argv + .find(value => value.startsWith("--nodes=")) + ?.slice("--nodes=".length) ?? "100,250,500" + const nodeCounts = rawCounts.split(",").map(value => Number(value)) + if ( + nodeCounts.length === 0 || + nodeCounts.some( + value => !Number.isSafeInteger(value) || value < 5 || value > 1000, + ) + ) { + throw new Error("--nodes must contain integers between 5 and 1000") + } + + return { + nodeCounts, + iterations: numericArgument("iterations", 5), + signerCount: numericArgument("signers", 4), + baseLatencyMs: numericArgument("base-latency-ms", 20), + jitterMs: numericArgument("jitter-ms", 80), + slowPeerRate: numericArgument("slow-peer-rate", 0.05), + slowPeerExtraMs: numericArgument("slow-peer-extra-ms", 220), + transientFailureRate: numericArgument("transient-failure-rate", 0.05), + maxAttempts: numericArgument("max-attempts", 3), + requestTimeoutMs: numericArgument("request-timeout-ms", 1500), + } +} + +function peerIdentity(index: number): string { + return `0x${index.toString(16).padStart(64, "0")}` +} + +function deterministicUnit( + peerIndex: number, + blockNumber: number, + phase: number, +): number { + let value = + Math.imul(peerIndex + 1, 0x45d9f3b) ^ + Math.imul(blockNumber + 17, 0x119de1f3) ^ + Math.imul(phase + 31, 0x3449f) + value ^= value >>> 16 + return (value >>> 0) / 0x1_0000_0000 +} + +function peerDelayMs( + peerIndex: number, + blockNumber: number, + phase: number, + config: EmulatorConfig, +): number { + const jitterUnit = deterministicUnit(peerIndex, blockNumber, phase) + const slowUnit = deterministicUnit(peerIndex, blockNumber, phase + 100) + return ( + config.baseLatencyMs + + Math.floor(jitterUnit * config.jitterMs) + + (slowUnit < config.slowPeerRate ? config.slowPeerExtraMs : 0) + ) +} + +function shouldFailFirstAttempt( + peerIndex: number, + blockNumber: number, + phase: number, + config: EmulatorConfig, +): boolean { + return ( + deterministicUnit(peerIndex, blockNumber, phase + 200) < + config.transientFailureRate + ) +} + +function json(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { + status, + headers: { "content-type": "application/json" }, + }) +} + +const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + async fetch(request): Promise { + const round = activeRound + if (!round) return json({ error: "round-not-active" }, 503) + + const url = new URL(request.url) + const match = url.pathname.match(/^\/(block|aggregate)\/(\d+)\/(\d+)$/) + if (!match) return json({ error: "not-found" }, 404) + + const phaseName = match[1] + const peerIndex = Number(match[2]) + const attempt = Number(match[3]) + if ( + !Number.isSafeInteger(peerIndex) || + peerIndex < 0 || + peerIndex >= round.identities.length + ) { + return json({ error: "unknown-peer" }, 404) + } + + const phase = phaseName === "block" ? 1 : 2 + if ( + attempt === 0 && + shouldFailFirstAttempt( + peerIndex, + round.block.number, + phase, + round.config, + ) + ) { + return json({ error: "simulated-transient-failure" }, 503) + } + + await Bun.sleep( + peerDelayMs(peerIndex, round.block.number, phase, round.config), + ) + + if (phaseName === "block") { + return json({ + result: 200, + response: { + syncData: `1:${round.block.number}:${round.block.hash}`, + }, + }) + } + + const payload = (await request.json()) as { + aggregate?: BlockSyncAggregateV1 + } + const admission = admitSyncAggregate( + payload.aggregate, + round.block, + round.secretary, + round.identities[peerIndex], + round.identities, + ) + if (!admission.ok) { + return json( + { result: admission.status, message: admission.message }, + admission.status, + ) + } + return json({ + result: 200, + accepted: admission.acceptedPeerIds.length, + }) + }, +}) + +function bodyBytes(value: unknown): number { + return encoder.encode(JSON.stringify(value)).byteLength +} + +async function postWithRetry( + pathForAttempt: (attempt: number) => string, + body: unknown, + config: EmulatorConfig, +): Promise { + const started = performance.now() + const requestBody = JSON.stringify(body) + const requestBytes = encoder.encode(requestBody).byteLength + let responseBytes = 0 + let parsed: unknown = null + + for (let attempt = 0; attempt < config.maxAttempts; attempt++) { + try { + const response = await fetch( + `http://127.0.0.1:${server.port}${pathForAttempt(attempt)}`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: requestBody, + signal: AbortSignal.timeout(config.requestTimeoutMs), + }, + ) + const raw = await response.text() + responseBytes += encoder.encode(raw).byteLength + parsed = raw ? JSON.parse(raw) : null + if (response.ok) { + return { + ok: true, + attempts: attempt + 1, + elapsedMs: performance.now() - started, + requestBytes: requestBytes * (attempt + 1), + responseBytes, + body: parsed, + } + } + } catch { + parsed = null + } + } + + return { + ok: false, + attempts: config.maxAttempts, + elapsedMs: performance.now() - started, + requestBytes: requestBytes * config.maxAttempts, + responseBytes, + body: parsed, + } +} + +function percentile(values: number[], quantile: number): number { + if (values.length === 0) return 0 + const sorted = [...values].sort((a, b) => a - b) + const index = Math.min( + sorted.length - 1, + Math.max(0, Math.ceil(quantile * sorted.length) - 1), + ) + return sorted[index] +} + +function rounded(value: number): number { + return Number(value.toFixed(3)) +} + +async function runIteration( + nodeCount: number, + blockNumber: number, + config: EmulatorConfig, +): Promise { + const identities = Array.from({ length: nodeCount }, (_, index) => + peerIdentity(index), + ) + const signers = identities.slice(0, config.signerCount) + const secretary = signers[0] + const block: SyncAggregateBlockView = { + number: blockNumber, + hash: `block-${nodeCount}-${blockNumber}`, + validation_data: { + signatures: Object.fromEntries( + signers.map(identity => [identity, "signature"]), + ), + }, + content: { peerlist: identities }, + } + activeRound = { block, identities, secretary, config } + + const started = performance.now() + const blockStarted = performance.now() + const blockBody = { blockNumber: block.number, blockHash: block.hash } + const blockResults = await Promise.all( + identities.slice(config.signerCount).map((identity, offset) => { + const peerIndex = offset + config.signerCount + return postWithRetry( + attempt => `/block/${peerIndex}/${attempt}`, + blockBody, + config, + ).then(result => ({ identity, result })) + }), + ) + const blockPhaseMs = performance.now() - blockStarted + + const responses = blockResults + .filter(entry => entry.result.ok) + .map(entry => ({ + pubkey: entry.identity, + result: entry.result.body as { + result: number + response?: unknown + }, + })) + const aggregate = buildSyncAggregate( + { number: block.number, hash: block.hash }, + secretary, + responses, + ) + + const aggregateStarted = performance.now() + const aggregateBody = { aggregate } + const aggregateResults = await Promise.all( + identities.slice(1).map((_, offset) => { + const peerIndex = offset + 1 + return postWithRetry( + attempt => `/aggregate/${peerIndex}/${attempt}`, + aggregateBody, + config, + ) + }), + ) + const aggregatePhaseMs = performance.now() - aggregateStarted + activeRound = null + + const allResults = [ + ...blockResults.map(entry => entry.result), + ...aggregateResults, + ] + return { + nodeCount, + blockNumber, + aggregateIdentities: aggregate.syncedPeerIds.length, + aggregateBytes: bodyBytes(aggregate), + blockDeliverySuccesses: blockResults.filter(entry => entry.result.ok) + .length, + aggregateDeliverySuccesses: aggregateResults.filter(result => result.ok) + .length, + logicalCalls: allResults.length, + httpAttempts: allResults.reduce( + (total, result) => total + result.attempts, + 0, + ), + retryAttempts: allResults.reduce( + (total, result) => total + result.attempts - 1, + 0, + ), + wireBytes: allResults.reduce( + (total, result) => + total + result.requestBytes + result.responseBytes, + 0, + ), + elapsedMs: performance.now() - started, + blockPhaseMs, + aggregatePhaseMs, + requestLatenciesMs: allResults.map(result => result.elapsedMs), + } +} + +async function runScenario( + nodeCount: number, + config: EmulatorConfig, +): Promise { + const eventLoopDelaysMs: number[] = [] + let expectedSampleAt = performance.now() + 10 + const eventLoopSampler = setInterval(() => { + const now = performance.now() + eventLoopDelaysMs.push(Math.max(0, now - expectedSampleAt)) + expectedSampleAt = now + 10 + }, 10) + const cpuBefore = process.cpuUsage() + let rssPeakBytes = process.memoryUsage().rss + const rssSampler = setInterval(() => { + rssPeakBytes = Math.max(rssPeakBytes, process.memoryUsage().rss) + }, 20) + + const iterations: IterationResult[] = [] + for (let index = 0; index < config.iterations; index++) { + const result = await runIteration( + nodeCount, + nodeCount * 10_000 + index, + config, + ) + iterations.push(result) + console.error( + `sync-scale nodes=${nodeCount} iteration=${index + 1}/${config.iterations} elapsed_ms=${rounded(result.elapsedMs)} attempts=${result.httpAttempts}`, + ) + } + + clearInterval(rssSampler) + clearInterval(eventLoopSampler) + const cpu = process.cpuUsage(cpuBefore) + const legacy = estimatePostBlockTraffic( + nodeCount, + config.signerCount, + false, + ) + const aggregate = estimatePostBlockTraffic( + nodeCount, + config.signerCount, + true, + ) + const elapsed = iterations.map(result => result.elapsedMs) + const requestLatencies = iterations.flatMap( + result => result.requestLatenciesMs, + ) + const logicalCalls = iterations.reduce( + (total, result) => total + result.logicalCalls, + 0, + ) + const attempts = iterations.reduce( + (total, result) => total + result.httpAttempts, + 0, + ) + + return { + nodeCount, + signerCount: config.signerCount, + iterations: config.iterations, + legacyCallsPerBlock: legacy.totalRequests, + aggregateCallsPerBlock: aggregate.totalRequests, + modeledReductionPercent: rounded( + ((legacy.totalRequests - aggregate.totalRequests) / + legacy.totalRequests) * + 100, + ), + observedLogicalCallsPerBlock: rounded(logicalCalls / config.iterations), + observedHttpAttemptsPerBlock: rounded(attempts / config.iterations), + observedRetryAttempts: attempts - logicalCalls, + aggregateBytes: Math.max( + ...iterations.map(result => result.aggregateBytes), + ), + wireBytesPerBlock: rounded( + iterations.reduce((total, result) => total + result.wireBytes, 0) / + config.iterations, + ), + elapsedMs: { + mean: rounded( + elapsed.reduce((total, value) => total + value, 0) / + elapsed.length, + ), + p50: rounded(percentile(elapsed, 0.5)), + p95: rounded(percentile(elapsed, 0.95)), + p99: rounded(percentile(elapsed, 0.99)), + max: rounded(Math.max(...elapsed)), + }, + requestLatencyMs: { + p50: rounded(percentile(requestLatencies, 0.5)), + p95: rounded(percentile(requestLatencies, 0.95)), + p99: rounded(percentile(requestLatencies, 0.99)), + max: rounded(Math.max(...requestLatencies)), + }, + cpuMs: rounded((cpu.user + cpu.system) / 1000), + rssPeakBytes, + eventLoopDelayMs: { + mean: rounded( + eventLoopDelaysMs.reduce((total, value) => total + value, 0) / + Math.max(1, eventLoopDelaysMs.length), + ), + p95: rounded(percentile(eventLoopDelaysMs, 0.95)), + p99: rounded(percentile(eventLoopDelaysMs, 0.99)), + max: rounded(Math.max(0, ...eventLoopDelaysMs)), + }, + allDeliveriesAdmitted: iterations.every( + result => + result.blockDeliverySuccesses === + nodeCount - config.signerCount && + result.aggregateDeliverySuccesses === nodeCount - 1 && + result.aggregateIdentities === + nodeCount - config.signerCount + 1, + ), + } +} + +function validateSafetyCases(): Record { + const identities = Array.from({ length: 6 }, (_, index) => + peerIdentity(index), + ) + const block: SyncAggregateBlockView = { + number: 42, + hash: "block-42", + validation_data: { + signatures: Object.fromEntries( + identities.slice(0, 4).map(identity => [identity, {}]), + ), + }, + content: { peerlist: identities }, + } + const aggregate = buildSyncAggregate( + block, + identities[0], + identities.slice(4).map(pubkey => ({ + pubkey, + result: { + result: 200, + response: { syncData: "1:42:block-42" }, + }, + })), + ) + const valid = admitSyncAggregate( + aggregate, + block, + identities[0], + identities[1], + identities, + ) + const nonSigner = admitSyncAggregate( + aggregate, + block, + identities[5], + identities[1], + identities, + ) + const wrongBlock = admitSyncAggregate( + aggregate, + { ...block, hash: "wrong-block" }, + identities[0], + identities[1], + identities, + ) + return { + validAggregateAccepted: valid.ok, + nonSignerRejected: + !nonSigner.ok && "status" in nonSigner && nonSigner.status === 403, + wrongBlockRejected: + !wrongBlock.ok && + "status" in wrongBlock && + wrongBlock.status === 400, + } +} + +async function main(): Promise { + const config = parseConfig() + if ( + !Number.isSafeInteger(config.iterations) || + config.iterations < 1 || + !Number.isSafeInteger(config.signerCount) || + config.signerCount < 1 || + config.slowPeerRate > 1 || + config.transientFailureRate > 1 + ) { + throw new Error("Invalid emulator configuration") + } + + const results: ScenarioResult[] = [] + for (const nodeCount of config.nodeCounts) { + if (config.signerCount >= nodeCount) { + throw new Error("signer count must be smaller than node count") + } + results.push(await runScenario(nodeCount, config)) + } + + const safety = validateSafetyCases() + const passed = + results.every( + result => + result.allDeliveriesAdmitted && + result.observedLogicalCallsPerBlock === + result.aggregateCallsPerBlock, + ) && Object.values(safety).every(Boolean) + + // eslint-disable-next-line no-console -- stdout is the machine-readable report. + console.log( + JSON.stringify( + { + kind: "block-sync-scale-emulator-v1", + generatedAt: new Date().toISOString(), + config, + safety, + results, + passed, + }, + null, + 2, + ), + ) + if (!passed) process.exitCode = 1 +} + +try { + await main() +} finally { + activeRound = null + server.stop(true) +} diff --git a/testing/devnet/scripts/run-sync-scale-vps.sh b/testing/devnet/scripts/run-sync-scale-vps.sh new file mode 100755 index 000000000..3b06fd64b --- /dev/null +++ b/testing/devnet/scripts/run-sync-scale-vps.sh @@ -0,0 +1,99 @@ +#!/bin/bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +NODE_COUNTS="${NODE_COUNTS:-100,250,500}" +ITERATIONS="${ITERATIONS:-5}" +MIN_AVAILABLE_KIB="${MIN_AVAILABLE_KIB:-2097152}" +RESULTS_DIR="${RESULTS_DIR:-${ROOT_DIR}/.poc-results}" +TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)" +REPORT_PATH="${RESULTS_DIR}/sync-scale-${TIMESTAMP}.json" +POC_NODES=( + demos-devnet-node-1 + demos-devnet-node-2 + demos-devnet-node-3 + demos-devnet-node-4 + demos-devnet-node-5 + demos-devnet-node-6 +) +DACS_SERVICES=(dacs-gateway dacs-oracle dacs-dd dacs-auditor) +PAUSED_NODES=() + +resume_nodes() { + for container in "${PAUSED_NODES[@]}"; do + docker unpause "${container}" >/dev/null 2>&1 || true + done +} +trap resume_nodes EXIT INT TERM + +for service in "${DACS_SERVICES[@]}"; do + if [[ "$(systemctl is-active "${service}")" != "active" ]]; then + echo "Refusing scale test: ${service} is not active" >&2 + exit 1 + fi +done + +for container in "${POC_NODES[@]}"; do + if [[ "$(docker inspect -f '{{.State.Running}}' "${container}" 2>/dev/null || true)" == "true" ]]; then + docker pause "${container}" >/dev/null + PAUSED_NODES+=("${container}") + fi +done + +AVAILABLE_KIB="$(awk '/MemAvailable:/ { print $2 }' /proc/meminfo)" +if (( AVAILABLE_KIB < MIN_AVAILABLE_KIB )); then + echo "Refusing scale test: only ${AVAILABLE_KIB} KiB memory available" >&2 + exit 1 +fi + +mkdir -p "${RESULTS_DIR}" +echo "Running bounded sync emulator: nodes=${NODE_COUNTS} iterations=${ITERATIONS}" >&2 +echo "Six-node POC paused=${#PAUSED_NODES[@]}; live DACS remains active" >&2 + +cd "${ROOT_DIR}" +bun testing/devnet/scripts/run-sync-scale-emulator.ts \ + --nodes="${NODE_COUNTS}" \ + --iterations="${ITERATIONS}" >"${REPORT_PATH}" & +EMULATOR_PID=$! + +while kill -0 "${EMULATOR_PID}" 2>/dev/null; do + AVAILABLE_KIB="$(awk '/MemAvailable:/ { print $2 }' /proc/meminfo)" + if (( AVAILABLE_KIB < MIN_AVAILABLE_KIB )); then + echo "Aborting scale test: memory guard crossed" >&2 + kill "${EMULATOR_PID}" 2>/dev/null || true + wait "${EMULATOR_PID}" || true + exit 1 + fi + sleep 1 +done + +wait "${EMULATOR_PID}" + +for service in "${DACS_SERVICES[@]}"; do + if [[ "$(systemctl is-active "${service}")" != "active" ]]; then + echo "Scale test completed but ${service} is not active" >&2 + exit 1 + fi +done + +resume_nodes +PAUSED_NODES=() +POC_PORTS=(53551 53553 53555 53557 53559 53561) +for attempt in $(seq 1 12); do + READY=0 + for port in "${POC_PORTS[@]}"; do + if curl -fsS --max-time 2 "http://127.0.0.1:${port}/health" >/dev/null 2>&1; then + READY=$((READY + 1)) + fi + done + if (( READY == ${#POC_PORTS[@]} )); then + break + fi + sleep 5 +done +if (( READY != ${#POC_PORTS[@]} )); then + echo "Scale test passed, but only ${READY}/${#POC_PORTS[@]} POC nodes recovered" >&2 + exit 1 +fi + +echo "SYNC_SCALE_REPORT=${REPORT_PATH}"