diff --git a/.github/workflows/pcs.yml b/.github/workflows/pcs.yml index e565538f..bb91cdeb 100644 --- a/.github/workflows/pcs.yml +++ b/.github/workflows/pcs.yml @@ -6,18 +6,35 @@ on: paths: - 'misc/pcs.tex' - '.github/workflows/pcs.yml' + pull_request: + paths: + - 'misc/pcs.tex' + - '.github/workflows/pcs.yml' workflow_dispatch: -permissions: - contents: write - concurrency: group: pcs-${{ github.ref }} cancel-in-progress: true jobs: + check-pdf: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - name: Compile LaTeX + uses: xu-cheng/latex-action@v3 + with: + working_directory: misc + root_file: pcs.tex + build-pdf: + if: github.event_name != 'pull_request' runs-on: ubuntu-latest + permissions: + contents: write steps: - uses: actions/checkout@v4 - name: Compile LaTeX @@ -26,7 +43,6 @@ jobs: working_directory: misc root_file: pcs.tex - name: Publish PDF as release asset - if: github.event_name != 'pull_request' uses: softprops/action-gh-release@v2 with: tag_name: pcs-latest diff --git a/.github/workflows/pdf.yml b/.github/workflows/pdf.yml index 7098ce24..b9434ccf 100644 --- a/.github/workflows/pdf.yml +++ b/.github/workflows/pdf.yml @@ -7,18 +7,36 @@ on: - 'misc/doc.tex' - 'misc/images/**' - '.github/workflows/pdf.yml' + pull_request: + paths: + - 'misc/doc.tex' + - 'misc/images/**' + - '.github/workflows/pdf.yml' workflow_dispatch: -permissions: - contents: write - concurrency: group: pdf-${{ github.ref }} cancel-in-progress: true jobs: + check-pdf: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - name: Compile LaTeX + uses: xu-cheng/latex-action@v3 + with: + working_directory: misc + root_file: doc.tex + build-pdf: + if: github.event_name != 'pull_request' runs-on: ubuntu-latest + permissions: + contents: write steps: - uses: actions/checkout@v4 - name: Compile LaTeX @@ -27,7 +45,6 @@ jobs: working_directory: misc root_file: doc.tex - name: Publish PDF as release asset - if: github.event_name != 'pull_request' uses: softprops/action-gh-release@v2 with: tag_name: spec-latest diff --git a/.github/workflows/ring-switching.yml b/.github/workflows/ring-switching.yml index f0588d7f..71eba98c 100644 --- a/.github/workflows/ring-switching.yml +++ b/.github/workflows/ring-switching.yml @@ -6,18 +6,35 @@ on: paths: - 'misc/ring-switching.tex' - '.github/workflows/ring-switching.yml' + pull_request: + paths: + - 'misc/ring-switching.tex' + - '.github/workflows/ring-switching.yml' workflow_dispatch: -permissions: - contents: write - concurrency: group: ring-switching-${{ github.ref }} cancel-in-progress: true jobs: + check-pdf: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - name: Compile LaTeX + uses: xu-cheng/latex-action@v3 + with: + working_directory: misc + root_file: ring-switching.tex + build-pdf: + if: github.event_name != 'pull_request' runs-on: ubuntu-latest + permissions: + contents: write steps: - uses: actions/checkout@v4 - name: Compile LaTeX @@ -26,7 +43,6 @@ jobs: working_directory: misc root_file: ring-switching.tex - name: Publish PDF as release asset - if: github.event_name != 'pull_request' uses: softprops/action-gh-release@v2 with: tag_name: ring-switching-latest diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 00000000..aa8d84e4 --- /dev/null +++ b/.github/workflows/rust.yml @@ -0,0 +1,32 @@ +name: Rust + +on: + push: + branches: [ "main" ] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: rust-${{ github.ref }} + cancel-in-progress: true + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + components: clippy + - uses: Swatinem/rust-cache@v2 + - name: Test + run: cargo test --release --workspace + - name: Clippy + run: cargo clippy --release --workspace --all-targets -- -D warnings + - name: Rustdoc + env: + RUSTDOCFLAGS: -D warnings + run: cargo doc --release --workspace --no-deps diff --git a/Cargo.lock b/Cargo.lock index d4203dbc..8151ed67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -152,7 +152,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -278,6 +278,7 @@ dependencies = [ "blake3", "lean_vm", "primitives", + "rand", ] [[package]] @@ -355,6 +356,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" name = "pcs" version = "0.1.0" dependencies = [ + "bincode", "blake3", "fiat_shamir", "primitives", @@ -382,6 +384,7 @@ dependencies = [ name = "primitives" version = "0.1.0" dependencies = [ + "bincode", "rayon", "serde", "tracing-forest", @@ -522,7 +525,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -563,35 +566,24 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syn" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a207d6d6a2b7fc470b80443726053f18a2481b7e1eee970597051596567987a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "thiserror" -version = "2.0.19" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.19" +version = "2.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn 3.0.2", + "syn", ] [[package]] @@ -622,7 +614,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] [[package]] @@ -775,5 +767,5 @@ checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn", ] diff --git a/README.md b/README.md index 5f8b682d..de58b90e 100644 --- a/README.md +++ b/README.md @@ -1,72 +1,96 @@

leanVM-b

- + leanVM-b

- Documentation + Specification source + Latest main-branch PDF

-- WARNING: Highly experimental / sloppy (currently). -- Proving architecture is volontarily kept simple for now +- Warning: highly experimental. +- The proving architecture is intentionally kept simple. # Benchmarks -Machine: M4 Max +Measured on an AMD Ryzen 7 PRO 8700GE with 11 Rayon workers. Timings vary by machine; cycles and proof sizes describe the protocol configuration more reliably. ### XMSS aggregation ```bash -RAYON_NUM_THREADS=11 cargo run --release -- xmss --n-signatures 890 +RAYON_NUM_THREADS=11 cargo run --release -- xmss --n-signatures 890 --log-inv-rate 1 ``` ``` XMSS aggregation, 890 signatures - cycles (VM steps) : 1,528,683 = 2^20.544 ( 1,717.621 / XMSS) - XOR instructions : 125,491 = 2^16.937 ( 141.001 / XMSS) - MUL instructions : 295,495 = 2^18.173 ( 332.017 / XMSS) - SET instructions : 354,194 = 2^18.434 ( 397.971 / XMSS) - DEREF instructions : 508,569 = 2^18.956 ( 571.426 / XMSS) - JUMP instructions : 114,813 = 2^16.809 ( 129.003 / XMSS) - BLAKE3 instructions : 130,121 = 2^16.989 ( 146.203 / XMSS) - committed witness size : 2^25.662 - data memory : 2^22 padded (2^21.707 used) - proof size : 588.68 KiB - proving (incl. witness gen) : 1.358 s - verifying : 0.00594 s - throughput : 655.559 XMSS/s + cycles (VM steps) : 1,513,580 = 2^20.53 ( 1,700.652 / XMSS) + XOR instructions : 125,491 = 2^16.937 ( 141.001 / XMSS) + MUL instructions : 292,821 = 2^18.16 ( 329.012 / XMSS) + SET instructions : 341,765 = 2^18.383 ( 384.006 / XMSS) + DEREF instructions : 508,569 = 2^18.956 ( 571.426 / XMSS) + JUMP instructions : 114,813 = 2^16.809 ( 129.003 / XMSS) + BLAKE3 instructions : 130,121 = 2^16.989 ( 146.203 / XMSS) + PACK64X2 instructions : 0 = - ( 0 / XMSS) + committed witness size : 2^26.364 + data memory : 2^22 padded (2^21.701 used) + proof size : 359.617 KiB + proving (incl. witness gen) : 1.905 s + verifying : 0.00426 s + throughput : 467.072 XMSS/s ``` ### Recursion ```bash -RAYON_NUM_THREADS=11 cargo run --release -- recursion --n 2 +RAYON_NUM_THREADS=11 cargo run --release -- recursion --n 2 --log-inv-rate 2 ``` ``` -recursion 2→1: 2 inner proofs of 852,207 cycles each - guest cycles (VM steps) : 1,955,570 = 2^20.899 (1.147 / inner cycle) - XOR instructions : 519,080 = 2^18.986 - MUL instructions : 651,514 = 2^19.313 - SET instructions : 161,965 = 2^17.305 - DEREF instructions : 561,871 = 2^19.1 - JUMP instructions : 18,431 = 2^14.17 - BLAKE3 instructions : 42,709 = 2^15.382 - committed witness size : 2^25.902 - data memory : 2^22 padded (2^21.21 used) - recursive proof size : 590.859 KiB - outer proving : 1.548 s - complete recursive verify : 0.292 s +recursion 2→1: 2 inner proofs of 1,472,224 cycles each + guest cycles (VM steps) : 869,886 = 2^19.73 (0.295 / inner cycle) + XOR instructions : 215,490 = 2^17.717 + MUL instructions : 268,787 = 2^18.036 + SET instructions : 70,556 = 2^16.106 + DEREF instructions : 265,183 = 2^18.017 + JUMP instructions : 9,299 = 2^13.183 + BLAKE3 instructions : 18,115 = 2^14.145 + PACK64X2 instructions : 22,456 = 2^14.455 + committed witness size : 2^25.506 + data memory : 2^21 padded (2^20.044 used) + recursive proof size : 235.305 KiB + outer proving : 1.26 s + complete recursive verify : 0.0303 s ``` -## Security, proof size etc +### Fibonacci -- security = 120 bits, proven, unique-decoding regime, Ligerito -- proof size = BIG (≈ 0.7 MiB) -Both will be improved later. +```bash +RAYON_NUM_THREADS=11 cargo run --release -- fibonacci --n 2000000 --log-inv-rate 1 +``` + +``` +Fibonacci (in the exponent, i.e. modulo 2^64 - 1), N = 2,000,000 + cycles (VM steps) : 2,034,017 + XOR instructions : 2^10.966 + MUL instructions : 2^20.937 + SET instructions : 2^12.552 + DEREF instructions : 2^13.967 + JUMP instructions : 2^10.968 + BLAKE3 instructions : 0 + PACK64X2 instructions : 0 + committed witness size : 2^25.658 + proof size : 336.4 KiB + proving (incl. witness gen) : 1.031051291s + verifying : 3.274ms + throughput : 1,972,760 cycles/s +``` + +## Security + +- 128-bit proven (LDR Johnson) ## Credits diff --git a/crates/fiat_shamir/src/sponge.rs b/crates/fiat_shamir/src/sponge.rs index 4051e2cf..e21e4bf0 100644 --- a/crates/fiat_shamir/src/sponge.rs +++ b/crates/fiat_shamir/src/sponge.rs @@ -1,3 +1,4 @@ +// CREDIT: https://github.com/signalapp/libsignal/blob/main/rust/poksho/src/shosha256.rs, AGPL-3.0-only. //! The VM-native Fiat–Shamir sponge: THE verifier-randomness source for the //! whole stack — flock's zerocheck / lincheck, the Ligerito PCS, and //! leanVM-b's own protocol (whose `ProverState` / `VerifierState` wrap this @@ -11,11 +12,15 @@ //! the streaming hasher cannot be reproduced by the one 64-byte compression the //! machine has. //! +//! Scalars are `E = F192` (the tower challenge field): their three +//! little-endian `K = F64` limbs occupy the first three compression lanes, +//! with the scalar domain tag in the fourth. +//! //! Construction adapted from Signal's ShoSha256 "Stateful Hash Object" //! (`libsignal/rust/poksho/src/shosha256.rs`, © 2020 Signal Messenger, LLC, //! AGPL-3.0-only): a chaining value advanced by domain-separated absorb / //! squeeze steps. Here the underlying hash is the VM's BLAKE3 compression -//! rather than SHA-256, inputs are GF(2^128) field elements, and — because +//! rather than SHA-256, inputs are `K = GF(2^64)` field words, and — because //! every absorb is domain-tagged per compression — no explicit double-hash //! ratchet is needed. //! @@ -24,42 +29,42 @@ //! integer, and a byte string cannot alias), byte strings are length-framed, //! and each squeeze ratchets the state (binding challenge order). -use primitives::field::F128; +use primitives::field::{F64, F192}; /// `f(a, b) = BLAKE3(a‖b)` on two 256-bit halves laid out little-endian into 64 /// bytes — *exactly* the VM's `Blake3` opcode: 64 input bytes → 32-byte digest, -/// split back into two field words. THE primitive; the sponge is a chain of +/// split back into four field words. THE primitive; the sponge is a chain of /// these, so a zkDSL program replays it with one `blake3(...)` per step. -pub fn compress(a: [F128; 2], b: [F128; 2]) -> [F128; 2] { +pub fn compress(a: [F64; 4], b: [F64; 4]) -> [F64; 4] { let mut input = [0u8; 64]; - for (slot, w) in input.chunks_exact_mut(16).zip([a[0], a[1], b[0], b[1]]) { - slot.copy_from_slice(&w.to_le_bytes()); + for (slot, w) in input.chunks_exact_mut(8).zip(a.into_iter().chain(b)) { + slot.copy_from_slice(&w.0.to_le_bytes()); } let d = *blake3::hash(&input).as_bytes(); - let word = |b: &[u8]| F128::from_le_bytes(b.try_into().unwrap()); - [word(&d[..16]), word(&d[16..])] + std::array::from_fn(|k| F64(u64::from_le_bytes(d[8 * k..8 * k + 8].try_into().unwrap()))) } -// Domain-separation tags, carried in the SECOND input word of every absorbed -// block, so no two roles (a scalar, a byte word, a length frame, a squeeze, a -// PoW step) can alias: the adversary controls only the FIRST word (the datum), -// never the tag. Distinct nonzero constants suffice. -const DS_SCALAR: F128 = F128::new(1, 0); -const DS_BYTE: F128 = F128::new(2, 0); -const DS_LEN: F128 = F128::new(3, 0); -const DS_SQUEEZE: F128 = F128::new(4, 0); -const DS_POW: F128 = F128::new(5, 0); - -/// `compress(base, (nonce, DS_POW))` has its low `bits` bits zero — the grinding -/// predicate over the VM compression. A CONTIGUOUS low-bit window (rather than -/// byte-wise leading zeros) so a recursive verifier re-checks it with a single -/// loop over the bit decomposition of the digest word (`grind_check` in -/// `guests/recursion.py`). `bits` is always `< 64`. +// Domain-separation tags. Scalar absorbs fill three data lanes and put the tag +// in lane 3; byte/length/PoW absorbs use at most two data lanes and put the tag +// in lane 2. No two roles can alias: the adversary never controls the tag. +// Distinct nonzero constants suffice. +const DS_SCALAR: F64 = F64(1); +const DS_BYTE: F64 = F64(2); +const DS_LEN: F64 = F64(3); +const DS_SQUEEZE: F64 = F64(4); +const DS_POW: F64 = F64(5); + +/// `compress(base, (nonce.c0, nonce.c1, nonce.c2, DS_POW))` has its low `bits` +/// bits zero — the +/// grinding predicate over the VM compression. A CONTIGUOUS low-bit window +/// (rather than byte-wise leading zeros) so a recursive verifier re-checks it +/// with a single loop over the bit decomposition of the digest word +/// (`grind_check` in `guests/recursion.py`). `bits` is always `< 64`. #[inline] -fn pow_bits_ok(base: [F128; 2], nonce: F128, bits: u32) -> bool { +fn pow_bits_ok(base: [F64; 4], nonce: F192, bits: u32) -> bool { debug_assert!(bits < 64, "grinding deficit fits the digest's low word"); - let digest = compress(base, [nonce, DS_POW])[0]; - digest.lo & ((1u64 << bits) - 1) == 0 + let digest = compress(base, [F64(nonce.c0), F64(nonce.c1), F64(nonce.c2), DS_POW])[0]; + digest.0 & ((1u64 << bits) - 1) == 0 } /// The shared Fiat–Shamir state (see the module docs). Protocol functions take @@ -68,7 +73,7 @@ fn pow_bits_ok(base: [F128; 2], nonce: F128, bits: u32) -> bool { #[derive(Clone)] pub struct Sponge { /// The 256-bit chaining value: a Merkle–Damgård hash of the transcript so far. - cv: [F128; 2], + cv: [F64; 4], } impl Sponge { @@ -77,9 +82,9 @@ impl Sponge { /// any challenge — there is no mid-protocol "observe public data" step to get /// wrong (or forget). (Untraced: the seed is the replay STARTING state, not an /// op of the recorded transcript.) - pub fn new(label: &[u8], statement: &[F128]) -> Self { - let mut s = Self { cv: [F128::ZERO, F128::ZERO] }; - s.absorb_bytes_untraced(b"leanvm-b/transcript/v1"); + pub fn new(label: &[u8], statement: &[F192]) -> Self { + let mut s = Self { cv: [F64::ZERO; 4] }; + s.absorb_bytes_untraced(b"leanvm-b/transcript/v2"); s.absorb_bytes_untraced(label); for &x in statement { s.observe_untraced(x); @@ -90,74 +95,77 @@ impl Sponge { /// A fresh chain at the zero state: the guest-side aggregation and export /// transcripts start here (no label), and the harness mirrors them. pub fn empty() -> Self { - Self { cv: [F128::ZERO; 2] } + Self { cv: [F64::ZERO; 4] } } - /// Absorb one scalar: `cv ← compress(cv, (x, DS_SCALAR))`. - pub fn observe(&mut self, x: F128) { + /// Absorb one 24-byte scalar (three little-endian `K` limbs): + /// `cv ← compress(cv, (c0, c1, c2, DS_SCALAR))`. + pub fn observe(&mut self, x: F192) { self.observe_untraced(x); trace(|| TraceOp::Observe(x)); } - fn observe_untraced(&mut self, x: F128) { - self.cv = compress(self.cv, [x, DS_SCALAR]); + fn observe_untraced(&mut self, x: F192) { + self.cv = compress(self.cv, [F64(x.c0), F64(x.c1), F64(x.c2), DS_SCALAR]); } /// Absorb a byte string (a protocol label, a Merkle root): a length frame - /// then its 16-byte words as tagged blocks, so a field element, a raw - /// integer, and a byte string cannot alias. + /// then its 16-byte (two-word) chunks as tagged blocks (the domain tag + /// occupies the third lane, leaving two data words per block), so a field + /// element, a raw integer, and a byte string cannot alias. pub fn absorb_bytes(&mut self, bytes: &[u8]) { self.absorb_bytes_untraced(bytes); trace(|| TraceOp::AbsorbBytes(bytes.to_vec())); } fn absorb_bytes_untraced(&mut self, bytes: &[u8]) { - self.cv = compress(self.cv, [F128::new(bytes.len() as u64, 0), DS_LEN]); + self.cv = compress(self.cv, [F64(bytes.len() as u64), F64::ZERO, DS_LEN, F64::ZERO]); for chunk in bytes.chunks(16) { let mut buf = [0u8; 16]; buf[..chunk.len()].copy_from_slice(chunk); - let w = F128::from_le_bytes(buf); - self.cv = compress(self.cv, [w, DS_BYTE]); + let w = |o: usize| F64(u64::from_le_bytes(buf[o..o + 8].try_into().unwrap())); + self.cv = compress(self.cv, [w(0), w(8), DS_BYTE, F64::ZERO]); } } - /// Squeeze a challenge and ratchet: the challenge is the first word of - /// `compress(cv, (0, DS_SQUEEZE))`, whose full output becomes the new state — - /// domain-separated from absorbs, so a challenge cannot be confused with a - /// continued absorb. In Fiat–Shamir everything is public; soundness comes from - /// each challenge being a random-oracle image of the entire prior transcript. - pub fn sample(&mut self) -> F128 { + /// Squeeze a challenge and ratchet: the challenge's three limbs are the + /// first three words of `compress(cv, (0, 0, DS_SQUEEZE, 0))`, whose full output + /// becomes the new state — domain-separated from absorbs, so a challenge + /// cannot be confused with a continued absorb. In Fiat–Shamir everything is + /// public; soundness comes from each challenge being a random-oracle image + /// of the entire prior transcript. + pub fn sample(&mut self) -> F192 { let v = self.sample_untraced(); trace(|| TraceOp::Sample(v)); v } - fn sample_untraced(&mut self) -> F128 { - let out = compress(self.cv, [F128::ZERO, DS_SQUEEZE]); + fn sample_untraced(&mut self) -> F192 { + let out = compress(self.cv, [F64::ZERO, F64::ZERO, DS_SQUEEZE, F64::ZERO]); self.cv = out; - out[0] + F192::new(out[0].0, out[1].0, out[2].0) } /// Squeeze `n` challenges, in order. - pub fn sample_vec(&mut self, n: usize) -> Vec { + pub fn sample_vec(&mut self, n: usize) -> Vec { (0..n).map(|_| self.sample()).collect() } - /// The PoW base `compress(cv, (0, DS_POW))`, read without mutating the live - /// state (the nonce is bound separately by [`Self::absorb_nonce`]). - fn pow_base(&self) -> [F128; 2] { - compress(self.cv, [F128::ZERO, DS_POW]) + /// The PoW base `compress(cv, (0, 0, DS_POW, 0))`, read without mutating the + /// live state (the nonce is bound separately by [`Self::absorb_nonce`]). + fn pow_base(&self) -> [F64; 4] { + compress(self.cv, [F64::ZERO, F64::ZERO, DS_POW, F64::ZERO]) } /// The current 256-bit chaining value. - pub fn state(&self) -> [F128; 2] { + pub fn state(&self) -> [F64; 4] { self.cv } - /// The grinding digest this state yields for `nonce` (read-only preview; + /// The grinding digest word this state yields for `nonce` (read-only preview; /// [`Self::verify_pow`] is the mutating check). - pub fn pow_digest(&self, nonce: F128) -> F128 { - compress(self.pow_base(), [nonce, DS_POW])[0] + pub fn pow_digest(&self, nonce: F192) -> F64 { + compress(self.pow_base(), [F64(nonce.c0), F64(nonce.c1), F64(nonce.c2), DS_POW])[0] } /// Re-run recorded verifier transcript ops through this sponge, asserting @@ -173,7 +181,10 @@ impl Sponge { assert_eq!(self.sample_untraced(), *v, "trace replay diverged") } TraceOp::Pow { nonce, bits, .. } => { - assert!(self.verify_pow_field_untraced(*nonce, *bits), "trace replay: grind failed") + assert!( + self.verify_pow_field_untraced(*nonce, *bits), + "trace replay: grind failed" + ) } TraceOp::StreamRaw(_) | TraceOp::Opening => {} } @@ -181,8 +192,8 @@ impl Sponge { } /// Bind a grinding nonce into the state (both sides, so they stay in lockstep). - fn absorb_nonce(&mut self, nonce: F128) { - self.cv = compress(self.cv, [nonce, DS_POW]); + fn absorb_nonce(&mut self, nonce: F192) { + self.cv = compress(self.cv, [F64(nonce.c0), F64(nonce.c1), F64(nonce.c2), DS_POW]); } /// Prover-side PoW grind: find the smallest `u64` nonce whose PoW hash clears @@ -197,7 +208,7 @@ impl Sponge { } else if (1u64 << bits.min(63)) < PARALLEL_GRIND_MIN_HASHES { let mut n: u64 = 0; loop { - if pow_bits_ok(base, F128::new(n, 0), bits) { + if pow_bits_ok(base, F192::new(n, 0, 0), bits) { break n; } n = n.wrapping_add(1); @@ -211,37 +222,46 @@ impl Sponge { loop { if let Some(n) = (start..start.saturating_add(block)) .into_par_iter() - .find_first(|&n| pow_bits_ok(base, F128::new(n, 0), bits)) + .find_first(|&n| pow_bits_ok(base, F192::new(n, 0, 0), bits)) { break n; } start = start.saturating_add(block); } }; - self.absorb_nonce(F128::new(nonce, 0)); + self.absorb_nonce(F192::new(nonce, 0, 0)); nonce } /// Verifier-side mirror of [`Self::grind_pow`]: check `nonce` clears the `bits` /// PoW against the current state, then bind it regardless (so the sponge stays /// in lockstep with an honest prover — a failed check rejects at the call - /// site). `bits = 0` accepts only the canonical nonce `0`. + /// site). `bits = 0` accepts only the canonical nonce `0`, which keeps proofs + /// non-malleable at zero-bit grinding sites. pub fn verify_pow(&mut self, nonce: u64, bits: u32) -> bool { - self.verify_pow_field(F128::new(nonce, 0), bits) + self.verify_pow_field(F192::new(nonce, 0, 0), bits) } /// Verify a nonce transported as a field word. Allowing the complete field /// domain does not weaken grinding: each candidate still requires one hash /// and succeeds with probability 2^-bits. Honest provers remain canonical /// and search the deterministic u64 subset in [`Self::grind_pow`]. - pub fn verify_pow_field(&mut self, nonce: F128, bits: u32) -> bool { - trace(|| TraceOp::Pow { nonce, bits, digest: self.pow_digest(nonce) }); + pub fn verify_pow_field(&mut self, nonce: F192, bits: u32) -> bool { + trace(|| TraceOp::Pow { + nonce, + bits, + digest: self.pow_digest(nonce), + }); self.verify_pow_field_untraced(nonce, bits) } - fn verify_pow_field_untraced(&mut self, nonce: F128, bits: u32) -> bool { + fn verify_pow_field_untraced(&mut self, nonce: F192, bits: u32) -> bool { let base = self.pow_base(); - let ok = if bits == 0 { nonce == F128::ZERO } else { pow_bits_ok(base, nonce, bits) }; + let ok = if bits == 0 { + nonce == F192::ZERO + } else { + pow_bits_ok(base, nonce, bits) + }; self.absorb_nonce(nonce); ok } @@ -257,16 +277,20 @@ impl Sponge { #[derive(Clone, Debug, PartialEq, Eq)] pub enum TraceOp { /// A stream word consumed without binding (grinding nonces). - StreamRaw(F128), + StreamRaw(F192), /// An absorbed scalar (transmitted or derived — the sponge cannot tell). - Observe(F128), + Observe(F192), /// `absorb_bytes` (labels, roots). AbsorbBytes(Vec), - Sample(F128), - /// A grinding check: the nonce, the required bits, and the digest the + Sample(F192), + /// A grinding check: the nonce, the required bits, and the digest word the /// pre-absorb state yields for that nonce (so trace consumers never need /// to track sponge state in lockstep). - Pow { nonce: F128, bits: u32, digest: F128 }, + Pow { + nonce: F192, + bits: u32, + digest: F64, + }, /// An opening hint consumed (the Ligerito hint channel). Opening, } @@ -300,8 +324,8 @@ pub fn trace(op: impl FnOnce() -> TraceOp) { mod tests { use super::*; - fn f(k: u64) -> F128 { - F128::new(k, k ^ 0x1234) + fn f(k: u64) -> F192 { + F192::new(k, k ^ 0x1234, k.rotate_left(17)) } /// A challenge binds every prior absorbed scalar: flipping one observed value @@ -326,14 +350,17 @@ mod tests { } /// A scalar and a byte string cannot alias (distinct domain tags), so - /// observing a scalar vs absorbing its 16-byte encoding diverge. + /// observing a scalar vs absorbing its 24-byte encoding diverge. #[test] fn sponge_domain_separation() { let x = f(9); let mut a = Sponge::new(b"t", &[]); a.observe(x); let mut b = Sponge::new(b"t", &[]); - let bytes = x.to_le_bytes(); + let mut bytes = [0u8; 24]; + bytes[..8].copy_from_slice(&x.c0.to_le_bytes()); + bytes[8..16].copy_from_slice(&x.c1.to_le_bytes()); + bytes[16..].copy_from_slice(&x.c2.to_le_bytes()); b.absorb_bytes(&bytes); assert_ne!(a.sample(), b.sample()); } @@ -347,12 +374,9 @@ mod tests { let mut clone = sp.clone(); clone.grind_pow(8) }; - assert!(pow_bits_ok(base, F128::new(good, 0), 8)); + assert!(pow_bits_ok(base, F192::new(good, 0, 0), 8)); // A random wrong nonce almost surely fails an 8-bit grind. - assert!( - !pow_bits_ok(base, F128::new(good.wrapping_add(1).wrapping_mul(3) | 1, 0), 8) - || good != 0 - ); + assert!(!pow_bits_ok(base, F192::new(good.wrapping_add(1).wrapping_mul(3) | 1, 0, 0), 8,) || good != 0); } /// Recursive proofs transport the nonce as one field word. Its high limb is @@ -362,7 +386,7 @@ mod tests { let mut verifier = Sponge::new(b"t", &[f(1)]); let base = verifier.pow_base(); let nonce = (0..u64::MAX) - .map(|lo| F128::new(lo, 1)) + .map(|lo| F192::new(lo, 1, 2)) .find(|&nonce| pow_bits_ok(base, nonce, 8)) .expect("an 8-bit grind has a solution"); @@ -372,6 +396,6 @@ mod tests { assert_eq!(verifier.state(), expected.state()); let mut zero_bits = Sponge::new(b"t", &[f(1)]); - assert!(!zero_bits.verify_pow_field(F128::new(0, 1), 0)); + assert!(!zero_bits.verify_pow_field(F192::new(0, 1, 0), 0)); } } diff --git a/crates/fiat_shamir/src/transcript.rs b/crates/fiat_shamir/src/transcript.rs index 59d7c7e6..73d0928a 100644 --- a/crates/fiat_shamir/src/transcript.rs +++ b/crates/fiat_shamir/src/transcript.rs @@ -24,9 +24,9 @@ //! functions take these SAME states (`ps`/`vs`), drawing their challenges from //! the one shared sponge while their proof data rides its own structs. -use primitives::field::F128; use crate::sponge::trace; pub use crate::sponge::{Sponge, TraceOp, trace_start, trace_take}; +use primitives::field::{F64, F192}; /// A complete proof: the scalar transcript stream plus the Ligerito opening hint /// channel — **two** channels, no bolted-on side field. The commitment root and @@ -44,7 +44,7 @@ pub use crate::sponge::{Sponge, TraceOp, trace_start, trace_take}; pub struct Proof { /// Every transmitted field scalar, in protocol order (plus flock's scalar /// sub-proof as trailing raw transport words). - pub stream: Vec, + pub stream: Vec, /// Ligerito openings (sumcheck messages + Merkle roots/paths), in order. pub openings: Vec, } @@ -59,18 +59,21 @@ pub enum Error { NotFullyConsumed, /// A grinding nonce failed its proof-of-work check. PowFailed, + /// A transmitted field element used as a narrower encoding had nonzero + /// limbs outside that encoding. + NonCanonicalEncoding, } /// Prover side: writes scalars into the stream and opening hints to the side. pub struct ProverState { sponge: Sponge, - stream: Vec, + stream: Vec, openings: Vec, } impl ProverState { /// `statement` is the public input, seeded into the sponge (see [`Sponge::new`]). - pub fn new(label: &[u8], statement: &[F128]) -> Self { + pub fn new(label: &[u8], statement: &[F192]) -> Self { Self { sponge: Sponge::new(label, statement), stream: Vec::new(), @@ -81,27 +84,27 @@ impl ProverState { /// Transmit a scalar into the proof AND bind it into the sponge (the two are /// inseparable — you cannot send without binding). #[inline] - pub fn add_scalar(&mut self, x: F128) { + pub fn add_scalar(&mut self, x: F192) { self.sponge.observe(x); self.stream.push(x); } - pub fn add_scalars(&mut self, xs: &[F128]) { + pub fn add_scalars(&mut self, xs: &[F192]) { for &x in xs { self.add_scalar(x); } } - pub fn sample(&mut self) -> F128 { + pub fn sample(&mut self) -> F192 { self.sponge.sample() } /// Prover mirror of [`VerifierState::observe_scalar`]. - pub fn observe_scalar(&mut self, x: F128) { + pub fn observe_scalar(&mut self, x: F192) { self.sponge.observe(x); } - pub fn sample_vec(&mut self, n: usize) -> Vec { + pub fn sample_vec(&mut self, n: usize) -> Vec { (0..n).map(|_| self.sponge.sample()).collect() } @@ -117,7 +120,7 @@ impl ProverState { /// no-work nonce `0`. pub fn grind(&mut self, bits: u32) { let nonce = self.sponge.grind_pow(bits); - self.stream.push(F128::new(nonce, 0)); + self.stream.push(F192::new(nonce, 0, 0)); } /// Absorb a byte string (a sub-protocol label, a Merkle root) — data both @@ -152,7 +155,7 @@ impl ProverState { /// hints in order. pub struct VerifierState<'a, O> { sponge: Sponge, - stream: &'a [F128], + stream: &'a [F192], offset: usize, openings: &'a [O], oi: usize, @@ -161,7 +164,7 @@ pub struct VerifierState<'a, O> { impl<'a, O> VerifierState<'a, O> { /// `statement` is the public input, seeded into the sponge (see [`Sponge::new`]) /// — must match the prover's, or the sponges diverge and verification fails. - pub fn new(label: &[u8], proof: &'a Proof, statement: &[F128]) -> Self { + pub fn new(label: &[u8], proof: &'a Proof, statement: &[F192]) -> Self { Self { sponge: Sponge::new(label, statement), stream: &proof.stream, @@ -174,46 +177,48 @@ impl<'a, O> VerifierState<'a, O> { /// A verifier state with EMPTY transport channels — a challenge source for /// unit tests that drive sub-protocols without a transmitted stream (leaks /// one small allocation; do not use outside tests). - pub fn detached(label: &[u8], statement: &[F128]) -> VerifierState<'static, O> { - let empty = Box::leak(Box::new(Proof { stream: Vec::new(), openings: Vec::new() })); + pub fn detached(label: &[u8], statement: &[F192]) -> VerifierState<'static, O> { + let empty = Box::leak(Box::new(Proof { + stream: Vec::new(), + openings: Vec::new(), + })); VerifierState::new(label, empty, statement) } /// Read the next scalar, binding it into the sponge (mirrors `add_scalar`). #[inline] - pub fn next_scalar(&mut self) -> Result { + pub fn next_scalar(&mut self) -> Result { let x = *self.stream.get(self.offset).ok_or(Error::ExceededStream)?; self.offset += 1; self.sponge.observe(x); Ok(x) } - pub fn next_scalars(&mut self, n: usize) -> Result, Error> { + pub fn next_scalars(&mut self, n: usize) -> Result, Error> { (0..n).map(|_| self.next_scalar()).collect() } /// Advance the stream cursor by one **without** binding into the sponge — the /// read counterpart of the raw nonce push in [`ProverState::grind`]. - fn take_raw(&mut self) -> Result { + fn take_raw(&mut self) -> Result { let x = *self.stream.get(self.offset).ok_or(Error::ExceededStream)?; self.offset += 1; trace(|| TraceOp::StreamRaw(x)); Ok(x) } - - pub fn sample(&mut self) -> F128 { + pub fn sample(&mut self) -> F192 { self.sponge.sample() } - pub fn sample_vec(&mut self, n: usize) -> Vec { + pub fn sample_vec(&mut self, n: usize) -> Vec { (0..n).map(|_| self.sample()).collect() } /// Absorb a value both parties compute themselves (never transmitted): /// protocol steps that bind derived values before sampling, e.g. the /// stacked-bytecode claim reduction (`leaf::verify_balance`). - pub fn observe_scalar(&mut self, x: F128) { + pub fn observe_scalar(&mut self, x: F192) { self.sponge.observe(x); } @@ -238,7 +243,7 @@ impl<'a, O> VerifierState<'a, O> { /// The sponge's current chaining value (recursion harnesses snapshot the /// phase-boundary states as guest debug checkpoints). - pub fn sponge_state(&self) -> [F128; 2] { + pub fn sponge_state(&self) -> [F64; 4] { self.sponge.state() } @@ -275,8 +280,8 @@ impl<'a, O> VerifierState<'a, O> { mod tests { use super::*; - fn f(k: u64) -> F128 { - F128::new(k, k ^ 0x1234) + fn f(k: u64) -> F192 { + F192::new(k, k ^ 0x1234, k.rotate_left(17)) } /// Prover and verifier stay in lockstep across a mixed transcript @@ -298,5 +303,4 @@ mod tests { assert_eq!(vs.sample(), c2); assert!(vs.finish().is_ok()); } - } diff --git a/crates/flock/src/blake3.rs b/crates/flock/src/blake3.rs index febaaba4..b79c7066 100644 --- a/crates/flock/src/blake3.rs +++ b/crates/flock/src/blake3.rs @@ -1,4 +1,4 @@ -// Credit: https://github.com/succinctlabs/flock (flock-prover), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-prover), MIT OR Apache-2.0. //! Monolithic BLAKE3 compression-function R1CS — one R1CS instance per //! `compress(cv, m, counter, block_len, flags) → state[16]` call. Encodes //! the 16-word state init, all 7 rounds (8 G's per round + the message @@ -97,11 +97,9 @@ //! openings at fixed indices pin them to claimed memory and bytecode values. use crate::blake3_witness::{BitRecord, add_carry_parts, or_bit_at, or_u32_at_bit, xor_dedup}; -use pcs::{ProverState, VerifierState}; -use primitives::field::F128; -use pcs::Commitment; use crate::r1cs::{BlockR1cs, SparseBinaryMatrix}; use crate::verifier; +use primitives::field::F192; // --------------------------------------------------------------------------- // Public constants @@ -156,16 +154,8 @@ pub const G_LANES: [[usize; 4]; N_G_PER_ROUND] = [ /// Message-index pairs `(mx, my)` consumed by G index `g` within a round, /// indexing into the (already-permuted) per-round message buffer. -pub const G_MSG_IDX: [[usize; 2]; N_G_PER_ROUND] = [ - [0, 1], - [2, 3], - [4, 5], - [6, 7], - [8, 9], - [10, 11], - [12, 13], - [14, 15], -]; +pub const G_MSG_IDX: [[usize; 2]; N_G_PER_ROUND] = + [[0, 1], [2, 3], [4, 5], [6, 7], [8, 9], [10, 11], [12, 13], [14, 15]]; // --------------------------------------------------------------------------- // Layout positions (bit indices into the per-block z slice of length K) @@ -272,13 +262,7 @@ fn permute(m: &mut [u32; 16]) { /// BLAKE3 compression function. Returns the full 16-word output state /// (post-finalization XOR). For chaining, the new CV is `out[0..8]`. -pub fn blake3_compress( - cv: &[u32; 8], - block_words: &[u32; 16], - counter: u64, - block_len: u32, - flags: u32, -) -> [u32; 16] { +pub fn blake3_compress(cv: &[u32; 8], block_words: &[u32; 16], counter: u64, block_len: u32, flags: u32) -> [u32; 16] { let counter_low = counter as u32; let counter_high = (counter >> 32) as u32; let mut state = [ @@ -524,59 +508,23 @@ pub fn build_matrices() -> (SparseBinaryMatrix, SparseBinaryMatrix) { let my = Word::from_slot_base(m_bit(my_idx, 0)); // tmp_0 = a + b - let tmp_0 = write_add_carry_rows( - &mut a_rows, - &mut b_rows, - &a, - &b, - g_add_carry_bit(g, ADD_TMP0, 0), - ); + let tmp_0 = write_add_carry_rows(&mut a_rows, &mut b_rows, &a, &b, g_add_carry_bit(g, ADD_TMP0, 0)); // a_1 = tmp_0 + mx - let a_1 = write_add_carry_rows( - &mut a_rows, - &mut b_rows, - &tmp_0, - &mx, - g_add_carry_bit(g, ADD_A1, 0), - ); + let a_1 = write_add_carry_rows(&mut a_rows, &mut b_rows, &tmp_0, &mx, g_add_carry_bit(g, ADD_A1, 0)); // d_1 = rotr16(d ^ a_1) let d_1 = d.xor(&a_1).dedup().rotr(16); // c_1 = c + d_1 - let c_1 = write_add_carry_rows( - &mut a_rows, - &mut b_rows, - &c, - &d_1, - g_add_carry_bit(g, ADD_C1, 0), - ); + let c_1 = write_add_carry_rows(&mut a_rows, &mut b_rows, &c, &d_1, g_add_carry_bit(g, ADD_C1, 0)); // b_1 = rotr12(b ^ c_1) let b_1 = b.xor(&c_1).dedup().rotr(12); // tmp_1 = a_1 + b_1 - let tmp_1 = write_add_carry_rows( - &mut a_rows, - &mut b_rows, - &a_1, - &b_1, - g_add_carry_bit(g, ADD_TMP1, 0), - ); + let tmp_1 = write_add_carry_rows(&mut a_rows, &mut b_rows, &a_1, &b_1, g_add_carry_bit(g, ADD_TMP1, 0)); // a_2 = tmp_1 + my (= a_new — cascades) - let a_2 = write_add_carry_rows( - &mut a_rows, - &mut b_rows, - &tmp_1, - &my, - g_add_carry_bit(g, ADD_A2, 0), - ); + let a_2 = write_add_carry_rows(&mut a_rows, &mut b_rows, &tmp_1, &my, g_add_carry_bit(g, ADD_A2, 0)); // d_2 = rotr8(d_1 ^ a_2) let d_2 = d_1.xor(&a_2).dedup().rotr(8); // c_2 = c_1 + d_2 (= c_new — cascades) - let c_2 = write_add_carry_rows( - &mut a_rows, - &mut b_rows, - &c_1, - &d_2, - g_add_carry_bit(g, ADD_C2, 0), - ); + let c_2 = write_add_carry_rows(&mut a_rows, &mut b_rows, &c_1, &d_2, g_add_carry_bit(g, ADD_C2, 0)); // b_new = rotr7(b_1 ^ c_2) (materialized lin-id) let b_new_word = b_1.xor(&c_2).dedup().rotr(7); for i in 0..WORD_BITS { @@ -630,15 +578,245 @@ pub fn build_matrices() -> (SparseBinaryMatrix, SparseBinaryMatrix) { (to_mat(a_rows), to_mat(b_rows)) } +// --------------------------------------------------------------------------- +// Circuit-walk evaluation (flock §Circuit walking) +// +// Evaluates the two bilinear forms +// +// uᵀ A_0 w and uᵀ B_0 w +// +// for arbitrary row weights `u` and column weights `w` (length K each) by +// walking the UNSUBSTITUTED compression circuit forward: the same cascade +// `build_matrices` threads symbolically, evaluated over F192 values. A lane +// is a 32-vector of wire values; a committed slot contributes `w[slot]`, an +// intermediate wire the running linear combination. Row i's contribution +// `u[i]·⟨A_i, w⟩` / `u[i]·⟨B_i, w⟩` is accumulated exactly where +// `build_matrices` would emit that row, with `⟨row, w⟩` read off the threaded +// wire values. Cost: O(circuit) field ops (~50K muls), never the ~21M +// substituted nonzeros — and the matrices need not be materialized at all. +// This is what lets a verifier evaluate the matrix MLEs directly instead of +// paying the sparse-matrix cost (or deferring the claim). +// --------------------------------------------------------------------------- + +/// One lane's wire values: bit `i` of the word, as the F192 combination +/// `⟨lin_func_i, w⟩`. +type WireWord = [F192; WORD_BITS]; + +#[inline] +fn wire_from_slot_base(w: &[F192], base: usize) -> WireWord { + std::array::from_fn(|i| w[base + i]) +} + +/// Constant word: a set bit is the `[Z_CONST]` lin_func, a clear bit empty. +#[inline] +fn wire_from_const(w: &[F192], val: u32) -> WireWord { + std::array::from_fn(|i| { + if (val >> i) & 1 == 1 { + w[Z_CONST_POS] + } else { + F192::ZERO + } + }) +} + +#[inline] +fn wire_xor(x: &WireWord, y: &WireWord) -> WireWord { + std::array::from_fn(|i| x[i] + y[i]) +} + +#[inline] +fn wire_rotr(x: &WireWord, n: usize) -> WireWord { + std::array::from_fn(|i| x[(i + n) % WORD_BITS]) +} + +/// Pair of accumulators for the A-side and B-side bilinear forms, plus the +/// running sum of `u` over rows whose B-side is the single `[Z_CONST]` entry +/// (lin-id / free-input rows) — factored so those rows cost one B-side +/// F-addition instead of a multiplication each. +struct WalkAcc { + a: F192, + b: F192, + /// Σ u[row] over rows with `B_row = [Z_CONST]`; folded in once at the end + /// as `b += w[Z_CONST_POS] · u_bconst`. + u_bconst: F192, +} + +/// Walk one 32-bit ADD (mirror of `write_add_carry_rows` + `Word::add_sum`): +/// accumulate the 31 carry rows into `acc` and return the sum-bit wires. +/// +/// carry row cb+i: A = X[i] ⊕ cin[i], B = Y[i] ⊕ cin[i] +/// sum[i] = X[i] ⊕ Y[i] ⊕ cin[i] +/// +/// with `cin[i] = ⊕_{j WireWord { + let mut out = [F192::ZERO; WORD_BITS]; + let mut cin = F192::ZERO; + for i in 0..WORD_BITS { + let a_side = x[i] + cin; + let b_side = y[i] + cin; + out[i] = a_side + y[i]; + if i < CARRY_BITS_PER_ADD { + let ui = u[carry_base + i]; + acc.a += ui * a_side; + acc.b += ui * b_side; + cin += w[carry_base + i]; + } + } + out +} + +/// Walk 32 consecutive `lin_func · 1` rows (lin-id / out_lo / out_hi): +/// row base+i has `A = `, `B = [Z_CONST]`. +fn walk_lin_rows(acc: &mut WalkAcc, u: &[F192], vals: &WireWord, base: usize) { + for i in 0..WORD_BITS { + acc.a += u[base + i] * vals[i]; + acc.u_bconst += u[base + i]; + } +} + +/// `(uᵀ A_0 w, uᵀ B_0 w)` by the forward circuit walk — the exact matrices +/// [`build_matrices`] emits, never materialized. +pub fn bilinear_walk_pair(u: &[F192], w: &[F192]) -> (F192, F192) { + assert_eq!(u.len(), K); + assert_eq!(w.len(), K); + let wc = w[Z_CONST_POS]; + let mut acc = WalkAcc { + a: F192::ZERO, + b: F192::ZERO, + u_bconst: F192::ZERO, + }; + // Σ u[row] over rows with A = B = [Z_CONST] (just the constant row now + // that every compression input is a free row): folded in at the end on + // both sides. + let u_abconst = u[Z_CONST_POS]; + + // Free-input rows for the 512 message bits: A = [slot], B = [Z_CONST]. + for j in 0..16 * WORD_BITS { + let s = M_BASE + j; + acc.a += u[s] * w[s]; + acc.u_bconst += u[s]; + } + + // Free-input rows for the 256 chaining-value bits and the 128 metadata + // bits (counter lo/hi, block_len, flags): A = [slot], B = [Z_CONST] — the + // same shape as the message bits (the generalized circuit no longer pins + // them to constants; the embedding protocol binds them instead). + for j in 0..8 * WORD_BITS { + let s = CV_BASE + j; + acc.a += u[s] * w[s]; + acc.u_bconst += u[s]; + } + for base in [T_LO_BASE, T_HI_BASE, BLEN_BASE, FLAGS_BASE] { + for j in 0..WORD_BITS { + let s = base + j; + acc.a += u[s] * w[s]; + acc.u_bconst += u[s]; + } + } + + // The G cascade, over wire values (mirrors `initial_lane_words`). + let msg_idx = per_round_msg_idx(); + let mut state: [WireWord; 16] = std::array::from_fn(|_| [F192::ZERO; WORD_BITS]); + for wd in 0..8 { + state[wd] = wire_from_slot_base(w, cv_bit(wd, 0)); + } + for i in 0..4 { + state[8 + i] = wire_from_const(w, BLAKE3_IV[i]); + } + state[12] = wire_from_slot_base(w, T_LO_BASE); + state[13] = wire_from_slot_base(w, T_HI_BASE); + state[14] = wire_from_slot_base(w, BLEN_BASE); + state[15] = wire_from_slot_base(w, FLAGS_BASE); + + for r in 0..N_ROUNDS { + for g_in_round in 0..N_G_PER_ROUND { + let g = r * N_G_PER_ROUND + g_in_round; + let [la, lb, lc, ld] = G_LANES[g_in_round]; + let [mx_idx, my_idx] = msg_idx[r][g_in_round]; + let (a, b, c, d) = (state[la], state[lb], state[lc], state[ld]); + let mx = wire_from_slot_base(w, m_bit(mx_idx, 0)); + let my = wire_from_slot_base(w, m_bit(my_idx, 0)); + + let tmp_0 = walk_add(&mut acc, u, w, &a, &b, g_add_carry_bit(g, ADD_TMP0, 0)); + let a_1 = walk_add(&mut acc, u, w, &tmp_0, &mx, g_add_carry_bit(g, ADD_A1, 0)); + let d_1 = wire_rotr(&wire_xor(&d, &a_1), 16); + let c_1 = walk_add(&mut acc, u, w, &c, &d_1, g_add_carry_bit(g, ADD_C1, 0)); + let b_1 = wire_rotr(&wire_xor(&b, &c_1), 12); + let tmp_1 = walk_add(&mut acc, u, w, &a_1, &b_1, g_add_carry_bit(g, ADD_TMP1, 0)); + let a_2 = walk_add(&mut acc, u, w, &tmp_1, &my, g_add_carry_bit(g, ADD_A2, 0)); + let d_2 = wire_rotr(&wire_xor(&d_1, &a_2), 8); + let c_2 = walk_add(&mut acc, u, w, &c_1, &d_2, g_add_carry_bit(g, ADD_C2, 0)); + let b_new = wire_rotr(&wire_xor(&b_1, &c_2), 7); + walk_lin_rows(&mut acc, u, &b_new, g_lin_bit(g, LIN_B_NEW, 0)); + walk_lin_rows(&mut acc, u, &d_2, g_lin_bit(g, LIN_D_NEW, 0)); + + state[la] = a_2; + state[lb] = wire_from_slot_base(w, g_lin_bit(g, LIN_B_NEW, 0)); + state[lc] = c_2; + state[ld] = wire_from_slot_base(w, g_lin_bit(g, LIN_D_NEW, 0)); + } + } + + // Finalization rows: out_lo[w] = state[w] ⊕ state[w+8], + // out_hi[w] = state[w+8] ⊕ cv[w]. Padding rows are empty: no contribution. + for wd in 0..8 { + let lo = wire_xor(&state[wd], &state[wd + 8]); + walk_lin_rows(&mut acc, u, &lo, out_lo_bit(wd, 0)); + let cv_w = wire_from_slot_base(w, cv_bit(wd, 0)); + let hi = wire_xor(&state[wd + 8], &cv_w); + walk_lin_rows(&mut acc, u, &hi, out_hi_bit(wd, 0)); + } + + // Fold in the factored constant-B and constant-A/B row sums. + (acc.a + wc * u_abconst, acc.b + wc * (acc.u_bconst + u_abconst)) +} + +/// `α·(uᵀ A_0 w) + (uᵀ B_0 w)` — the α-batched form lincheck's verifier +/// consumes, by one circuit walk. +pub fn bilinear_walk(alpha: F192, u: &[F192], w: &[F192]) -> F192 { + let (va, vb) = bilinear_walk_pair(u, w); + alpha * va + vb +} + +/// Walk-capable [`crate::lincheck::LincheckCircuit`] over the BLAKE3 R1CS: +/// `bilinear_form` answers lincheck's verifier in O(circuit) field ops via +/// [`bilinear_walk`], so `lincheck::verify` never materializes the +/// ~21M-nonzero substituted matrices' column marginal. The prover-side +/// `fold_alpha_batched` delegates to the (lazily built) CSC fold — the +/// verifier's fast path never calls it. +pub struct WalkLincheckCircuit<'a> { + r1cs: &'a BlockR1cs, +} + +impl<'a> WalkLincheckCircuit<'a> { + pub fn new(r1cs: &'a BlockR1cs) -> Self { + Self { r1cs } + } +} + +impl crate::lincheck::LincheckCircuit for WalkLincheckCircuit<'_> { + fn n_cols(&self) -> usize { + K + } + fn const_pin_col(&self) -> Option { + self.r1cs.const_pin + } + fn fold_alpha_batched(&self, alpha: F192, eq_inner: &[F192]) -> Vec { + self.r1cs.csc_lincheck_circuit().fold_alpha_batched(alpha, eq_inner) + } + fn bilinear_form(&self, alpha: F192, u: &[F192], w: &[F192]) -> Option { + Some(bilinear_walk(alpha, u, w)) + } +} + /// [`BlockR1cs::family_digest`] of this module's circuit, baked as a constant: /// recomputing it means building and hashing ~21M matrix entries (~300 ms), /// which embedding protocols would otherwise pay inside their first prove. /// The `family_digest_matches_baked` test recomputes and compares — a circuit /// change fails it until this constant is updated alongside. pub const FAMILY_DIGEST: [u8; 32] = [ - 0xaf, 0xed, 0x74, 0x72, 0xc6, 0xf7, 0x71, 0xa8, 0x57, 0x59, 0x92, 0x72, 0xff, 0x33, 0xa4, - 0xda, 0x86, 0xb2, 0x1f, 0x26, 0x00, 0xf0, 0x57, 0xfa, 0x0d, 0xa7, 0x97, 0xd1, 0x58, 0x63, - 0xeb, 0x58, + 0xaf, 0xed, 0x74, 0x72, 0xc6, 0xf7, 0x71, 0xa8, 0x57, 0x59, 0x92, 0x72, 0xff, 0x33, 0xa4, 0xda, 0x86, 0xb2, 0x1f, + 0x26, 0x00, 0xf0, 0x57, 0xfa, 0x0d, 0xa7, 0x97, 0xd1, 0x58, 0x63, 0xeb, 0x58, ]; /// Build a [`BlockR1cs`] batching `2^n_blocks_log` independent BLAKE3 @@ -692,13 +870,7 @@ fn write_word(z: &mut [bool], base: usize, val: u32) { } /// Build the witness block for ONE compression. Length = `K`. -pub fn build_block_witness( - cv: &[u32; 8], - m: &[u32; 16], - counter: u64, - block_len: u32, - flags: u32, -) -> Vec { +pub fn build_block_witness(cv: &[u32; 8], m: &[u32; 16], counter: u64, block_len: u32, flags: u32) -> Vec { let mut z = vec![false; K]; z[Z_CONST_POS] = true; // Inputs. @@ -755,8 +927,7 @@ pub fn build_block_witness( let d_1 = (d ^ a_1).rotate_right(16); let c_1 = add_with_witness_carry_only(c, d_1, &mut z, g_add_carry_bit(g, ADD_C1, 0)); let b_1 = (b ^ c_1).rotate_right(12); - let tmp_1 = - add_with_witness_carry_only(a_1, b_1, &mut z, g_add_carry_bit(g, ADD_TMP1, 0)); + let tmp_1 = add_with_witness_carry_only(a_1, b_1, &mut z, g_add_carry_bit(g, ADD_TMP1, 0)); let a_2 = add_with_witness_carry_only(tmp_1, my, &mut z, g_add_carry_bit(g, ADD_A2, 0)); let d_2 = (d_1 ^ a_2).rotate_right(8); let c_2 = add_with_witness_carry_only(c_1, d_2, &mut z, g_add_carry_bit(g, ADD_C2, 0)); @@ -837,7 +1008,7 @@ pub fn generate_witness(blocks: &[Compression], n_blocks_log: usize) -> Vec ( - Vec, - Vec, - Vec, + Vec, + Vec, + Vec, ) { - use primitives::field::F128; + use primitives::field::F192; use rayon::prelude::*; let n_total = 1usize << n_blocks_log; let n_blocks = blocks.len(); @@ -1035,39 +1207,36 @@ pub fn generate_witness_with_ab_packed( "{n_blocks} compressions > 2^{n_blocks_log} = {n_total} slots" ); - const F128_PER_BLOCK: usize = K / 128; - let total_f128 = n_total * F128_PER_BLOCK; - let mut z = vec![F128::ZERO; total_f128]; - let mut a = vec![F128::ZERO; total_f128]; - let mut b = vec![F128::ZERO; total_f128]; + const PACKED_PER_BLOCK: usize = K / 128; + let total_packed = n_total * PACKED_PER_BLOCK; + let mut z = vec![F192::ZERO; total_packed]; + let mut a = vec![F192::ZERO; total_packed]; + let mut b = vec![F192::ZERO; total_packed]; // Constant-wire pin (see lincheck's `LincheckCircuit::const_pin_col`): padding slots get the pinned // compression of the all-zero message (constant wire = 1), matching // [`generate_witness_with_ab_packed_and_lincheck`]. let padding = padding_block(); - z.par_chunks_mut(F128_PER_BLOCK) - .zip(a.par_chunks_mut(F128_PER_BLOCK)) - .zip(b.par_chunks_mut(F128_PER_BLOCK)) + z.par_chunks_mut(PACKED_PER_BLOCK) + .zip(a.par_chunks_mut(PACKED_PER_BLOCK)) + .zip(b.par_chunks_mut(PACKED_PER_BLOCK)) .enumerate() .for_each(|(idx, ((z_c, a_c), b_c))| { - let (cv, m, t, bl, fl) = if idx < n_blocks { - &blocks[idx] - } else { - &padding - }; - // SAFETY: F128 is repr(C, align(16)) with LE u64 halves — same - // byte layout as a u64 pair. - let z_u64: &mut [u64] = unsafe { - std::slice::from_raw_parts_mut(z_c.as_mut_ptr() as *mut u64, z_c.len() * 2) - }; - let a_u64: &mut [u64] = unsafe { - std::slice::from_raw_parts_mut(a_c.as_mut_ptr() as *mut u64, a_c.len() * 2) - }; - let b_u64: &mut [u64] = unsafe { - std::slice::from_raw_parts_mut(b_c.as_mut_ptr() as *mut u64, b_c.len() * 2) - }; - build_block_witness_ab_packed_into(cv, m, *t, *bl, *fl, z_u64, a_u64, b_u64); + let (cv, m, t, bl, fl) = if idx < n_blocks { &blocks[idx] } else { &padding }; + let mut z_u64 = vec![0u64; z_c.len() * 2]; + let mut a_u64 = vec![0u64; a_c.len() * 2]; + let mut b_u64 = vec![0u64; b_c.len() * 2]; + build_block_witness_ab_packed_into(cv, m, *t, *bl, *fl, &mut z_u64, &mut a_u64, &mut b_u64); + for (dst, words) in z_c.iter_mut().zip(z_u64.chunks_exact(2)) { + *dst = F192::new(words[0], words[1], 0); + } + for (dst, words) in a_c.iter_mut().zip(a_u64.chunks_exact(2)) { + *dst = F192::new(words[0], words[1], 0); + } + for (dst, words) in b_c.iter_mut().zip(b_u64.chunks_exact(2)) { + *dst = F192::new(words[0], words[1], 0); + } }); (z, a, b) @@ -1090,9 +1259,9 @@ pub fn generate_witness_with_ab_packed_and_lincheck( blocks: &[Compression], n_blocks_log: usize, ) -> ( - Vec, - Vec, - Vec, + Vec, + Vec, + Vec, Vec, ) { // Constant-wire pin (see lincheck's `LincheckCircuit::const_pin_col`): fill padding blocks with the @@ -1112,15 +1281,26 @@ pub fn generate_witness_with_ab_packed_and_lincheck( ) } +/// Serialize the 128-bit packed-witness subspace of F192. The third limb is +/// constrained to zero by construction and is not part of Flock's bit cube. +fn packed_128_bytes(words: &[F192]) -> Vec { + let mut out = Vec::with_capacity(words.len() * 16); + for word in words { + debug_assert_eq!(word.c2, 0, "packed Flock witness escaped 128-bit subspace"); + out.extend_from_slice(&word.c0.to_le_bytes()); + out.extend_from_slice(&word.c1.to_le_bytes()); + } + out +} + // --------------------------------------------------------------------------- // Convenience API: Blake3Setup // --------------------------------------------------------------------------- -/// Bundles the monolithic BLAKE3 compression R1CS sized for `n_blocks` -/// compressions. +/// Bundles the monolithic BLAKE3 compression R1CS for the smallest supported +/// power-of-two shape that can hold `n_blocks` compressions. #[derive(Clone, Debug)] pub struct Blake3Setup { - pub n_blocks: usize, pub r1cs: BlockR1cs, } @@ -1135,7 +1315,7 @@ impl Blake3Setup { // the prove-cycle scratch buffers (see scratch::prewarm_prover). r1cs.csc_lincheck_circuit(); primitives::scratch::prewarm_prover(r1cs.m); - Self { n_blocks, r1cs } + Self { r1cs } } pub fn m(&self) -> usize { @@ -1157,6 +1337,7 @@ impl Blake3Setup { mod tests { use super::*; use crate::test_rng::Rng; + use primitives::pretty_integer; #[test] fn family_digest_matches_baked() { @@ -1167,6 +1348,97 @@ mod tests { ); } + /// Timing: the three ways the native verifier can evaluate the A_0/B_0 + /// bilinear forms. Run with + /// `cargo test --release -p flock bench_bilinear -- --ignored --nocapture`. + #[test] + #[ignore] + fn bench_bilinear_walk_vs_matrices() { + let mut rng = Rng::new(0xBE9C); + let u: Vec = rng.ext_vec(K); + let w: Vec = rng.ext_vec(K); + let alpha = rng.ext(); + + // One-time setup costs the sparse paths pay (process-cached in prod, + // but real for a one-shot native verifier). + let t = std::time::Instant::now(); + let (ma, mb) = (build_matrices().0, build_matrices().1); + println!("build_matrices (×2 redundant here): {:?}", t.elapsed()); + let nnz: usize = + ma.rows.iter().map(|r| r.len()).sum::() + mb.rows.iter().map(|r| r.len()).sum::(); + println!("total nonzeros (A_0 + B_0): {}", pretty_integer(nnz)); + let r1cs = build_block_r1cs(3); + let t = std::time::Instant::now(); + let csc = r1cs.csc_lincheck_circuit(); + println!("CSC transpose build: {:?}", t.elapsed()); + + // (a) check_reduced-style naive contraction, both matrices. + let contract = |m: &SparseBinaryMatrix| -> F192 { + let mut acc = F192::ZERO; + for (i, row) in m.rows.iter().enumerate() { + let s = row.iter().map(|&j| w[j]).fold(F192::ZERO, |a, x| a + x); + acc += u[i] * s; + } + acc + }; + let t = std::time::Instant::now(); + let (da, db) = (contract(&ma), contract(&mb)); + let t_naive = t.elapsed(); + println!("naive sparse contraction (A + B): {t_naive:?}"); + + // (b) lincheck-verifier-style CSC marginal + inner product. + use crate::lincheck::LincheckCircuit; + let t = std::time::Instant::now(); + let marginal = csc.fold_alpha_batched(alpha, &u); + let form_csc = pcs::ring_switch::inner_product_ext(&marginal, &w); + let t_csc = t.elapsed(); + println!("CSC marginal fold + inner product: {t_csc:?}"); + + // (c) the circuit walk. + let t = std::time::Instant::now(); + let (wa, wb) = bilinear_walk_pair(&u, &w); + let t_walk = t.elapsed(); + println!("bilinear_walk_pair: {t_walk:?}"); + + assert_eq!((wa, wb), (da, db)); + assert_eq!(alpha * wa + wb, form_csc); + println!( + "speedup: {:.1}× vs naive, {:.1}× vs CSC", + t_naive.as_secs_f64() / t_walk.as_secs_f64(), + t_csc.as_secs_f64() / t_walk.as_secs_f64() + ); + } + + /// The circuit walk computes the same bilinear forms as the materialized + /// matrices, for fully random (unstructured) row/column weights: any + /// missing, extra, or misplaced row contribution would break equality. + #[test] + fn bilinear_walk_matches_matrices() { + let (ma, mb) = matrices(); + let mut rng = Rng::new(0xC12C); + for trial in 0..3 { + let alpha = rng.ext(); + let u: Vec = rng.ext_vec(K); + let w: Vec = rng.ext_vec(K); + let contract = |m: &SparseBinaryMatrix| -> F192 { + m.rows + .iter() + .enumerate() + .map(|(i, row)| u[i] * row.iter().map(|&j| w[j]).fold(F192::ZERO, |acc, x| acc + x)) + .fold(F192::ZERO, |acc, x| acc + x) + }; + let (direct_a, direct_b) = (contract(ma), contract(mb)); + let (walk_a, walk_b) = bilinear_walk_pair(&u, &w); + assert_eq!(walk_a, direct_a, "A-side, trial {trial}"); + assert_eq!(walk_b, direct_b, "B-side, trial {trial}"); + assert_eq!( + bilinear_walk(alpha, &u, &w), + alpha * direct_a + direct_b, + "alpha-batched, trial {trial}" + ); + } + } + /// BLAKE3 chunk flags (subset). const CHUNK_START: u32 = 1 << 0; const CHUNK_END: u32 = 1 << 1; @@ -1193,13 +1465,7 @@ mod tests { /// (a single root-block, single-chunk, ROOT-flagged compression). #[test] fn compress_matches_blake3_crate_empty() { - let state = blake3_compress( - &BLAKE3_IV, - &[0u32; 16], - 0, - 0, - CHUNK_START | CHUNK_END | ROOT, - ); + let state = blake3_compress(&BLAKE3_IV, &[0u32; 16], 0, 0, CHUNK_START | CHUNK_END | ROOT); let mut got = [0u8; 32]; for w in 0..8 { got[w * 4..w * 4 + 4].copy_from_slice(&state[w].to_le_bytes()); @@ -1266,15 +1532,18 @@ mod tests { .map(|_| { let cv: [u32; 8] = std::array::from_fn(|_| rng.next_u32()); let m: [u32; 16] = std::array::from_fn(|_| rng.next_u32()); - (cv, m, rng.next_u32() as u64 | ((rng.next_u32() as u64) << 32), rng.next_u32() % 65, rng.next_u32()) + ( + cv, + m, + rng.next_u32() as u64 | ((rng.next_u32() as u64) << 32), + rng.next_u32() % 65, + rng.next_u32(), + ) }) .collect(); let z = generate_witness(&blocks, n_log); assert_eq!(z.len(), r1cs.n()); - assert!( - r1cs.satisfies(&z), - "witness for {n_blocks} compressions fails R1CS" - ); + assert!(r1cs.satisfies(&z), "witness for {n_blocks} compressions fails R1CS"); } } @@ -1288,10 +1557,7 @@ mod tests { assert!(r1cs.satisfies(&z)); // Flip a carry_aux bit inside G #10 (middle of round 1). z[g_add_carry_bit(10, ADD_A2, 5)] ^= true; - assert!( - !r1cs.satisfies(&z), - "tampered carry bit should violate R1CS" - ); + assert!(!r1cs.satisfies(&z), "tampered carry bit should violate R1CS"); } /// The fused generator produces (z, a, b) byte-identical to @@ -1313,8 +1579,7 @@ mod tests { let (z1, a1, b1) = generate_witness_with_ab_packed(&blocks, n_log); let lincheck_ref = pack_z_lincheck_from_packed(&z1, r1cs.m, r1cs.k_log); - let (z2, a2, b2, lincheck_new) = - generate_witness_with_ab_packed_and_lincheck(&blocks, n_log); + let (z2, a2, b2, lincheck_new) = generate_witness_with_ab_packed_and_lincheck(&blocks, n_log); assert_eq!(z1, z2, "z mismatch at n_blocks={n_blocks}"); assert_eq!(a1, a2, "a mismatch at n_blocks={n_blocks}"); assert_eq!(b1, b2, "b mismatch at n_blocks={n_blocks}"); @@ -1327,9 +1592,7 @@ mod tests { #[test] fn setup_sizes_correctly() { - for &(n_blocks, expected_n_log) in - &[(1usize, 3), (8, 3), (9, 4), (16, 4), (17, 5), (1000, 10)] - { + for &(n_blocks, expected_n_log) in &[(1usize, 3), (8, 3), (9, 4), (16, 4), (17, 5), (1000, 10)] { let setup = Blake3Setup::new(n_blocks); assert_eq!(setup.n_blocks_log(), expected_n_log, "n_blocks={n_blocks}"); assert_eq!(setup.m(), K_LOG + expected_n_log); @@ -1349,11 +1612,10 @@ mod tests { let inner_rest_len = r1cs.k_log - r1cs.k_skip; // Correctly-shaped buffers (padding-only generation), then zeroed. - let (mut z, mut a, mut b, mut zlc) = - generate_witness_with_ab_packed_and_lincheck(&[], setup.n_blocks_log()); - z.fill(F128::ZERO); - a.fill(F128::ZERO); - b.fill(F128::ZERO); + let (mut z, mut a, mut b, mut zlc) = generate_witness_with_ab_packed_and_lincheck(&[], setup.n_blocks_log()); + z.fill(F192::ZERO); + a.fill(F192::ZERO); + b.fill(F192::ZERO); zlc.fill(0); // Prover side: the reduction happily runs on the zero witness. @@ -1361,20 +1623,13 @@ mod tests { k_log: r1cs.k_log, useful_bits_per_block: r1cs.useful_bits, }; - let as_bytes = |v: &[F128]| unsafe { - std::slice::from_raw_parts( - v.as_ptr() as *const u8, - std::mem::size_of_val(v), - ) - }; + let a_bytes = packed_128_bytes(&a); + let b_bytes = packed_128_bytes(&b); + let z_bytes = packed_128_bytes(&z); let mut ps = pcs::ProverState::new(b"const-pin-poc", &[]); - let (zc_claim, _s_hat_v_c) = crate::zerocheck::prove_packed_padded( - as_bytes(&a), - as_bytes(&b), - as_bytes(&z), // C = I, so c == z - r1cs.m, - &padding, - &mut ps, + let (zc_claim, _s_hat_v_c) = crate::zerocheck::prove_packed_padded_capture_s_hat_v_c( + &a_bytes, &b_bytes, &z_bytes, // C = I, so c == z + r1cs.m, &padding, &mut ps, ); let x_ab = crate::lincheck::QuirkyPoint { z_skip: zc_claim.z, @@ -1395,8 +1650,7 @@ mod tests { // Verifier side: zerocheck accepts, the lincheck const-wire pin rejects. let mut vs = pcs::VerifierState::new(b"const-pin-poc", &proof_t, &[]); - let zc = crate::zerocheck::verify(r1cs.m, &mut vs) - .expect("zerocheck accepts the all-zero witness"); + let zc = crate::zerocheck::verify(r1cs.m, &mut vs).expect("zerocheck accepts the all-zero witness"); let x_ab_v = crate::lincheck::QuirkyPoint { z_skip: zc.z, x_inner_rest: zc.mlv_challenges[..inner_rest_len].to_vec(), @@ -1413,19 +1667,14 @@ mod tests { &mut vs, ); assert!( - matches!( - res, - Err(crate::lincheck::VerifyError::ConsistencyFailed { .. }) - ), + matches!(res, Err(crate::lincheck::VerifyError::ConsistencyFailed { .. })), "all-zero witness must be rejected by the constant-wire pin; got {res:?}" ); } } -// ===== leanVM-b stacked BLAKE3 reduction (grafted) ===== -// (No Blake3StackProof struct: the zerocheck / lincheck / ring-switch scalars -// ride the shared transcript stream, and the one hash-bearing Ligerito rides -// the caller's opening channel.) +// The zerocheck, lincheck, and ring-switch scalars use the shared transcript; +// the caller carries the Ligerito opening. /// One claim on the committed packed BLAKE3 witness `q_pkd`, as left by the /// Flock reduction and handed to the PCS. `claim` is the `ẑ(point) = value` @@ -1435,7 +1684,7 @@ mod tests { #[derive(Clone, Debug)] pub struct WitnessClaim { pub claim: crate::proof::ZClaim, - pub s_hat_v: Option>, + pub s_hat_v: Option>, } /// The two claims on the committed witness `q_pkd` left by the Flock BLAKE3 @@ -1446,7 +1695,7 @@ pub struct WitnessClaim { /// This is the clean seam between Flock's reduction and the PCS: the reduction /// produces these; the PCS opens them (see [`Blake3Setup::prove_reduction`]). #[derive(Clone, Debug)] -pub struct ReducedClaims { +pub struct PackedWitnessClaims { pub ab: WitnessClaim, pub c: WitnessClaim, } @@ -1461,17 +1710,6 @@ pub struct ReductionReplay { pub lc_claim: crate::lincheck::LincheckClaim, } -/// Construct a multilinear `x_outer_full` of length `m − k_skip` from a -/// QuirkyPoint: concatenate `x_inner_rest` and `x_outer`. This is the format -/// the PCS expects (k_skip = 6 absorbed via `z_skip`; everything else is -/// multilinear). -fn quirky_x_outer_full(point: &crate::lincheck::QuirkyPoint) -> Vec { - let mut v = Vec::with_capacity(point.x_inner_rest.len() + point.x_outer.len()); - v.extend_from_slice(&point.x_inner_rest); - v.extend_from_slice(&point.x_outer); - v -} - impl Blake3Setup { /// **Flock reduction (prover).** Run the BLAKE3 zerocheck and lincheck on /// the shared transcript, reducing R1CS validity of `blocks` to two @@ -1479,21 +1717,24 @@ impl Blake3Setup { /// statement is already transcript-bound: the embedding protocol seeds /// with the circuit family digest and announces the count.) Returns: /// - `z_packed`: the regenerated packed witness the PCS later opens against; - /// - the [`ReducedClaims`] `(ab, c)` on `q_pkd`, with ring-switch weights. + /// - the [`PackedWitnessClaims`] `(ab, c)` on `q_pkd`, with ring-switch weights. /// /// Does NOT open the PCS; the caller discharges the returned claims in the - /// one stacked opening (`lean_vm`'s `pcs::open`, or - /// [`Self::prove_validity_stacked`] for a standalone roundtrip). - pub fn prove_reduction( + /// one stacked opening (`lean_vm`'s `pcs::open`). + pub fn prove_reduction( &self, blocks: &[Compression], - stack_commitment: &Commitment, - ps: &mut ProverState, - ) -> (Vec, ReducedClaims) { - assert_eq!(blocks.len(), self.n_blocks); + ps: &mut fiat_shamir::transcript::ProverState, + ) -> (Vec, PackedWitnessClaims) { + assert!( + blocks.len() <= self.n_block_slots(), + "{} compressions exceed this setup's {} slots", + blocks.len(), + self.n_block_slots() + ); let n_log = self.n_blocks_log(); let t_witness = std::time::Instant::now(); - let (z_packed, a_packed_f128, b_packed_f128, z_packed_lincheck) = + let (z_packed, a_packed_words, b_packed_words, z_packed_lincheck) = generate_witness_with_ab_packed_and_lincheck(blocks, n_log); if std::env::var_os("FLOCK_PROVE_TRACE").is_some() { eprintln!( @@ -1501,43 +1742,33 @@ impl Blake3Setup { t_witness.elapsed().as_secs_f64() * 1e3, ); } - let reduced = self.prove_reduction_precomputed( - &z_packed, - &a_packed_f128, - &b_packed_f128, - &z_packed_lincheck, - ps, - ); - // The embedding protocol has already transcript-bound the commitment. - let _ = stack_commitment; + let reduced = + self.prove_reduction_precomputed(&z_packed, &a_packed_words, &b_packed_words, &z_packed_lincheck, ps); (z_packed, reduced) } /// **Flock reduction from a prepared witness (prover).** This is the /// witness-generation-free counterpart of [`Self::prove_reduction`] for - /// embedders that already generated `q_pkd` together with its `A·z`, `B·z`, - /// and lincheck-stripe buffers before committing it. Reusing those buffers - /// avoids repeating the fused witness pass after commitment. - pub fn prove_reduction_precomputed( + /// embedders that already generated the packed `z`, `A·z`, `B·z`, and + /// lincheck-stripe buffers before committing the flattened witness. + pub fn prove_reduction_precomputed( &self, - z_packed: &[F128], - a_packed_f128: &[F128], - b_packed_f128: &[F128], + z_packed: &[F192], + a_packed_words: &[F192], + b_packed_words: &[F192], z_packed_lincheck: &[u8], - ps: &mut ProverState, - ) -> ReducedClaims { + ps: &mut fiat_shamir::transcript::ProverState, + ) -> PackedWitnessClaims { let trace = std::env::var_os("FLOCK_PROVE_TRACE").is_some(); let t_reduction = std::time::Instant::now(); - let packed_len = 1usize << (self.r1cs.m - pcs::LOG_PACKING); + // The fused generator packs 128 Boolean coordinates in each F192 + // container; the third tower limb is constrained to zero. + let packed_len = 1usize << (self.r1cs.m - 7); assert_eq!(z_packed.len(), packed_len, "wrong packed witness length"); - assert_eq!(a_packed_f128.len(), packed_len, "wrong packed A·z length"); - assert_eq!(b_packed_f128.len(), packed_len, "wrong packed B·z length"); - assert_eq!( - z_packed_lincheck.len(), - packed_len * core::mem::size_of::(), - "wrong lincheck stripe length" - ); + assert_eq!(a_packed_words.len(), packed_len, "wrong packed A·z length"); + assert_eq!(b_packed_words.len(), packed_len, "wrong packed B·z length"); + assert_eq!(z_packed_lincheck.len(), packed_len * 16, "wrong lincheck stripe length"); // No bind_statement here: the embedding protocol (leanVM-b) seeds its // transcript with the circuit-FAMILY digest and binds the instance @@ -1550,26 +1781,16 @@ impl Blake3Setup { }; let t_zerocheck = std::time::Instant::now(); let (zc_claim, s_hat_v_c) = { - let a_packed: &[u8] = unsafe { - std::slice::from_raw_parts( - a_packed_f128.as_ptr() as *const u8, - a_packed_f128.len() * core::mem::size_of::(), - ) - }; - let b_packed: &[u8] = unsafe { - std::slice::from_raw_parts( - b_packed_f128.as_ptr() as *const u8, - b_packed_f128.len() * core::mem::size_of::(), - ) - }; - let c_packed: &[u8] = unsafe { - std::slice::from_raw_parts( - z_packed.as_ptr() as *const u8, - z_packed.len() * core::mem::size_of::(), - ) - }; - crate::zerocheck::prove_packed_padded( - a_packed, b_packed, c_packed, self.r1cs.m, &padding, ps, + let a_packed = packed_128_bytes(a_packed_words); + let b_packed = packed_128_bytes(b_packed_words); + let c_packed = packed_128_bytes(z_packed); + crate::zerocheck::prove_packed_padded_capture_s_hat_v_c( + &a_packed, + &b_packed, + &c_packed, + self.r1cs.m, + &padding, + ps, ) }; let zerocheck_time = t_zerocheck.elapsed(); @@ -1609,18 +1830,21 @@ impl Blake3Setup { }, value: zc_claim.c_eval, }; - let s_hat_v_ab = if self.r1cs.k_log >= pcs::LOG_PACKING { - Some(pcs::ring_switch::s_hat_v_from_z_vec( - &z_vec_pre, - &lc_claim.r_inner_rest[1..], - )) + let s_hat_v_ab = if self.r1cs.k_log >= pcs::pack::LOG_PACKING { + Some(pcs::ring_switch::s_hat_v_from_z_vec(&z_vec_pre, &lc_claim.r_inner_rest)) } else { None }; - let reduced = ReducedClaims { - ab: WitnessClaim { claim: ab, s_hat_v: s_hat_v_ab }, - c: WitnessClaim { claim: c, s_hat_v: Some(s_hat_v_c) }, + let reduced = PackedWitnessClaims { + ab: WitnessClaim { + claim: ab, + s_hat_v: s_hat_v_ab, + }, + c: WitnessClaim { + claim: c, + s_hat_v: Some(s_hat_v_c), + }, }; if trace { let reduction_time = t_reduction.elapsed(); @@ -1636,132 +1860,32 @@ impl Blake3Setup { reduced } - /// Prove `blocks` are valid compressions in two clean phases: - /// 1. [`Self::prove_reduction`] — Flock zerocheck + lincheck → the `(ab, c)` - /// claims on the committed witness `q_pkd`; - /// 2. the PCS: discharge those claims *together with* the caller's own - /// `stack_pd` point claims in ONE stacked Ligerito open over `stack` (the - /// caller's committed witness, with `q_pkd` the aligned sub-block at - /// `stack_offset`). - /// - /// `stack_data`/`stack_commitment` are the caller's commit; the transcript - /// `sponge` is shared. - #[allow(clippy::too_many_arguments)] - pub fn prove_validity_stacked( - &self, - blocks: &[Compression], - stack: &[F128], - stack_offset: usize, - stack_data: &pcs::ProverData, - stack_commitment: &Commitment, - stack_pd: &[(Vec, F128)], - ps: &mut ProverState, - ) -> pcs::ligerito::LigeritoProof { - let trace = std::env::var_os("FLOCK_PROVE_TRACE").is_some(); - let t_total = std::time::Instant::now(); - - // Phase 1 — Flock reduction: zerocheck + lincheck → claims on q_pkd. - let t_reduction = std::time::Instant::now(); - let (z_packed, reduced) = self.prove_reduction(blocks, stack_commitment, ps); - let reduction_time = t_reduction.elapsed(); - debug_assert_eq!( - &stack[stack_offset..stack_offset + z_packed.len()], - z_packed.as_slice(), - "committed q_pkd slice must equal the regenerated packed witness" - ); - - // Phase 2 — PCS: discharge the reduction's claims (plus the caller's - // full-stack point claims) in one stacked open. - let t_open = std::time::Instant::now(); - let proof = self.discharge_reduction_stacked( - &z_packed, - &reduced, - stack, - stack_offset, - stack_data, - stack_commitment, - stack_pd, - ps, - ); - if trace { - eprintln!( - "[flock prove] stacked: {:.2} ms (reduction: {:.2} ms, open: {:.2} ms)", - t_total.elapsed().as_secs_f64() * 1e3, - reduction_time.as_secs_f64() * 1e3, - t_open.elapsed().as_secs_f64() * 1e3, - ); - } - proof - } - - /// Phase 2 of [`Self::prove_validity_stacked`]: the PCS open of the - /// reduction's `(ab, c)` claims on `q_pkd` (`z_packed`), lifted into the - /// caller's `stack` and batched with the caller's `stack_pd` point claims. - #[allow(clippy::too_many_arguments)] - pub fn discharge_reduction_stacked( - &self, - z_packed: &[F128], - reduced: &ReducedClaims, - stack: &[F128], - stack_offset: usize, - stack_data: &pcs::ProverData, - stack_commitment: &Commitment, - stack_pd: &[(Vec, F128)], - ps: &mut ProverState, - ) -> pcs::ligerito::LigeritoProof { - let padding = crate::zerocheck::PaddingSpec { - k_log: self.r1cs.k_log, - useful_bits_per_block: self.r1cs.useful_bits, - }; - let ab_x = quirky_x_outer_full(&reduced.ab.claim.point); - let c_x = quirky_x_outer_full(&reduced.c.claim.point); - // This standalone-flock path takes general full-stack point claims. - let pd: Vec = stack_pd - .iter() - .map(|(point, value)| pcs::StackClaim::Point { point, value: *value }) - .collect(); - let lig_config = stacked_lig_config(stack_commitment); - pcs::open_batch_mixed_ligerito_stacked( - z_packed, - &[ab_x.as_slice(), c_x.as_slice()], - &[reduced.ab.s_hat_v.as_deref(), reduced.c.s_hat_v.as_deref()], - &padding, - stack, - stack_offset, - stack_data, - stack_commitment, - &pd, - &lig_config, - ps, - ) - } - /// **Flock reduction (verifier).** Replay the BLAKE3 zerocheck and /// lincheck straight off the shared transcript stream, recovering the two /// `(ab, c)` evaluation claims on the committed witness `q_pkd`. Mirror of /// [`Self::prove_reduction`]; the PCS then discharges the returned claims. - pub fn verify_reduction( + pub fn verify_reduction( &self, - stack_commitment: &Commitment, - vs: &mut VerifierState<'_>, + vs: &mut fiat_shamir::transcript::VerifierState<'_, O>, ) -> Result { // Mirror of prove_reduction: the statement is bound by the embedding // protocol's seed (family digest) + announced count + commitment root. - let _ = stack_commitment; - let zc_claim = crate::zerocheck::verify(self.r1cs.m, vs) - .map_err(verifier::VerifyError::Zerocheck)?; + let zc_claim = crate::zerocheck::verify(self.r1cs.m, vs).map_err(verifier::VerifyError::Zerocheck)?; let inner_rest_len = self.r1cs.k_log - self.r1cs.k_skip; let x_ab = crate::lincheck::QuirkyPoint { z_skip: zc_claim.z, x_inner_rest: zc_claim.mlv_challenges[..inner_rest_len].to_vec(), x_outer: zc_claim.mlv_challenges[inner_rest_len..].to_vec(), }; + // Walk-capable circuit: the verifier's lincheck consistency check is + // one circuit walk (O(circuit) field ops) instead of the ∝ NNZ CSC + // marginal fold. Same transcript, same accept/reject. let lc_claim = crate::lincheck::verify( self.r1cs.m, self.r1cs.k_log, self.r1cs.k_skip, - self.r1cs.csc_lincheck_circuit(), + &WalkLincheckCircuit::new(&self.r1cs), &x_ab, zc_claim.a_eval, zc_claim.b_eval, @@ -1785,57 +1909,11 @@ impl Blake3Setup { }, value: zc_claim.c_eval, }; - Ok(ReductionReplay { ab, c, zc_claim, lc_claim }) - } - - /// Verifier mirror of [`Self::prove_validity_stacked`], in the same two - /// phases: (1) [`Self::verify_reduction`] replays zerocheck + lincheck to - /// recover the `(ab, c)` claims on `q_pkd`, then (2) the stacked Ligerito - /// opening of those claims (and the caller's `stack_pd`) is verified against - /// `stack_commitment`. `stack_offset` and the derived `qpkd_vars` locate - /// `q_pkd` inside the stack. - pub fn verify_validity_stacked( - &self, - stack_commitment: &Commitment, - stack_offset: usize, - stack_pd: &[(Vec, F128)], - open: &pcs::ligerito::LigeritoProof, - vs: &mut VerifierState<'_>, - ) -> Result<(), verifier::VerifyError> { - // Phase 1 — Flock reduction: replay zerocheck + lincheck → (ab, c). - let ReductionReplay { ab, c, .. } = self.verify_reduction(stack_commitment, vs)?; - - // Phase 2 — PCS: verify the stacked opening of (ab, c) + stack_pd. - let ab_x = quirky_x_outer_full(&ab.point); - let c_x = quirky_x_outer_full(&c.point); - let qpkd_vars = self.r1cs.m - pcs::LOG_PACKING; - let pd: Vec = stack_pd - .iter() - .map(|(point, value)| pcs::StackClaim::Point { point, value: *value }) - .collect(); - let lig_config = stacked_lig_config(stack_commitment); - pcs::verify_opening_batch_mixed_ligerito_stacked( - stack_commitment, - stack_offset, - qpkd_vars, - &[ab.value, c.value], - &[ab.point.z_skip, c.point.z_skip], - &[ab_x.as_slice(), c_x.as_slice()], - &pd, - open, - &lig_config, - vs, - ) - .map(|_| ()) - .map_err(verifier::VerifyError::Pcs) + Ok(ReductionReplay { + ab, + c, + zc_claim, + lc_claim, + }) } } - -/// The Ligerito config for a stacked open against -/// `stack_commitment` — derived from the commitment's own `(m, profile)` params, -/// so both sides agree by construction. -fn stacked_lig_config(stack_commitment: &Commitment) -> pcs::ligerito::LigeritoConfig { - pcs::ligerito::LigeritoSecurityConfig::derive_config(stack_commitment.params.m) - .and_then(|sec| sec.to_config()) - .expect("ligerito config for stacked open") -} diff --git a/crates/flock/src/blake3_witness.rs b/crates/flock/src/blake3_witness.rs index 56eeca32..8c0a4eb8 100644 --- a/crates/flock/src/blake3_witness.rs +++ b/crates/flock/src/blake3_witness.rs @@ -1,12 +1,12 @@ -// Credit: https://github.com/succinctlabs/flock (flock-prover), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-prover), MIT OR Apache-2.0. //! Bit-packing and R1CS-row helpers for the monolithic hash R1CS modules //! (only `blake3` in this vendored subset). use std::sync::OnceLock; -use primitives::bits::transpose_8_u64s_to_64_bytes; -use primitives::field::F128; use crate::r1cs::{BlockR1cs, SparseBinaryMatrix, WitnessLayout}; +use primitives::bits::transpose_8_u64s_to_64_bytes; +use primitives::field::F192; /// OR the low 32 bits of `val` into `buf` starting at bit-offset `bit_off`. /// Handles u64 straddling when `bit_off % 64 > 32`. @@ -122,15 +122,9 @@ pub(crate) fn build_block_r1cs_with_matrices( b_0: SparseBinaryMatrix, const_pin: Option, ) -> BlockR1cs { - assert!( - n_blocks_log >= 3, - "lincheck needs n_outer ≥ 8 — pick n_blocks_log ≥ 3" - ); + assert!(n_blocks_log >= 3, "lincheck needs n_outer ≥ 8 — pick n_blocks_log ≥ 3"); let k = 1usize << k_log; - assert!( - useful_bits <= k, - "useful_bits ({useful_bits}) must be ≤ 2^k_log ({k})" - ); + assert!(useful_bits <= k, "useful_bits ({useful_bits}) must be ≤ 2^k_log ({k})"); BlockR1cs { m: k_log + n_blocks_log, k_log, @@ -157,7 +151,7 @@ pub(crate) fn build_block_r1cs_with_matrices( /// Drive the parallel chunked witness build for `n_blocks` instances padded /// to `2^n_blocks_log` slots. Returns `(z, a, b, z_lincheck)` packed in -/// F128 form (z/a/b) and byte-stripe form (z_lincheck). +/// F192 form (z/a/b) and byte-stripe form (z_lincheck). /// /// `per_block(initial, z_u64, a_u64, b_u64)` populates one block's worth of /// `(z, a, b)` data — 3 zero-initialized `u64`-buffers of length `K / 64`. @@ -176,14 +170,14 @@ pub(crate) fn drive_witness_packed_and_lincheck( n_blocks_log: usize, k_log: usize, per_block: F, -) -> (Vec, Vec, Vec, Vec) +) -> (Vec, Vec, Vec, Vec) where F: Fn(&S, &mut [u64], &mut [u64], &mut [u64]) + Sync, { use rayon::prelude::*; let k = 1usize << k_log; - let f128_per_block = k / 128; + let packed_per_block = k / 128; let u64_per_block = k / 64; let n_total = 1usize << n_blocks_log; let n_blocks = initial_states.len(); @@ -196,35 +190,31 @@ where "lincheck stripe layout requires n_total ≥ 8 and divisible by 8" ); - let total_f128 = n_total * f128_per_block; + let total_packed = n_total * packed_per_block; // z/a/b are allocated uninitialized and zeroed *inside* the parallel loop // (one memset per 8-block group), so the ~192 MB zero-fill scales with the // thread count instead of running serially on the main thread before the // parallel build. The per-block builders OR 1-bits into pre-zeroed words, // so each group must be zeroed before its `per_block` calls. `z_lincheck` // stays `vec![0u8; _]` (lazy `alloc_zeroed`/mmap — no eager memset). - let mut z = primitives::scratch::take_f128(total_f128); - let mut a = primitives::scratch::take_f128(total_f128); - let mut b = primitives::scratch::take_f128(total_f128); + let mut z = primitives::scratch::take_f192(total_packed); + let mut a = primitives::scratch::take_f192(total_packed); + let mut b = primitives::scratch::take_f192(total_packed); let mut z_lincheck = vec![0u8; (n_total / 8) * k]; - z.par_chunks_mut(8 * f128_per_block) - .zip(a.par_chunks_mut(8 * f128_per_block)) - .zip(b.par_chunks_mut(8 * f128_per_block)) + z.par_chunks_mut(8 * packed_per_block) + .zip(a.par_chunks_mut(8 * packed_per_block)) + .zip(b.par_chunks_mut(8 * packed_per_block)) .zip(z_lincheck.par_chunks_mut(k)) .enumerate() .for_each(|(g, (((z_grp, a_grp), b_grp), stripe))| { - // Zero this group's z/a/b up front (parallel memset — the buffers - // were uninit-allocated). The per-block builder ORs 1-bits into - // pre-zeroed words; any slot left unbuilt (no padding block) stays - // zero, which the lincheck transpose below reads correctly. - // SAFETY: F128 is `Copy` (no Drop) and the all-zero bit pattern is - // the valid `F128::ZERO`, so a byte memset is a correct init. - unsafe { - std::ptr::write_bytes(z_grp.as_mut_ptr(), 0, z_grp.len()); - std::ptr::write_bytes(a_grp.as_mut_ptr(), 0, a_grp.len()); - std::ptr::write_bytes(b_grp.as_mut_ptr(), 0, b_grp.len()); - } + // The circuit witness remains 128-bit packed even though protocol + // scalars are F192. Build contiguous u64 pairs, then embed each + // pair as (lo, hi, 0); F192's 24-byte stride cannot be viewed as a + // contiguous u64-pair array. + let mut z_words = vec![0u64; 8 * u64_per_block]; + let mut a_words = vec![0u64; 8 * u64_per_block]; + let mut b_words = vec![0u64; 8 * u64_per_block]; for k_in in 0..8 { let global_idx = 8 * g + k_in; let init: &S = if global_idx < n_blocks { @@ -237,46 +227,34 @@ where // No padding block — leave this slot zero. continue; }; - let z_chunk = &mut z_grp[k_in * f128_per_block..(k_in + 1) * f128_per_block]; - let a_chunk = &mut a_grp[k_in * f128_per_block..(k_in + 1) * f128_per_block]; - let b_chunk = &mut b_grp[k_in * f128_per_block..(k_in + 1) * f128_per_block]; - // SAFETY: F128 is `repr(C, align(16))` with two `u64` fields in - // LE order — same byte layout as a u64 pair. - let z_u64: &mut [u64] = unsafe { - std::slice::from_raw_parts_mut( - z_chunk.as_mut_ptr() as *mut u64, - z_chunk.len() * 2, - ) - }; - let a_u64: &mut [u64] = unsafe { - std::slice::from_raw_parts_mut( - a_chunk.as_mut_ptr() as *mut u64, - a_chunk.len() * 2, - ) - }; - let b_u64: &mut [u64] = unsafe { - std::slice::from_raw_parts_mut( - b_chunk.as_mut_ptr() as *mut u64, - b_chunk.len() * 2, - ) - }; + let range = k_in * u64_per_block..(k_in + 1) * u64_per_block; + let z_u64 = &mut z_words[range.clone()]; + let a_u64 = &mut a_words[range.clone()]; + let b_u64 = &mut b_words[range]; per_block(init, z_u64, a_u64, b_u64); } + for (dst, words) in z_grp.iter_mut().zip(z_words.chunks_exact(2)) { + *dst = F192::new(words[0], words[1], 0); + } + for (dst, words) in a_grp.iter_mut().zip(a_words.chunks_exact(2)) { + *dst = F192::new(words[0], words[1], 0); + } + for (dst, words) in b_grp.iter_mut().zip(b_words.chunks_exact(2)) { + *dst = F192::new(words[0], words[1], 0); + } + // Bit-transpose 8 z chunks into the lincheck stripe. - let z_u64_all: &[u64] = unsafe { - std::slice::from_raw_parts(z_grp.as_ptr() as *const u64, z_grp.len() * 2) - }; for i in 0..u64_per_block { let lanes: [u64; 8] = [ - z_u64_all[i], - z_u64_all[u64_per_block + i], - z_u64_all[2 * u64_per_block + i], - z_u64_all[3 * u64_per_block + i], - z_u64_all[4 * u64_per_block + i], - z_u64_all[5 * u64_per_block + i], - z_u64_all[6 * u64_per_block + i], - z_u64_all[7 * u64_per_block + i], + z_words[i], + z_words[u64_per_block + i], + z_words[2 * u64_per_block + i], + z_words[3 * u64_per_block + i], + z_words[4 * u64_per_block + i], + z_words[5 * u64_per_block + i], + z_words[6 * u64_per_block + i], + z_words[7 * u64_per_block + i], ]; transpose_8_u64s_to_64_bytes(&lanes, &mut stripe[i * 64..i * 64 + 64]); } diff --git a/crates/flock/src/lib.rs b/crates/flock/src/lib.rs index 990c324d..d95d4a4b 100644 --- a/crates/flock/src/lib.rs +++ b/crates/flock/src/lib.rs @@ -1,4 +1,4 @@ -// Credit: https://github.com/succinctlabs/flock, MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock, MIT OR Apache-2.0. //! flock: a batched R1CS proving system for hash circuits over GF(2), reduced //! to evaluation claims on the committed packed witness. //! diff --git a/crates/flock/src/lincheck.rs b/crates/flock/src/lincheck.rs index 7ec8cb87..b7e023e6 100644 --- a/crates/flock/src/lincheck.rs +++ b/crates/flock/src/lincheck.rs @@ -1,4 +1,4 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. //! Lincheck PIOP for **block-diagonal** R1CS over GF(2). //! //! Reduces three MLE evaluation claims (`â(x)=v`, `b̂(x')=v'`, `ĉ(x'')=v''`) @@ -34,7 +34,7 @@ //! `(z, ρ-values)`, so lincheck only needs to fold `z` **once** at that //! shared point. //! -//! 1. **Prover sends** one length-`k = 2^k_log` F128 vector +//! 1. **Prover sends** one length-`k = 2^k_log` F192 vector //! `z_vec[i_inner] = ẑ(i_inner, x_ab.x_outer)`. //! 2. **Verifier checks** *two* consistency equations against the same //! `z_vec`: @@ -67,7 +67,7 @@ //! To compose with the **zerocheck's univariate skip** for the first `k_skip` //! variables, claim points use the [`QuirkyPoint`] representation: //! -//! `x = (z_skip ∈ F_{2^128}, x_inner_rest ∈ F_{2^128}^{k_log − k_skip}, x_outer ∈ F_{2^128}^{n_log})` +//! `x = (z_skip ∈ F_{2^192}, x_inner_rest ∈ F_{2^192}^{k_log − k_skip}, x_outer ∈ F_{2^192}^{n_log})` //! //! - `z_skip` is the univariate-skip challenge; it represents all `k_skip` //! skip variables collapsed via the polynomial extension with Lagrange @@ -117,11 +117,11 @@ //! `byte_idx` and apply it across all `i_inner` with one lookup + one XOR //! per byte. -use pcs::{ProverState, VerifierState}; -use primitives::field::F128; -use primitives::multilinear::{build_eq, inner_product}; use crate::r1cs::SparseBinaryMatrix; -use crate::zerocheck::multilinear::lagrange_weights_naive; +use fiat_shamir::transcript::{ProverState, VerifierState}; +use pcs::ring_switch::inner_product_ext; +use primitives::field::F192; +use primitives::multilinear::{eq_table as build_eq, lagrange_weights_naive}; // --------------------------------------------------------------------------- // LincheckCircuit: the per-block linear structure lincheck consumes @@ -135,9 +135,11 @@ use crate::zerocheck::multilinear::lagrange_weights_naive; // marginal of base matrix `M ∈ {A_0, B_0}` — cost ∝ NNZ. // // `LincheckCircuit` is the seam: the prover and verifier take -// `&dyn LincheckCircuit` instead of a pair of matrices. The one live impl is +// `&dyn LincheckCircuit` instead of a pair of matrices. Two live impls: // [`CscCircuit`] (the cached column-major transpose of BLAKE3's `(A_0, B_0)`, -// see `BlockR1cs::csc_lincheck_circuit`). +// see `BlockR1cs::csc_lincheck_circuit`) — the prover's marginal fold — and +// `blake3::WalkLincheckCircuit`, whose `bilinear_form` lets the verifier skip +// the marginal entirely via the circuit walk (flock.tex §Circuit walking). /// Per-block linear structure consumed by lincheck. Implementations produce /// the α-batched column marginal `comb_vec[c] = α · ξ_A(c) + ξ_B(c)` either @@ -148,7 +150,7 @@ pub trait LincheckCircuit: Sync { /// Compute `comb_vec[c] = α · (eq^T · A_0)[c] + (eq^T · B_0)[c]` over /// `c ∈ [0, n_cols())`. `eq_inner.len() == n_cols()`. - fn fold_alpha_batched(&self, alpha: F128, eq_inner: &[F128]) -> Vec; + fn fold_alpha_batched(&self, alpha: F192, eq_inner: &[F192]) -> Vec; /// Column index of a constant-one wire to pin, or `None` if the circuit has /// no such wire. When `Some(col)`, lincheck folds one extra `β`-term into the @@ -160,6 +162,22 @@ pub trait LincheckCircuit: Sync { fn const_pin_col(&self) -> Option { None } + + /// Optional verifier-side fast path (flock.tex §Circuit walking): the + /// α-batched bilinear form + /// + /// `α·(uᵀ A_0 w) + (uᵀ B_0 w)` + /// + /// for arbitrary row weights `u` and column weights `w` (length + /// `n_cols()` each), WITHOUT materializing the length-k column marginal. + /// [`verify`] only ever consumes the marginal through one inner product + /// against a column-weight vector, so an implementation that can walk its + /// circuit (O(circuit) field ops — see `blake3::bilinear_walk`) answers + /// here and never pays the ∝ NNZ marginal. Default `None`: the verifier + /// falls back to `fold_alpha_batched`. + fn bilinear_form(&self, _alpha: F192, _u: &[F192], _w: &[F192]) -> Option { + None + } } /// Column-major (CSC) `LincheckCircuit`: `(A_0, B_0)` transposed once into @@ -253,15 +271,15 @@ impl LincheckCircuit for CscCircuit { fn const_pin_col(&self) -> Option { self.const_pin } - fn fold_alpha_batched(&self, alpha: F128, eq_inner: &[F128]) -> Vec { + fn fold_alpha_batched(&self, alpha: F192, eq_inner: &[F192]) -> Vec { use rayon::prelude::*; assert_eq!(eq_inner.len(), self.n_cols); let one_col = |c: usize| { - let mut sa = F128::ZERO; + let mut sa = F192::ZERO; for &r in &self.a_rows[self.a_col_ptr[c] as usize..self.a_col_ptr[c + 1] as usize] { sa += eq_inner[r as usize]; } - let mut sb = F128::ZERO; + let mut sb = F192::ZERO; for &r in &self.b_rows[self.b_col_ptr[c] as usize..self.b_col_ptr[c + 1] as usize] { sb += eq_inner[r as usize]; } @@ -270,10 +288,8 @@ impl LincheckCircuit for CscCircuit { if self.n_cols < SUMCHECK_PAR_THRESHOLD { return (0..self.n_cols).map(one_col).collect(); } - let mut out = vec![F128::ZERO; self.n_cols]; - out.par_iter_mut() - .enumerate() - .for_each(|(c, slot)| *slot = one_col(c)); + let mut out = vec![F192::ZERO; self.n_cols]; + out.par_iter_mut().enumerate().for_each(|(c, slot)| *slot = one_col(c)); out } } @@ -290,13 +306,13 @@ impl LincheckCircuit for CscCircuit { /// zerocheck's extract_c output uses. #[derive(Clone, Debug, PartialEq, Eq)] pub struct QuirkyPoint { - /// Univariate-skip challenge ∈ F₁₂₈. Binds all `k_skip` skip variables. - pub z_skip: F128, + /// Univariate-skip challenge ∈ F₁₉₂ (tower). Binds all `k_skip` skip variables. + pub z_skip: F192, /// Multilinear coords for the inner dims *after* the skip block. Length /// `k_log − k_skip`. - pub x_inner_rest: Vec, + pub x_inner_rest: Vec, /// Multilinear coords for the outer dims. Length `n_log = m − k_log`. - pub x_outer: Vec, + pub x_outer: Vec, } // Lincheck prover message: a partial product-sumcheck that proves the two @@ -312,19 +328,19 @@ pub struct QuirkyPoint { #[derive(Clone, Debug, PartialEq, Eq)] pub struct LincheckClaim { /// The A/B batching challenge (sampled first). - pub alpha: F128, + pub alpha: F192, /// The constant-pin challenge (sampled after `alpha`; zero when the /// circuit has no pin column). - pub beta: F128, + pub beta: F192, /// The sumcheck round challenges, in round order (MSB-first binding). - pub r_rounds: Vec, + pub r_rounds: Vec, /// Univariate-skip post-vector random sample. - pub r_inner_skip: F128, + pub r_inner_skip: F192, /// Multilinear post-vector random sample, length `k_log − k_skip`. - pub r_inner_rest: Vec, + pub r_inner_rest: Vec, /// `ẑ((r_inner_skip, r_inner_rest), x_ab.x_outer)` — the single /// `z`-claim derived from the A and B consistency checks. - pub w: F128, + pub w: F192, } /// Reasons the verifier may reject. @@ -376,16 +392,11 @@ pub enum VerifyError { /// /// `output[i_inner] = Σ_{i_outer ∈ {0,1}^n_log} z[i_inner, i_outer] · eq_outer[i_outer]` /// -/// Equivalently, `output[i_inner] = ẑ(i_inner_as_F128, x_outer)` for boolean +/// Equivalently, `output[i_inner] = ẑ(i_inner_as_F192, x_outer)` for boolean /// `i_inner`. Used as the cross-check oracle for the production /// `partial_fold_packed_z_triple`. #[cfg(test)] -pub fn partial_fold_packed_z( - z_packed: &[u8], - m: usize, - k_log: usize, - eq_outer: &[F128], -) -> Vec { +pub fn partial_fold_packed_z(z_packed: &[u8], m: usize, k_log: usize, eq_outer: &[F192]) -> Vec { let n_log = m - k_log; let k = 1usize << k_log; let n_outer = 1usize << n_log; @@ -394,7 +405,7 @@ pub fn partial_fold_packed_z( assert!(n_log >= 3, "need n_outer ≥ 8 for byte stripes"); let n_stripes = n_outer / 8; - let mut out = vec![F128::ZERO; k]; + let mut out = vec![F192::ZERO; k]; for byte_idx in 0..n_stripes { let stripe = &z_packed[byte_idx * k..(byte_idx + 1) * k]; for (i_inner, &byte) in stripe.iter().enumerate() { @@ -413,7 +424,7 @@ pub fn partial_fold_packed_z( out } -/// Padding-aware variant of [`partial_fold_packed_z_fast`]. Skips rows +/// Padding-aware variant of `partial_fold_packed_z_fast`. Skips rows /// `i_inner ∈ [useful_bits, k)` — those rows hold zero in every block of an /// honestly padded witness, so the fold over the outer dim is zero. Output /// is byte-identical to the dense path on such witnesses. @@ -422,8 +433,8 @@ pub fn partial_fold_packed_z_fast_padded( m: usize, k_log: usize, useful_bits: usize, - eq_outer: &[F128], -) -> Vec { + eq_outer: &[F192], +) -> Vec { use rayon::prelude::*; let n_log = m - k_log; @@ -446,10 +457,10 @@ pub fn partial_fold_packed_z_fast_padded( .par_chunks(bytes_per_chunk) .enumerate() .fold( - || vec![F128::ZERO; k], + || vec![F192::ZERO; k], |mut acc, (chunk_idx, chunk_bytes)| { let stripe_start = chunk_idx * stripes_per_chunk; - let mut table = vec![F128::ZERO; 256]; + let mut table = vec![F192::ZERO; 256]; for (rel_stripe, stripe) in chunk_bytes.chunks(k).enumerate() { let byte_idx = stripe_start + rel_stripe; build_sum_table(&eq_outer[8 * byte_idx..8 * byte_idx + 8], &mut table); @@ -461,7 +472,7 @@ pub fn partial_fold_packed_z_fast_padded( }, ) .reduce( - || vec![F128::ZERO; k], + || vec![F192::ZERO; k], |mut a, b| { for (x, y) in a.iter_mut().zip(b.iter()) { *x += *y; @@ -481,21 +492,86 @@ const NEON_TILE_T: usize = 8; /// for one BLOCK_K=8 block of i_inner positions, keeping all 8 accumulators /// in NEON Q-registers. /// -/// The 8 z index bytes of a stripe are consecutive, so they are fetched with -/// **one** unaligned 8-byte scalar load and shifted out of the register rather -/// than with eight `LDRB`s: the gather already issues one 128-bit table load -/// per index, and a second load per index would nearly double this kernel's -/// load-port pressure for data that is already in a register. +/// # Safety +/// - `tile_bytes_ptr` must point to at least `TILE_T * k` bytes. +/// - `tables_ptr` must point to at least `TILE_T * 256` F192 entries. +/// - `out_ptr` must point to at least 8 F192 entries of mutable storage. +/// x86-64 twin of the tiled gather kernel below. /// -/// Stripes are swept two at a time so each accumulator update folds both table -/// lookups with one `EOR3`, halving the number of accumulator updates and the -/// length of the serial dependency chain through each accumulator. +/// `VPTERNLOGQ` is the exact counterpart of AArch64's `EOR3`: an arbitrary +/// three-input bitwise function in one instruction, so immediate `0x96` +/// (`a ^ b ^ c`) folds a paired-stripe accumulate the same way. Only the +/// `c0`/`c1` limbs ride in the vector — `c2` is scalar and takes two XORs. +/// +/// Before this, x86 had no tiled kernel at all: the dispatcher sent every +/// non-AArch64 target to the generic `partial_fold_packed_z_fast_padded`. /// /// # Safety /// - `tile_bytes_ptr` must point to at least `TILE_T * k` bytes, with `bs + 8` -/// readable in every stripe row (guaranteed by `bs + BLOCK_K ≤ k`). -/// - `tables_ptr` must point to at least `TILE_T * 256 * 16` bytes. -/// - `out_ptr` must point to at least 8 F128 (128 bytes) of mutable storage. +/// readable in every stripe row (guaranteed by `bs + BLOCK_K <= k`). +/// - `tables_ptr` must point to at least `TILE_T * 256` `F192`. +/// - `out_ptr` must point to at least 8 writable `F192`. +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx512vl"))] +#[inline(never)] +#[allow(unsafe_op_in_unsafe_fn)] +unsafe fn process_block_neon_single( + tile_bytes_ptr: *const u8, + k: usize, + bs: usize, + tables_ptr: *const F192, + out_ptr: *mut F192, +) { + use std::arch::x86_64::*; + const TILE_T: usize = NEON_TILE_T; + // `F192` is `#[repr(C)]` with `c0, c1, c2`, so a 128-bit load at `&c0` + // covers exactly the `(c0, c1)` pair. + const XOR3: i32 = 0x96; + + let mut acc01 = [_mm_setzero_si128(); 8]; + let mut acc2 = [0u64; 8]; + for i in 0..8 { + let out = &*out_ptr.add(i); + acc01[i] = _mm_loadu_si128((&out.c0 as *const u64).cast()); + acc2[i] = out.c2; + } + + // One unaligned 8-byte load per stripe replaces eight LDRB-equivalents, + // and stripes are swept in pairs so each vector accumulator folds both + // table entries with a single VPTERNLOGQ. + let mut t = 0; + while t + 1 < TILE_T { + let ta0 = tables_ptr.add(t * 256); + let ta1 = tables_ptr.add((t + 1) * 256); + let w0 = (tile_bytes_ptr.add(t * k + bs) as *const u64).read_unaligned(); + let w1 = (tile_bytes_ptr.add((t + 1) * k + bs) as *const u64).read_unaligned(); + for i in 0..8 { + let e0 = &*ta0.add(((w0 >> (8 * i)) & 0xff) as usize); + let e1 = &*ta1.add(((w1 >> (8 * i)) & 0xff) as usize); + let v0 = _mm_loadu_si128((&e0.c0 as *const u64).cast()); + let v1 = _mm_loadu_si128((&e1.c0 as *const u64).cast()); + acc01[i] = _mm_ternarylogic_epi64::(acc01[i], v0, v1); + acc2[i] ^= e0.c2 ^ e1.c2; + } + t += 2; + } + if t < TILE_T { + let ta = tables_ptr.add(t * 256); + let w = (tile_bytes_ptr.add(t * k + bs) as *const u64).read_unaligned(); + for i in 0..8 { + let entry = &*ta.add(((w >> (8 * i)) & 0xff) as usize); + let v = _mm_loadu_si128((&entry.c0 as *const u64).cast()); + acc01[i] = _mm_xor_si128(acc01[i], v); + acc2[i] ^= entry.c2; + } + } + + for i in 0..8 { + let out = &mut *out_ptr.add(i); + _mm_storeu_si128((&mut out.c0 as *mut u64).cast(), acc01[i]); + out.c2 = acc2[i]; + } +} + #[cfg(target_arch = "aarch64")] #[inline(never)] #[allow(unsafe_op_in_unsafe_fn)] @@ -503,94 +579,60 @@ unsafe fn process_block_neon_single( tile_bytes_ptr: *const u8, k: usize, bs: usize, - tables_ptr: *const u8, - out_ptr: *mut F128, + tables_ptr: *const F192, + out_ptr: *mut F192, ) { - use primitives::field::neon::xor3_u8; + use primitives::field::neon::xor3_u64; use std::arch::aarch64::*; const TILE_T: usize = NEON_TILE_T; - let o = out_ptr as *mut u8; - - let mut a0 = vld1q_u8(o); - let mut a1 = vld1q_u8(o.add(16)); - let mut a2 = vld1q_u8(o.add(32)); - let mut a3 = vld1q_u8(o.add(48)); - let mut a4 = vld1q_u8(o.add(64)); - let mut a5 = vld1q_u8(o.add(80)); - let mut a6 = vld1q_u8(o.add(96)); - let mut a7 = vld1q_u8(o.add(112)); + let mut acc01 = [vdupq_n_u64(0); 8]; + let mut acc2 = [0u64; 8]; + for i in 0..8 { + let out = &*out_ptr.add(i); + acc01[i] = vld1q_u64(&out.c0); + acc2[i] = out.c2; + } + // The 8 z index bytes of a stripe are consecutive, so fetch them with one + // unaligned 8-byte scalar load and shift them out of the register rather + // than with eight LDRBs: the gather already issues a table load per index, + // and a second load per index would nearly double this kernel's load-port + // pressure for data that is already in a register. + // + // Stripes are swept in pairs so each vector accumulator folds both table + // lookups with one EOR3, halving the accumulator updates and the serial + // dependency chain through each of the 8 live accumulators. The `c2` + // limbs are scalar, so they just take two XORs. let mut t = 0; while t + 1 < TILE_T { - let ta0 = tables_ptr.add(t * 256 * 16); - let ta1 = tables_ptr.add((t + 1) * 256 * 16); + let ta0 = tables_ptr.add(t * 256); + let ta1 = tables_ptr.add((t + 1) * 256); let w0 = (tile_bytes_ptr.add(t * k + bs) as *const u64).read_unaligned(); let w1 = (tile_bytes_ptr.add((t + 1) * k + bs) as *const u64).read_unaligned(); - - a0 = xor3_u8( - a0, - vld1q_u8(ta0.add((w0 & 0xff) as usize * 16)), - vld1q_u8(ta1.add((w1 & 0xff) as usize * 16)), - ); - a1 = xor3_u8( - a1, - vld1q_u8(ta0.add(((w0 >> 8) & 0xff) as usize * 16)), - vld1q_u8(ta1.add(((w1 >> 8) & 0xff) as usize * 16)), - ); - a2 = xor3_u8( - a2, - vld1q_u8(ta0.add(((w0 >> 16) & 0xff) as usize * 16)), - vld1q_u8(ta1.add(((w1 >> 16) & 0xff) as usize * 16)), - ); - a3 = xor3_u8( - a3, - vld1q_u8(ta0.add(((w0 >> 24) & 0xff) as usize * 16)), - vld1q_u8(ta1.add(((w1 >> 24) & 0xff) as usize * 16)), - ); - a4 = xor3_u8( - a4, - vld1q_u8(ta0.add(((w0 >> 32) & 0xff) as usize * 16)), - vld1q_u8(ta1.add(((w1 >> 32) & 0xff) as usize * 16)), - ); - a5 = xor3_u8( - a5, - vld1q_u8(ta0.add(((w0 >> 40) & 0xff) as usize * 16)), - vld1q_u8(ta1.add(((w1 >> 40) & 0xff) as usize * 16)), - ); - a6 = xor3_u8( - a6, - vld1q_u8(ta0.add(((w0 >> 48) & 0xff) as usize * 16)), - vld1q_u8(ta1.add(((w1 >> 48) & 0xff) as usize * 16)), - ); - a7 = xor3_u8( - a7, - vld1q_u8(ta0.add((w0 >> 56) as usize * 16)), - vld1q_u8(ta1.add((w1 >> 56) as usize * 16)), - ); + for i in 0..8 { + let e0 = &*ta0.add(((w0 >> (8 * i)) & 0xff) as usize); + let e1 = &*ta1.add(((w1 >> (8 * i)) & 0xff) as usize); + acc01[i] = xor3_u64(acc01[i], vld1q_u64(&e0.c0), vld1q_u64(&e1.c0)); + acc2[i] ^= e0.c2 ^ e1.c2; + } t += 2; } if t < TILE_T { - let ta = tables_ptr.add(t * 256 * 16); + let ta = tables_ptr.add(t * 256); let w = (tile_bytes_ptr.add(t * k + bs) as *const u64).read_unaligned(); - a0 = veorq_u8(a0, vld1q_u8(ta.add((w & 0xff) as usize * 16))); - a1 = veorq_u8(a1, vld1q_u8(ta.add(((w >> 8) & 0xff) as usize * 16))); - a2 = veorq_u8(a2, vld1q_u8(ta.add(((w >> 16) & 0xff) as usize * 16))); - a3 = veorq_u8(a3, vld1q_u8(ta.add(((w >> 24) & 0xff) as usize * 16))); - a4 = veorq_u8(a4, vld1q_u8(ta.add(((w >> 32) & 0xff) as usize * 16))); - a5 = veorq_u8(a5, vld1q_u8(ta.add(((w >> 40) & 0xff) as usize * 16))); - a6 = veorq_u8(a6, vld1q_u8(ta.add(((w >> 48) & 0xff) as usize * 16))); - a7 = veorq_u8(a7, vld1q_u8(ta.add((w >> 56) as usize * 16))); - } - - vst1q_u8(o, a0); - vst1q_u8(o.add(16), a1); - vst1q_u8(o.add(32), a2); - vst1q_u8(o.add(48), a3); - vst1q_u8(o.add(64), a4); - vst1q_u8(o.add(80), a5); - vst1q_u8(o.add(96), a6); - vst1q_u8(o.add(112), a7); + for i in 0..8 { + let entry = &*ta.add(((w >> (8 * i)) & 0xff) as usize); + acc01[i] = veorq_u64(acc01[i], vld1q_u64(&entry.c0)); + acc2[i] ^= entry.c2; + } + } + + for i in 0..8 { + let out = &mut *out_ptr.add(i); + vst1q_u64(&mut out.c0, acc01[i]); + out.c2 = acc2[i]; + } } /// **i_inner-partitioned** NEON partial fold: parallelizes over the @@ -600,24 +642,27 @@ unsafe fn process_block_neon_single( /// accumulator (2 MB at k = 2¹⁷). With P workers that's `P · 2 MB` of live /// accumulators — past ~3 workers it exceeds L2, so each worker's accumulator /// spills and gets re-streamed from **main memory** once per stripe-tile -/// (≈ `n_tiles · 2·k` F128 of memory traffic). Measured: scaling saturates at +/// (≈ `n_tiles · 2·k` F192 of memory traffic). Measured: scaling saturates at /// ~5× on 10 cores (memory-bound), not ~10×. /// /// Here the workers own **disjoint** slices of a single shared `out`, so the -/// total live accumulator is just `k` F128 = 2 MB — it stays L2-resident, never +/// total live accumulator is just `k` F192 = 2 MB — it stays L2-resident, never /// re-streamed from memory, and there is **no final reduction**. Main-memory /// traffic drops to one pass over z plus one write of `out`. Each worker still /// uses the register-tiled inner kernel (8 accumulators across `TILE_T` /// stripes); it just rebuilds the per-tile sum tables for its own slice (a few /// % of redundant table-build XORs, far cheaper than the memory re-streaming). -#[cfg(target_arch = "aarch64")] +#[cfg(any( + target_arch = "aarch64", + all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx512vl") +))] pub fn partial_fold_packed_z_neon_iblock_padded( z_packed: &[u8], m: usize, k_log: usize, useful_bits: usize, - eq_outer: &[F128], -) -> Vec { + eq_outer: &[F192], +) -> Vec { use rayon::prelude::*; const TILE_T: usize = NEON_TILE_T; @@ -644,7 +689,7 @@ pub fn partial_fold_packed_z_neon_iblock_padded( // contribute nothing. Rows [useful, k) stay zero from the vec init. let useful = (useful_bits.div_ceil(BLOCK_K) * BLOCK_K).min(k); - let mut out = vec![F128::ZERO; k]; + let mut out = vec![F192::ZERO; k]; if useful == 0 { return out; } @@ -674,31 +719,22 @@ pub fn partial_fold_packed_z_neon_iblock_padded( .for_each(|(ci, out_slice)| { let i_base = ci * i_chunk; let n_block = out_slice.len() / BLOCK_K; - // TILE_T × 256 F128 = 32 KB tables, L1-resident, rebuilt per tile. - let mut tables = vec![F128::ZERO; TILE_T * 256]; + // TILE_T × 256 F192 = 32 KB tables, L1-resident, rebuilt per tile. + let mut tables = vec![F192::ZERO; TILE_T * 256]; for tile in 0..n_tiles { let stripe_base = tile * TILE_T; for t in 0..TILE_T { let eq_off = 8 * (stripe_base + t); - build_sum_table( - &eq_outer[eq_off..eq_off + 8], - &mut tables[t * 256..(t + 1) * 256], - ); + build_sum_table(&eq_outer[eq_off..eq_off + 8], &mut tables[t * 256..(t + 1) * 256]); } - let tables_ptr = tables.as_ptr() as *const u8; + let tables_ptr = tables.as_ptr(); // Base of this (tile, i_base): process_block reads // z_base[t·k + bs] = z[(stripe_base+t)·k + i_base + bs]. let z_base = unsafe { z_packed.as_ptr().add(stripe_base * k + i_base) }; for b in 0..n_block { let i = b * BLOCK_K; unsafe { - process_block_neon_single( - z_base, - k, - i, - tables_ptr, - out_slice.as_mut_ptr().add(i), - ); + process_block_neon_single(z_base, k, i, tables_ptr, out_slice.as_mut_ptr().add(i)); } } } @@ -718,20 +754,23 @@ pub fn partial_fold_packed_z_neon_iblock_padded( /// tile tables exactly **once**, folds them into a private length-k partial, and the /// `p` partials are XOR-reduced at the end. The partial is the full length-k /// (256 KB at k_log=14 ⇒ spills L1 to L2), but the register-tiled inner kernel keeps -/// 8 F128 accumulators in NEON registers, so the L2 traffic is mild — measured ≈2 % +/// 8 F192 accumulators in NEON registers, so the L2 traffic is mild — measured ≈2 % /// ST cost at m=32, none at m=30 — and far cheaper than iblock's redundant tables: /// the fold scales ~8.5× vs iblock's ~6.5× on 10 P-cores at m=32, and the margin /// grows with the outer dim (the redundant-table cost it removes is ∝ `n_stripes`). /// /// # Safety / preconditions: identical to the iblock kernel. -#[cfg(target_arch = "aarch64")] +#[cfg(any( + target_arch = "aarch64", + all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx512vl") +))] pub fn partial_fold_packed_z_neon_oblock_padded( z_packed: &[u8], m: usize, k_log: usize, useful_bits: usize, - eq_outer: &[F128], -) -> Vec { + eq_outer: &[F192], +) -> Vec { use rayon::prelude::*; const TILE_T: usize = NEON_TILE_T; @@ -757,7 +796,7 @@ pub fn partial_fold_packed_z_neon_oblock_padded( // up to BLOCK_K; columns [useful, k) stay zero from the partial init. let useful = (useful_bits.div_ceil(BLOCK_K) * BLOCK_K).min(k); if useful == 0 { - return vec![F128::ZERO; k]; + return vec![F192::ZERO; k]; } // One private length-k partial per worker; workers own contiguous tile bands, @@ -766,50 +805,36 @@ pub fn partial_fold_packed_z_neon_oblock_padded( let tiles_per_worker = n_tiles.div_ceil(p); let n_workers = n_tiles.div_ceil(tiles_per_worker); // ≤ p, every band non-empty - let mut partials = vec![F128::ZERO; n_workers * k]; - partials - .par_chunks_mut(k) - .enumerate() - .for_each(|(w, partial)| { - let tile_lo = w * tiles_per_worker; - let tile_hi = ((w + 1) * tiles_per_worker).min(n_tiles); - // TILE_T × 256 F128 = 32 KB tables, L1-resident, built once per tile. - let mut tables = vec![F128::ZERO; TILE_T * 256]; - for tile in tile_lo..tile_hi { - let stripe_base = tile * TILE_T; - for t in 0..TILE_T { - let eq_off = 8 * (stripe_base + t); - build_sum_table( - &eq_outer[eq_off..eq_off + 8], - &mut tables[t * 256..(t + 1) * 256], - ); - } - let tables_ptr = tables.as_ptr() as *const u8; - let z_base = unsafe { z_packed.as_ptr().add(stripe_base * k) }; - let mut bs = 0usize; - while bs < useful { - unsafe { - process_block_neon_single( - z_base, - k, - bs, - tables_ptr, - partial.as_mut_ptr().add(bs), - ); - } - bs += BLOCK_K; + let mut partials = vec![F192::ZERO; n_workers * k]; + partials.par_chunks_mut(k).enumerate().for_each(|(w, partial)| { + let tile_lo = w * tiles_per_worker; + let tile_hi = ((w + 1) * tiles_per_worker).min(n_tiles); + // TILE_T × 256 F192 = 32 KB tables, L1-resident, built once per tile. + let mut tables = vec![F192::ZERO; TILE_T * 256]; + for tile in tile_lo..tile_hi { + let stripe_base = tile * TILE_T; + for t in 0..TILE_T { + let eq_off = 8 * (stripe_base + t); + build_sum_table(&eq_outer[eq_off..eq_off + 8], &mut tables[t * 256..(t + 1) * 256]); + } + let tables_ptr = tables.as_ptr(); + let z_base = unsafe { z_packed.as_ptr().add(stripe_base * k) }; + let mut bs = 0usize; + while bs < useful { + unsafe { + process_block_neon_single(z_base, k, bs, tables_ptr, partial.as_mut_ptr().add(bs)); } + bs += BLOCK_K; } - }); + } + }); // XOR-reduce the per-worker partials: parallel over columns, sequential over // workers so each 256 KB partial is streamed once (cache-friendly). let (first, rest) = partials.split_at(k); let mut out = first.to_vec(); for chunk in rest.chunks(k) { - out.par_iter_mut() - .zip(chunk.par_iter()) - .for_each(|(o, s)| *o += *s); + out.par_iter_mut().zip(chunk.par_iter()).for_each(|(o, s)| *o += *s); } out } @@ -823,10 +848,13 @@ fn partial_fold_packed_z_best( m: usize, k_log: usize, useful_bits: usize, - eq_outer: &[F128], -) -> Vec { + eq_outer: &[F192], +) -> Vec { if n_log_ok_for_tile(m, k_log, NEON_TILE_T) { - #[cfg(target_arch = "aarch64")] + #[cfg(any( + target_arch = "aarch64", + all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx512vl") + ))] { // Pick the partition that wins for this size. The outer(tile)-partitioned // `oblock` builds each tile's sum-tables once instead of once per worker, @@ -836,19 +864,15 @@ fn partial_fold_packed_z_best( // crossover sits at n_log ≈ 15–16 across k_log ∈ {11,14}, so gate oblock at // n_log ≥ 16; below that the L1-resident `iblock` wins. let n_log = m - k_log; - if n_log >= OBLOCK_MIN_N_LOG - { - return partial_fold_packed_z_neon_oblock_padded( - z_packed, - m, - k_log, - useful_bits, - eq_outer, - ); + if n_log >= OBLOCK_MIN_N_LOG { + return partial_fold_packed_z_neon_oblock_padded(z_packed, m, k_log, useful_bits, eq_outer); } partial_fold_packed_z_neon_iblock_padded(z_packed, m, k_log, useful_bits, eq_outer) } - #[cfg(not(target_arch = "aarch64"))] + #[cfg(not(any( + target_arch = "aarch64", + all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx512vl") + )))] { partial_fold_packed_z_fast_padded(z_packed, m, k_log, useful_bits, eq_outer) } @@ -860,7 +884,10 @@ fn partial_fold_packed_z_best( /// Outer-dimension threshold (`n_log = m − k_log`) at/above which the /// outer(tile)-partitioned fold beats the i_inner-partitioned one. See /// [`partial_fold_packed_z_best`] for the crossover calibration. -#[cfg(target_arch = "aarch64")] +#[cfg(any( + target_arch = "aarch64", + all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx512vl") +))] const OBLOCK_MIN_N_LOG: usize = 16; /// Quick test for "can we use the tiled fast path?". Tile uses `TILE_T` @@ -874,17 +901,17 @@ fn n_log_ok_for_tile(m: usize, k_log: usize, tile_t: usize) -> bool { n_stripes.is_multiple_of(tile_t) } -/// Build a 256-entry sum table over 8 F128 values: +/// Build a 256-entry sum table over 8 F192 values: /// `table[b] = Σ_{r: bit r of b is set} eq8[r]` /// /// Doubling construction (255 XORs): for each new bit position `i ∈ 0..8`, /// extend the table by XORing `eq8[i]` into each existing entry. This /// avoids the naive 8·256 = 2048 operations. #[inline] -fn build_sum_table(eq8: &[F128], table: &mut [F128]) { +fn build_sum_table(eq8: &[F192], table: &mut [F192]) { debug_assert_eq!(eq8.len(), 8); debug_assert_eq!(table.len(), 256); - table[0] = F128::ZERO; + table[0] = F192::ZERO; for i in 0..8 { let e = eq8[i]; let len = 1usize << i; @@ -910,8 +937,7 @@ pub fn pack_z_lincheck(z_logical: &[bool], m: usize, k_log: usize) -> Vec { assert_eq!(n_outer % 8, 0, "need n_outer ≥ 8 for byte stripes"); let n_stripes = n_outer / 8; - // Uninit alloc — every byte is written exactly once in the loop below. - let mut z_packed: Vec = primitives::alloc_uninit_vec(n_total / 8); + let mut z_packed = primitives::alloc_uninit(n_total / 8); for byte_idx in 0..n_stripes { for i_inner in 0..k { let mut byte = 0u8; @@ -922,57 +948,49 @@ pub fn pack_z_lincheck(z_logical: &[bool], m: usize, k_log: usize) -> Vec { byte |= 1u8 << r; } } - z_packed[byte_idx * k + i_inner] = byte; + z_packed[byte_idx * k + i_inner].write(byte); } } - z_packed + // SAFETY: the nested loops write every output byte exactly once. + unsafe { primitives::assume_init(z_packed) } } -/// Same output as [`pack_z_lincheck`] but reads bits from an F_{2^128}-packed -/// witness (polynomial basis: bit `i` of logical = bit `i % 128` of -/// `z_packed_f128[i / 128]`). -pub fn pack_z_lincheck_from_packed( - z_packed_f128: &[primitives::field::F128], - m: usize, - k_log: usize, -) -> Vec { +/// Same output as `pack_z_lincheck`, but reads bits from a 128-bit packed +/// witness embedded in F192. In the polynomial basis, logical bit `i` is bit +/// `i % 128` of `z_packed_words[i / 128]`. +pub fn pack_z_lincheck_from_packed(z_packed_words: &[primitives::field::F192], m: usize, k_log: usize) -> Vec { use rayon::prelude::*; let k = 1usize << k_log; let n_total = 1usize << m; - assert_eq!(z_packed_f128.len(), n_total / 128); + assert_eq!(z_packed_words.len(), n_total / 128); let n_outer = n_total / k; assert_eq!(n_outer % 8, 0, "need n_outer ≥ 8 for byte stripes"); - // Uninit alloc — the par_chunks_mut loop below writes every byte of - // every k-byte stripe exactly once. Saves ~10 ms of sequential - // zero-fill at m=29 (64 MB byte buffer) on the main thread. - let mut z_packed: Vec = primitives::alloc_uninit_vec(n_total / 8); + let mut z_packed = primitives::alloc_uninit(n_total / 8); // Each stripe (byte_idx) writes a disjoint k-byte chunk — process them in // parallel. Inside one stripe, k independent output bytes. - z_packed - .par_chunks_mut(k) - .enumerate() - .for_each(|(byte_idx, chunk)| { - for i_inner in 0..k { - let mut byte = 0u8; - for r in 0..8 { - let i_outer = 8 * byte_idx + r; - let logical_idx = i_inner + i_outer * k; - let f128_idx = logical_idx / 128; - let local_bit = logical_idx % 128; - let bit = if local_bit < 64 { - (z_packed_f128[f128_idx].lo >> local_bit) & 1 == 1 - } else { - (z_packed_f128[f128_idx].hi >> (local_bit - 64)) & 1 == 1 - }; - if bit { - byte |= 1u8 << r; - } + z_packed.par_chunks_mut(k).enumerate().for_each(|(byte_idx, chunk)| { + for i_inner in 0..k { + let mut byte = 0u8; + for r in 0..8 { + let i_outer = 8 * byte_idx + r; + let logical_idx = i_inner + i_outer * k; + let packed_word = logical_idx / 128; + let local_bit = logical_idx % 128; + let bit = if local_bit < 64 { + (z_packed_words[packed_word].c0 >> local_bit) & 1 == 1 + } else { + (z_packed_words[packed_word].c1 >> (local_bit - 64)) & 1 == 1 + }; + if bit { + byte |= 1u8 << r; } - chunk[i_inner] = byte; } - }); - z_packed + chunk[i_inner].write(byte); + } + }); + // SAFETY: every parallel chunk writes each of its output bytes exactly once. + unsafe { primitives::assume_init(z_packed) } } /// Build the **quirky eq table** for a claim point on the inner half: @@ -989,7 +1007,7 @@ pub fn pack_z_lincheck_from_packed( /// inner-rest dims occupy the next bits. /// /// Cost: 64 (Lagrange) + 32 (eq) + 2048 outer products ≈ tiny. -pub fn build_quirky_eq_table(z_skip: F128, x_inner_rest: &[F128], k_skip: usize) -> Vec { +pub fn build_quirky_eq_table(z_skip: F192, x_inner_rest: &[F192], k_skip: usize) -> Vec { let ell_skip = 1usize << k_skip; let ell_rest = 1usize << x_inner_rest.len(); let lambda_skip = lagrange_weights_naive(k_skip, z_skip); @@ -1013,15 +1031,15 @@ const SUMCHECK_PAR_THRESHOLD: usize = 1usize << 12; /// One round of product-sumcheck on `(c, z)`: compute `(q(1), q(∞))` = /// `(Σ c_hi·z_hi, Σ (c_hi+c_lo)·(z_hi+z_lo))` over the top-bit split. The /// `len()` of `c` and `z` is even; `half = len/2`. -fn sumcheck_round_eval_par(c: &[F128], z: &[F128]) -> (F128, F128) { +fn sumcheck_round_eval_par(c: &[F192], z: &[F192]) -> (F192, F192) { use rayon::prelude::*; let half = c.len() / 2; debug_assert_eq!(z.len(), c.len()); let (clo, chi) = c.split_at(half); let (zlo, zhi) = z.split_at(half); if half < SUMCHECK_PAR_THRESHOLD { - let mut e1 = F128::ZERO; - let mut einf = F128::ZERO; + let mut e1 = F192::ZERO; + let mut einf = F192::ZERO; for i in 0..half { e1 += chi[i] * zhi[i]; einf += (chi[i] + clo[i]) * (zhi[i] + zlo[i]); @@ -1035,12 +1053,12 @@ fn sumcheck_round_eval_par(c: &[F128], z: &[F128]) -> (F128, F128) { let einf_i = (chi[i] + clo[i]) * (zhi[i] + zlo[i]); (e1_i, einf_i) }) - .reduce(|| (F128::ZERO, F128::ZERO), |a, b| (a.0 + b.0, a.1 + b.1)) + .reduce(|| (F192::ZERO, F192::ZERO), |a, b| (a.0 + b.0, a.1 + b.1)) } /// Bind the top remaining variable of `v` at challenge `r`: `v[i] ← v[i] + /// r·(v[i+half] + v[i])` for `i ∈ [0, half)`, then truncate to `half`. In-place. -pub fn sumcheck_bind_top_in_place_par(v: &mut Vec, r: F128) { +pub fn sumcheck_bind_top_in_place_par(v: &mut Vec, r: F192) { use rayon::prelude::*; let half = v.len() / 2; if half < SUMCHECK_PAR_THRESHOLD { @@ -1050,11 +1068,27 @@ pub fn sumcheck_bind_top_in_place_par(v: &mut Vec, r: F128) { } else { let (lo, hi) = v.split_at_mut(half); let hi = &hi[..half]; - lo.par_iter_mut() - .zip(hi.par_iter()) - .for_each(|(lo_i, &hi_i)| { - *lo_i = *lo_i + r * (hi_i + *lo_i); - }); + lo.par_iter_mut().zip(hi.par_iter()).for_each(|(lo_i, &hi_i)| { + *lo_i = *lo_i + r * (hi_i + *lo_i); + }); + } + v.truncate(half); +} + +/// Tower (`F192`) twin of [`sumcheck_bind_top_in_place_par`], for the verifier. +pub fn sumcheck_bind_top_in_place_par_t(v: &mut Vec, r: F192) { + use rayon::prelude::*; + let half = v.len() / 2; + if half < SUMCHECK_PAR_THRESHOLD { + for i in 0..half { + v[i] = v[i] + r * (v[i + half] + v[i]); + } + } else { + let (lo, hi) = v.split_at_mut(half); + let hi = &hi[..half]; + lo.par_iter_mut().zip(hi.par_iter()).for_each(|(lo_i, &hi_i)| { + *lo_i = *lo_i + r * (hi_i + *lo_i); + }); } v.truncate(half); } @@ -1084,11 +1118,7 @@ pub fn sumcheck_bind_top_in_place_par(v: &mut Vec, r: F128) { /// well-defined next round — the caller guarantees this by only fusing when a /// later round exists). The returned message is bit-identical to /// `sumcheck_round_eval_par` run on the bound tables. -fn sumcheck_bind_both_and_eval_next( - comb: &mut Vec, - z: &mut Vec, - r: F128, -) -> (F128, F128) { +fn sumcheck_bind_both_and_eval_next(comb: &mut Vec, z: &mut Vec, r: F192) -> (F192, F192) { use rayon::prelude::*; let len = comb.len(); debug_assert_eq!(z.len(), len); @@ -1105,8 +1135,8 @@ fn sumcheck_bind_both_and_eval_next( let (zq2, zq3) = z_hi.split_at(half2); let (e1, einf) = if half2 < SUMCHECK_PAR_THRESHOLD { - let mut e1 = F128::ZERO; - let mut einf = F128::ZERO; + let mut e1 = F192::ZERO; + let mut einf = F192::ZERO; for i in 0..half2 { let lo = cq0[i] + r * (cq2[i] + cq0[i]); let hi = cq1[i] + r * (cq3[i] + cq1[i]); @@ -1140,7 +1170,7 @@ fn sumcheck_bind_both_and_eval_next( *z1 = zhi; (hi * zhi, (hi + lo) * (zhi + zlo)) }) - .reduce(|| (F128::ZERO, F128::ZERO), |a, b| (a.0 + b.0, a.1 + b.1)) + .reduce(|| (F192::ZERO, F192::ZERO), |a, b| (a.0 + b.0, a.1 + b.1)) }; comb.truncate(half); @@ -1158,8 +1188,8 @@ fn sumcheck_bind_both_and_eval_next( /// `s_hat_v` via [`pcs::ring_switch::s_hat_v_from_z_vec`], skipping a /// `fold_1b_rows` pass at open time. /// -/// Pays one extra `2^k_log` F128 clone (~2 MB at k_log=17) before the -pub fn prove_padded_capture_z_vec( +/// Pays one extra `2^k_log` F192 clone (~2 MB at k_log=17) before the +pub fn prove_padded_capture_z_vec( z_packed: &[u8], m: usize, k_log: usize, @@ -1167,24 +1197,14 @@ pub fn prove_padded_capture_z_vec( useful_bits: usize, circuit: &dyn LincheckCircuit, x_ab: &QuirkyPoint, - ps: &mut ProverState, -) -> (LincheckClaim, Vec) { - let (claim, captured) = prove_padded_inner( - z_packed, - m, - k_log, - k_skip, - useful_bits, - circuit, - x_ab, - true, - ps, - ); + ps: &mut ProverState, +) -> (LincheckClaim, Vec) { + let (claim, captured) = prove_padded_inner(z_packed, m, k_log, k_skip, useful_bits, circuit, x_ab, true, ps); (claim, captured.expect("capture=true must produce z_vec")) } #[allow(clippy::too_many_arguments)] -fn prove_padded_inner( +fn prove_padded_inner( z_packed: &[u8], m: usize, k_log: usize, @@ -1193,8 +1213,8 @@ fn prove_padded_inner( circuit: &dyn LincheckCircuit, x_ab: &QuirkyPoint, capture_z_vec: bool, - ps: &mut ProverState, -) -> (LincheckClaim, Option>) { + ps: &mut ProverState, +) -> (LincheckClaim, Option>) { let k = 1usize << k_log; let n_log = m - k_log; assert!(m >= k_log); @@ -1207,6 +1227,11 @@ fn prove_padded_inner( let trace = std::env::var("LINCHECK_TRACE").is_ok(); + // Keep local copies of the tower-valued claim point used throughout the fold. + let z_skip_g = x_ab.z_skip; + let x_inner_rest_g = x_ab.x_inner_rest.to_vec(); + let x_outer_g = x_ab.x_outer.to_vec(); + // 1. Sample α (matches verifier's order). Used to batch the two scalar // consistency checks v_a, v_b into a single sumcheck. let alpha = ps.sample(); @@ -1215,12 +1240,8 @@ fn prove_padded_inner( // the sparse-matrix default this is the fused single-pass row-fold; // per-hash circuit walkers compute the same `comb_vec` directly from // the constraint graph. - let t = if trace { - Some(std::time::Instant::now()) - } else { - None - }; - let eq_inner = build_quirky_eq_table(x_ab.z_skip, &x_ab.x_inner_rest, k_skip); + let t = if trace { Some(std::time::Instant::now()) } else { None }; + let eq_inner = build_quirky_eq_table(z_skip_g, &x_inner_rest_g, k_skip); if let Some(t) = t { eprintln!( "[lc] {:<26} {:>7.2} ms", @@ -1228,11 +1249,7 @@ fn prove_padded_inner( t.elapsed().as_secs_f64() * 1e3 ); } - let t = if trace { - Some(std::time::Instant::now()) - } else { - None - }; + let t = if trace { Some(std::time::Instant::now()) } else { None }; let mut comb_vec = circuit.fold_alpha_batched(alpha, &eq_inner); if let Some(t) = t { eprintln!( @@ -1247,19 +1264,15 @@ fn prove_padded_inner( // boolean index, eq(j*, ·) is the one-hot vector and this is a single // entry update. β is sampled after α; the verifier mirrors both. See // lincheck's `LincheckCircuit::const_pin_col`. - let mut beta = F128::ZERO; + let mut beta = F192::ZERO; if let Some(col) = circuit.const_pin_col() { beta = ps.sample(); comb_vec[col] += beta; } - // 3. Partial fold of z at the shared outer half (length-k F128 vector). - let t = if trace { - Some(std::time::Instant::now()) - } else { - None - }; - let eq_x_outer = build_eq(&x_ab.x_outer); + // 3. Partial fold of z at the shared outer half (length-k F192 vector). + let t = if trace { Some(std::time::Instant::now()) } else { None }; + let eq_x_outer = build_eq(&x_outer_g); let mut z_vec = partial_fold_packed_z_best(z_packed, m, k_log, useful_bits, &eq_x_outer); if let Some(t) = t { eprintln!( @@ -1271,16 +1284,8 @@ fn prove_padded_inner( // 3b. Optional capture: clone the pre-sumcheck z_vec for downstream reuse // (PCS open's AB-claim s_hat_v skipping fold_1b_rows). Only pay the // clone when explicitly requested. - let captured_z_vec: Option> = if capture_z_vec { - Some(z_vec.clone()) - } else { - None - }; - let t_sumcheck_start = if trace { - Some(std::time::Instant::now()) - } else { - None - }; + let captured_z_vec: Option> = if capture_z_vec { Some(z_vec.clone()) } else { None }; + let t_sumcheck_start = if trace { Some(std::time::Instant::now()) } else { None }; // 5. Standard multilinear product-sumcheck over the high `inner_rest_len` // bits of `i`. Each round binds the TOP remaining bit. After `inner_rest_len` rounds, both @@ -1319,7 +1324,9 @@ fn prove_padded_inner( // 6. Send `z_partial` (the post-sumcheck collapsed z_vec). Length 2^k_skip. let z_partial = z_vec.clone(); - ps.add_scalars(&z_partial); + for &x in z_partial.iter() { + ps.add_scalar(x); + } // 7. Sample fresh z_skip AFTER observing z_partial — gives Schwartz-Zippel // soundness on the φ8 (univariate-skip) dim. @@ -1329,7 +1336,7 @@ fn prove_padded_inner( // Equals ẑ_φ8(z_skip, r_rest, x_outer) when z_partial is honest; the // PCS catches mismatches downstream. let lambda = lagrange_weights_naive(k_skip, r_inner_skip); - let w = inner_product(&lambda, &z_partial); + let w = inner_product_ext(&lambda, &z_partial); // 9. Convert sumcheck challenges to LSB-first `x_inner_rest` order. The // loop binds the TOP bit each round, so r_rounds[0] bound bit @@ -1353,15 +1360,15 @@ fn prove_padded_inner( /// Verify a lincheck proof. Walks the sponge in lockstep with the prover, /// replays the α-batched product sumcheck against `v_a` and `v_b`, and /// derives the single output z-claim `w`. -pub fn verify( +pub fn verify( m: usize, k_log: usize, k_skip: usize, circuit: &dyn LincheckCircuit, x_ab: &QuirkyPoint, - v_a: F128, - v_b: F128, - vs: &mut VerifierState<'_>, + v_a: F192, + v_b: F192, + vs: &mut VerifierState<'_, O>, ) -> Result { let k = 1usize << k_log; let n_log = m - k_log; @@ -1408,9 +1415,12 @@ pub fn verify( // 1. Sample α (matches prover's order). let alpha = vs.sample(); - // 2. Build α-batched comb_vec via the circuit's per-block fold (same call - // the prover made — sparse default delegates to the fused row-fold; - // per-hash impls walk the constraint graph directly). + // 2. Row weights: the quirky eq table over the inner claim point — `u` in + // the final bilinear form. The α-batched column marginal the prover + // materializes (`fold_alpha_batched`, cost ∝ NNZ) is NOT built here: + // the verifier only ever consumes it through one inner product, so + // that work is deferred to step 5 (and walk-capable circuits answer it + // in O(circuit) ops without the marginal at all). let t = std::time::Instant::now(); let eq_inner = build_quirky_eq_table(x_ab.z_skip, &x_ab.x_inner_rest, k_skip); if trace { @@ -1419,27 +1429,19 @@ pub fn verify( fmt(t.elapsed().as_secs_f64()) ); } - let t = std::time::Instant::now(); - let mut comb_vec = circuit.fold_alpha_batched(alpha, &eq_inner); - if trace { - eprintln!( - " [lcv] circuit.fold_alpha_batched: {}", - fmt(t.elapsed().as_secs_f64()) - ); - } - // 3. Replay the multilinear product-sumcheck (inner_rest_len rounds), - // folding comb_vec in lockstep so we end up with the "comb_partial" - // vector of length 2^k_skip. Parallel fold for the early (large) rounds. - let t = std::time::Instant::now(); - // Constant-wire pin (mirror of prove): β sampled after α, comb gains +β at - // the constant column, and the initial target gains +β·1 — the honest - // all-ones constant column folds to 1. See lincheck's `LincheckCircuit::const_pin_col`. + // 3. Replay the multilinear product-sumcheck (inner_rest_len rounds). + // Only the transcript messages drive the running claim; the prover's + // lockstep comb_vec fold is linear, so its end state is reconstructed + // in step 5 as column weights instead of being folded here. + // Constant-wire pin (mirror of prove): β sampled after α, the comb's +β + // at the constant column surfaces in step 5 as `+β·w_col[col]`, and the + // initial target gains +β·1 — the honest all-ones constant column folds + // to 1. See lincheck's `LincheckCircuit::const_pin_col`. let mut target = alpha * v_a + v_b; - let mut beta = F128::ZERO; - if let Some(col) = circuit.const_pin_col() { + let mut beta = F192::ZERO; + if circuit.const_pin_col().is_some() { beta = vs.sample(); - comb_vec[col] += beta; target += beta; } let mut running = target; @@ -1451,27 +1453,52 @@ pub fn verify( // q(0) = claim + q(1) in char 2; q(X) = einf·X² + c1·X + e0. let e0 = running + e1; let c1 = e0 + e1 + einf; - running = einf * r * r + c1 * r + e0; - // Fold comb_vec at the same r (mirrors prover's fold). - sumcheck_bind_top_in_place_par(&mut comb_vec, r); + running = (einf * r + c1) * r + e0; r_rounds.push(r); } - debug_assert_eq!(comb_vec.len(), n_skip); + + // 4. Read + bind z_partial AFTER the sumcheck rounds (matches prover order). + let z_partial: Vec = vs.next_scalars(n_skip).map_err(VerifyError::Transcript)?; + + // Convert sumcheck challenges to LSB-first x_inner_rest order (same + // convention as prover; also the eq-ordering of the step-5 column weights). + let mut r_inner_rest = r_rounds.clone(); + r_inner_rest.reverse(); + + // 5. Final sumcheck consistency. The prover's comb_partial — comb_vec + // bound MSB-first at r_rounds — satisfies + // + // ⟨comb_partial, z_partial⟩ = Σ_c comb_vec[c] · w_col[c], + // w_col[i_skip + i_rest·2^k_skip] = z_partial[i_skip] · eq(r_inner_rest, i_rest), + // + // so the whole check collapses to ONE bilinear form + // `eq_innerᵀ·(α·A_0 + B_0)·w_col + β·w_col[pin]` against the running + // claim. Ties z_partial to the upstream v_a, v_b. Walk-capable circuits + // (`bilinear_form`) evaluate it in O(circuit) field ops; the fallback + // materializes the marginal and takes the inner product (identical + // value — exact field arithmetic). + let t = std::time::Instant::now(); + let eq_rest = build_eq(&r_inner_rest); + let mut w_col = Vec::with_capacity(k); + for &er in &eq_rest { + for &zp in &z_partial { + w_col.push(zp * er); + } + } + debug_assert_eq!(w_col.len(), k); + let mut final_sum = match circuit.bilinear_form(alpha, &eq_inner, &w_col) { + Some(v) => v, + None => inner_product_ext(&circuit.fold_alpha_batched(alpha, &eq_inner), &w_col), + }; + if let Some(col) = circuit.const_pin_col() { + final_sum += beta * w_col[col]; + } if trace { eprintln!( - " [lcv] sumcheck replay + comb_vec fold ({} rounds): {}", - inner_rest_len, + " [lcv] final bilinear form (walk or marginal): {}", fmt(t.elapsed().as_secs_f64()) ); } - - // 4. Read + bind z_partial AFTER the sumcheck rounds (matches prover order). - let z_partial = vs.next_scalars(n_skip).map_err(VerifyError::Transcript)?; - - // 5. Final sumcheck consistency: Σ comb_partial[i_skip] · z_partial[i_skip] - // must equal the running claim. Ties z_partial to the upstream v_a, v_b. - // Small (length 2^k_skip = 64); sequential. - let final_sum = inner_product(&comb_vec, &z_partial); if running != final_sum { return Err(VerifyError::ConsistencyFailed { which: "sumcheck-final", @@ -1486,7 +1513,7 @@ pub fn verify( // PCS catches mismatches downstream. let t = std::time::Instant::now(); let lambda = lagrange_weights_naive(k_skip, r_inner_skip); - let w = inner_product(&lambda, &z_partial); + let w = inner_product_ext(&lambda, &z_partial); if trace { eprintln!( " [lcv] final consistency + lagrange_weights_naive: {}", @@ -1494,11 +1521,6 @@ pub fn verify( ); } - // 8. Convert sumcheck challenges to LSB-first x_inner_rest order - // (same convention as prover). - let mut r_inner_rest = r_rounds.clone(); - r_inner_rest.reverse(); - Ok(LincheckClaim { alpha, beta, @@ -1527,10 +1549,9 @@ mod tests { k_skip: usize, circuit: &dyn LincheckCircuit, x_ab: &QuirkyPoint, - ps: &mut ProverState, + ps: &mut pcs::ProverState, ) -> LincheckClaim { - let (claim, _) = - prove_padded_capture_z_vec(z_packed, m, k_log, k_skip, 1 << k_log, circuit, x_ab, ps); + let (claim, _) = prove_padded_capture_z_vec(z_packed, m, k_log, k_skip, 1 << k_log, circuit, x_ab, ps); claim } @@ -1539,16 +1560,16 @@ mod tests { z_packed: &[u8], m: usize, k_log: usize, - eq_outer: &[F128], - ) -> Vec { + eq_outer: &[F192], + ) -> Vec { partial_fold_packed_z_fast_padded(z_packed, m, k_log, 1 << k_log, eq_outer) } /// Reference fold `M_0^T · eq` (the row-MLE at all boolean column indices), /// used to locate meaningful mutation targets and as a dense oracle. - fn sparse_row_fold(matrix: &SparseBinaryMatrix, eq_table: &[F128]) -> Vec { + fn sparse_row_fold(matrix: &SparseBinaryMatrix, eq_table: &[F192]) -> Vec { assert_eq!(eq_table.len(), matrix.num_rows); - let mut out = vec![F128::ZERO; matrix.num_cols]; + let mut out = vec![F192::ZERO; matrix.num_cols]; for (row_idx, row) in matrix.rows.iter().enumerate() { let e = eq_table[row_idx]; for &col in row { @@ -1560,11 +1581,11 @@ mod tests { /// Naive MLE evaluation: `f̂(point) = Σ_i eq(point, i) · f[i]` where i ∈ /// {0,1}^d and f[i] is given as a bool slice. - fn mle_eval_bool(f: &[bool], point: &[F128]) -> F128 { + fn mle_eval_bool(f: &[bool], point: &[F192]) -> F192 { let d = point.len(); assert_eq!(f.len(), 1 << d); let eq = build_eq(point); - let mut acc = F128::ZERO; + let mut acc = F192::ZERO; for (i, &b) in f.iter().enumerate() { if b { acc += eq[i]; @@ -1573,13 +1594,13 @@ mod tests { acc } - /// Sample a random `QuirkyPoint` for testing: z_skip ∈ F₁₂₈, + /// Sample a random `QuirkyPoint` for testing: z_skip ∈ F₁₉₂, /// x_inner_rest of length `k_log − k_skip`, x_outer of length `n_log`. fn random_quirky_point(m: usize, k_log: usize, k_skip: usize, rng: &mut Rng) -> QuirkyPoint { QuirkyPoint { - z_skip: rng.f128(), - x_inner_rest: rng.f128_vec(k_log - k_skip), - x_outer: rng.f128_vec(m - k_log), + z_skip: rng.ext(), + x_inner_rest: rng.ext_vec(k_log - k_skip), + x_outer: rng.ext_vec(m - k_log), } } @@ -1591,13 +1612,7 @@ mod tests { /// /// where `i = i_skip + 2^k_skip · i_inner_rest + 2^k_log · i_outer` (matches /// the linear-LSB indexing of `f`). - fn mle_eval_bool_quirky( - f: &[bool], - m: usize, - k_log: usize, - k_skip: usize, - point: &QuirkyPoint, - ) -> F128 { + fn mle_eval_bool_quirky(f: &[bool], m: usize, k_log: usize, k_skip: usize, point: &QuirkyPoint) -> F192 { let k_skip_dim = 1usize << k_skip; let inner_rest_len = k_log - k_skip; let inner_rest_dim = 1usize << inner_rest_len; @@ -1605,14 +1620,16 @@ mod tests { let n_outer = 1usize << (m - k_log); assert_eq!(f.len(), 1 << m); - let lambda = crate::zerocheck::multilinear::lagrange_weights_naive(k_skip, point.z_skip); + // Tower helpers: the point is F192 (the verifier's field), and the + // expected value must equal the F192 claim the verifier derives. + let lambda = lagrange_weights_naive(k_skip, point.z_skip); let eq_rest = build_eq(&point.x_inner_rest); let eq_outer = build_eq(&point.x_outer); debug_assert_eq!(lambda.len(), k_skip_dim); debug_assert_eq!(eq_rest.len(), inner_rest_dim); debug_assert_eq!(eq_outer.len(), n_outer); - let mut acc = F128::ZERO; + let mut acc = F192::ZERO; for i in 0..(1 << m) { if !f[i] { continue; @@ -1689,19 +1706,15 @@ mod tests { fn eq_table_matches_direct_formula() { for &d in &[1usize, 2, 3, 5, 8] { let mut rng = Rng::new(11 + d as u64); - let point = rng.f128_vec(d); + let point = rng.ext_vec(d); let table = build_eq(&point); assert_eq!(table.len(), 1 << d); for i in 0..(1 << d) { - let mut expected = F128::ONE; + let mut expected = F192::ONE; for j in 0..d { let bit = ((i >> j) & 1) as u64; // eq(r, bit) = (1 + r) if bit = 0 else r - let factor = if bit == 0 { - F128::ONE + point[j] - } else { - point[j] - }; + let factor = if bit == 0 { F192::ONE + point[j] } else { point[j] }; expected *= factor; } assert_eq!(table[i], expected, "mismatch at d={d}, i={i}"); @@ -1716,12 +1729,12 @@ mod tests { let k = 16; let nnz = 40; let matrix = random_sparse_matrix(k, nnz, &mut rng); - let eq_table: Vec = rng.f128_vec(k); + let eq_table: Vec = rng.ext_vec(k); let got = sparse_row_fold(&matrix, &eq_table); // Brute force: for each col j, sum eq[i] over rows i where M[i,j] = 1. - let mut expected = vec![F128::ZERO; k]; + let mut expected = vec![F192::ZERO; k]; for (i, row) in matrix.rows.iter().enumerate() { for &j in row { expected[j] += eq_table[i]; @@ -1738,7 +1751,7 @@ mod tests { let z = rng.bits(1 << m); let z_packed = pack_z_lincheck(&z, m, k_log); let n_log = m - k_log; - let outer_point = rng.f128_vec(n_log); + let outer_point = rng.ext_vec(n_log); let eq_outer = build_eq(&outer_point); let got = partial_fold_packed_z(&z_packed, m, k_log, &eq_outer); @@ -1746,7 +1759,7 @@ mod tests { let k = 1usize << k_log; assert_eq!(got.len(), k); for i_inner in 0..k { - let mut acc = F128::ZERO; + let mut acc = F192::ZERO; for i_outer in 0..(1usize << n_log) { let i = i_inner + i_outer * k; if z[i] { @@ -1767,7 +1780,7 @@ mod tests { let z = rng.bits(1 << m); let z_packed = pack_z_lincheck(&z, m, k_log); let n_log = m - k_log; - let p = rng.f128_vec(n_log); + let p = rng.ext_vec(n_log); let eq = build_eq(&p); let serial = partial_fold_packed_z(&z_packed, m, k_log, &eq); @@ -1788,12 +1801,11 @@ mod tests { let z = rng.bits(1 << m); let z_packed = pack_z_lincheck(&z, m, k_log); let n_log = m - k_log; - let p = rng.f128_vec(n_log); + let p = rng.ext_vec(n_log); let eq = build_eq(&p); let serial = partial_fold_packed_z(&z_packed, m, k_log, &eq); - let iblock = - partial_fold_packed_z_neon_iblock_padded(&z_packed, m, k_log, 1usize << k_log, &eq); + let iblock = partial_fold_packed_z_neon_iblock_padded(&z_packed, m, k_log, 1usize << k_log, &eq); assert_eq!(serial, iblock, "iblock at m={m}, k_log={k_log}"); } } @@ -1815,10 +1827,7 @@ mod tests { (22, 14, 15_409), // padded, non-byte-aligned (k=16384) ]; for &(m, k_log, useful_bits) in cases { - assert!( - n_log_ok_for_tile(m, k_log, NEON_TILE_T), - "case must be tile-eligible" - ); + assert!(n_log_ok_for_tile(m, k_log, NEON_TILE_T), "case must be tile-eligible"); let k = 1usize << k_log; let n_log = m - k_log; let n_blocks = 1usize << n_log; @@ -1831,11 +1840,9 @@ mod tests { } } let z_packed = pack_z_lincheck(&z, m, k_log); - let eq = build_eq(&rng.f128_vec(n_log)); - let want = - partial_fold_packed_z_neon_iblock_padded(&z_packed, m, k_log, useful_bits, &eq); - let got = - partial_fold_packed_z_neon_oblock_padded(&z_packed, m, k_log, useful_bits, &eq); + let eq = build_eq(&rng.ext_vec(n_log)); + let want = partial_fold_packed_z_neon_iblock_padded(&z_packed, m, k_log, useful_bits, &eq); + let got = partial_fold_packed_z_neon_oblock_padded(&z_packed, m, k_log, useful_bits, &eq); assert_eq!(want, got, "m={m} k_log={k_log} useful={useful_bits}"); } } @@ -1876,12 +1883,11 @@ mod tests { } } let z_packed = pack_z_lincheck(&z, m, k_log); - let outer_point = rng.f128_vec(n_log); + let outer_point = rng.ext_vec(n_log); let eq_outer = build_eq(&outer_point); let dense_fast = partial_fold_packed_z_fast_padded_dense(&z_packed, m, k_log, &eq_outer); - let padded_fast = - partial_fold_packed_z_fast_padded(&z_packed, m, k_log, useful_bits, &eq_outer); + let padded_fast = partial_fold_packed_z_fast_padded(&z_packed, m, k_log, useful_bits, &eq_outer); assert_eq!( dense_fast, padded_fast, "fast: m={m}, k_log={k_log}, useful={useful_bits}" @@ -1889,39 +1895,18 @@ mod tests { #[cfg(target_arch = "aarch64")] if n_log_ok_for_tile(m, k_log, NEON_TILE_T) { - let dense_neon = partial_fold_packed_z_neon_iblock_padded( - &z_packed, - m, - k_log, - 1usize << k_log, - &eq_outer, - ); - let padded_neon = partial_fold_packed_z_neon_iblock_padded( - &z_packed, - m, - k_log, - useful_bits, - &eq_outer, - ); + let dense_neon = + partial_fold_packed_z_neon_iblock_padded(&z_packed, m, k_log, 1usize << k_log, &eq_outer); + let padded_neon = partial_fold_packed_z_neon_iblock_padded(&z_packed, m, k_log, useful_bits, &eq_outer); assert_eq!( dense_neon, padded_neon, "neon: m={m}, k_log={k_log}, useful={useful_bits}" ); // i_inner-partitioned kernel: dense and padded must both match. - let dense_iblock = partial_fold_packed_z_neon_iblock_padded( - &z_packed, - m, - k_log, - 1usize << k_log, - &eq_outer, - ); - let padded_iblock = partial_fold_packed_z_neon_iblock_padded( - &z_packed, - m, - k_log, - useful_bits, - &eq_outer, - ); + let dense_iblock = + partial_fold_packed_z_neon_iblock_padded(&z_packed, m, k_log, 1usize << k_log, &eq_outer); + let padded_iblock = + partial_fold_packed_z_neon_iblock_padded(&z_packed, m, k_log, useful_bits, &eq_outer); assert_eq!( dense_neon, dense_iblock, "iblock dense: m={m}, k_log={k_log}, useful={useful_bits}" @@ -1944,7 +1929,7 @@ mod tests { let mut rng = Rng::new(44); let z = rng.bits(1 << m); let z_packed = pack_z_lincheck(&z, m, k_log); - let x_outer = rng.f128_vec(m - k_log); + let x_outer = rng.ext_vec(m - k_log); let eq_outer = build_eq(&x_outer); let z_partial = partial_fold_packed_z(&z_packed, m, k_log, &eq_outer); @@ -1956,11 +1941,7 @@ mod tests { // then m-k_log coords from x_outer. let mut point = Vec::with_capacity(m); for j in 0..k_log { - point.push(if (i_inner >> j) & 1 == 1 { - F128::ONE - } else { - F128::ZERO - }); + point.push(if (i_inner >> j) & 1 == 1 { F192::ONE } else { F192::ZERO }); } point.extend_from_slice(&x_outer); let z_eval = mle_eval_bool(&z, &point); @@ -2017,10 +1998,7 @@ mod tests { let proof_t = ch_p.into_proof(); let mut ch_v = pcs::VerifierState::new(b"flock-test-v0", &proof_t, &[]); - let claim_v = verify( - m, k_log, k_skip, &circuit, &x_ab, v_a, v_b, &mut ch_v, - ) - .unwrap_or_else(|e| { + let claim_v = verify(m, k_log, k_skip, &circuit, &x_ab, v_a, v_b, &mut ch_v).unwrap_or_else(|e| { panic!("verify rejected honest proof at m={m},k_log={k_log},k_skip={k_skip}: {e:?}") }); @@ -2074,11 +2052,13 @@ mod tests { // Pick a mutation position where BOTH row vectors are nonzero so the // mutation guarantees both checks would diverge. - let eq_inner = build_quirky_eq_table(x_ab.z_skip, &x_ab.x_inner_rest, k_skip); + let z_skip_g = x_ab.z_skip; + let x_inner_rest_g = x_ab.x_inner_rest.to_vec(); + let eq_inner = build_quirky_eq_table(z_skip_g, &x_inner_rest_g, k_skip); let row_a = sparse_row_fold(&a_0, &eq_inner); let row_b = sparse_row_fold(&b_0, &eq_inner); let idx = (0..k) - .find(|&i| row_a[i] != F128::ZERO || row_b[i] != F128::ZERO) + .find(|&i| row_a[i] != F192::ZERO || row_b[i] != F192::ZERO) .expect("no row-vector slot is nonzero in either A or B — test degenerate"); // Mutations target `z_partial` (the post-sumcheck length-2^k_skip @@ -2091,9 +2071,9 @@ mod tests { for (label, hi) in [("lo", false), ("hi", true)] { let mut bad = proof_t.clone(); if hi { - bad.stream[zp_word].hi ^= 1; + bad.stream[zp_word].c1 ^= 1; } else { - bad.stream[zp_word].lo ^= 1; + bad.stream[zp_word].c0 ^= 1; } let mut ch = pcs::VerifierState::new(b"flock-test-v0", &bad, &[]); let res = verify(m, k_log, k_skip, &circuit, &x_ab, v_a, v_b, &mut ch); @@ -2160,14 +2140,14 @@ mod tests { /// general-purpose `fold_1b_rows` over the materialized suffix tensor. #[test] fn s_hat_v_from_z_vec_matches_fold_1b_rows_ab() { - const K_SKIP: usize = 6; + const K_SKIP: usize = 6; // (m, k_log) — K_SKIP fixed at 6 (so x_inner_rest has k_log − 6 coords; // x_inner_rest[0] becomes ring-switch's prefix0 because // K_SKIP + 1 = LOG_PACKING = 7). n_log = m − k_log must be ≥ 3 for // partial_fold_packed_z's stripe layout. let cases: &[(usize, usize)] = &[(13, 10), (15, 11), (17, 13)]; for &(m, k_log) in cases { - assert!(k_log >= pcs::LOG_PACKING); + assert!(k_log >= pcs::pack::LOG_PACKING); assert!(k_log >= K_SKIP); let n_log = m - k_log; assert!(n_log >= 3); @@ -2180,23 +2160,22 @@ mod tests { // AB-shaped quirky point: x_inner_rest has k_log − K_SKIP coords; // x_outer has n_log coords. - let x_inner_rest: Vec = (0..(k_log - K_SKIP)).map(|_| rng.f128()).collect(); - let x_outer: Vec = (0..n_log).map(|_| rng.f128()).collect(); + let x_inner_rest: Vec = (0..(k_log - K_SKIP)).map(|_| rng.ext()).collect(); + let x_outer: Vec = (0..n_log).map(|_| rng.ext()).collect(); // Reference: ring-switch's fold_1b_rows over the materialized // suffix tensor, exactly the path open_batch hits today. let mut x_outer_full = Vec::with_capacity(x_inner_rest.len() + x_outer.len()); x_outer_full.extend_from_slice(&x_inner_rest); x_outer_full.extend_from_slice(&x_outer); - let suffix = &x_outer_full[1..]; - let suffix_tensor = primitives::multilinear::build_eq(suffix); - let want = pcs::ring_switch::fold_1b_rows_naive(&packed, &suffix_tensor); + let suffix_tensor = primitives::multilinear::eq_table(&x_outer_full); + let want = pcs::ring_switch::fold_1b_rows(&packed, &suffix_tensor); // New path: lincheck-shaped partial fold of z at x_outer, then a // strided fold against the inner-rest tail. - let eq_x_outer = primitives::multilinear::build_eq(&x_outer); + let eq_x_outer = primitives::multilinear::eq_table(&x_outer); let z_vec = partial_fold_packed_z(&z_packed_lincheck, m, k_log, &eq_x_outer); - let got = pcs::ring_switch::s_hat_v_from_z_vec(&z_vec, &x_inner_rest[1..]); + let got = pcs::ring_switch::s_hat_v_from_z_vec(&z_vec, &x_inner_rest); assert_eq!(got, want, "s_hat_v mismatch at m={m}, k_log={k_log}"); } diff --git a/crates/flock/src/proof.rs b/crates/flock/src/proof.rs index af7e33dd..f34e7fdd 100644 --- a/crates/flock/src/proof.rs +++ b/crates/flock/src/proof.rs @@ -1,13 +1,14 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. //! The evaluation-claim type shared by the zerocheck/lincheck reduction and //! the PCS. -use primitives::field::F128; use crate::lincheck::QuirkyPoint; +use primitives::field::F192; -/// A claim of the form `ẑ(point) = value` for the witness `z`. +/// A claim of the form `ẑ(point) = value` for the witness `z`. Tower-valued: +/// the flock verifier and the downstream PCS run over `F192`. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ZClaim { pub point: QuirkyPoint, - pub value: F128, + pub value: F192, } diff --git a/crates/flock/src/r1cs.rs b/crates/flock/src/r1cs.rs index 4744919c..508c2db7 100644 --- a/crates/flock/src/r1cs.rs +++ b/crates/flock/src/r1cs.rs @@ -1,4 +1,4 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. //! Block-diagonal R1CS over GF(2). //! //! The standard R1CS is `(A·z) ⊙ (B·z) ⊕ (C·z) = 0`. We fix `C = I` (the @@ -35,9 +35,9 @@ pub enum WitnessLayout { /// /// `k_skip` is the zerocheck's univariate-skip dimension (`k_skip ≤ k_log`). /// It defines how the m-dim claim point is laid out in the protocol: one -/// univariate F128 coord binds the LSB `k_skip` bits, `k_log − k_skip` -/// multilinear F128 coords bind the next inner bits, and `n_log` multilinear -/// F128 coords bind the outer bits. +/// univariate F192 coord binds the LSB `k_skip` bits, `k_log − k_skip` +/// multilinear F192 coords bind the next inner bits, and `n_log` multilinear +/// F192 coords bind the outer bits. #[derive(Debug)] pub struct BlockR1cs { pub m: usize, @@ -103,8 +103,7 @@ impl BlockR1cs { /// over the nonzeros) out of the prove path. pub fn csc_lincheck_circuit(&self) -> &crate::lincheck::CscCircuit { self.csc_cache.get_or_init(|| { - crate::lincheck::CscCircuit::from_matrices(&self.a_0, &self.b_0) - .with_const_pin(self.const_pin) + crate::lincheck::CscCircuit::from_matrices(&self.a_0, &self.b_0).with_const_pin(self.const_pin) }) } @@ -157,7 +156,6 @@ impl BlockR1cs { absorb_matrix(&mut h, &self.c_0); *h.finalize().as_bytes() } - } /// Length-prefixed absorption of a sparse matrix into a BLAKE3 hasher. diff --git a/crates/flock/src/test_rng.rs b/crates/flock/src/test_rng.rs index 14f2e021..4bb824c0 100644 --- a/crates/flock/src/test_rng.rs +++ b/crates/flock/src/test_rng.rs @@ -1,4 +1,4 @@ -use primitives::field::F128; +use primitives::field::F192; /// Deterministic SplitMix64 generator shared by the crate's unit tests. pub(crate) struct Rng(u64); @@ -28,11 +28,11 @@ impl Rng { (0..n).map(|_| self.bit()).collect() } - pub(crate) fn f128(&mut self) -> F128 { - F128::new(self.next_u64(), self.next_u64()) + pub(crate) fn ext(&mut self) -> F192 { + F192::new(self.next_u64(), self.next_u64(), self.next_u64()) } - pub(crate) fn f128_vec(&mut self, n: usize) -> Vec { - (0..n).map(|_| self.f128()).collect() + pub(crate) fn ext_vec(&mut self, n: usize) -> Vec { + (0..n).map(|_| self.ext()).collect() } } diff --git a/crates/flock/src/verifier.rs b/crates/flock/src/verifier.rs index 4cf434e3..f9e756bd 100644 --- a/crates/flock/src/verifier.rs +++ b/crates/flock/src/verifier.rs @@ -1,4 +1,4 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. //! Errors of the R1CS reduction (zerocheck + lincheck + PCS opening). use crate::lincheck; @@ -9,5 +9,4 @@ use crate::zerocheck; pub enum VerifyError { Zerocheck(zerocheck::VerifyError), Lincheck(lincheck::VerifyError), - Pcs(::pcs::VerifyError), } diff --git a/crates/flock/src/zerocheck.rs b/crates/flock/src/zerocheck.rs index 48a37a7b..60301388 100644 --- a/crates/flock/src/zerocheck.rs +++ b/crates/flock/src/zerocheck.rs @@ -1,13 +1,13 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. //! Zerocheck PIOP: prove a(y) · b(y) ⊕ c(y) = 0 for all y ∈ {0,1}^m. //! //! Inputs are three bit vectors of length 2^m. Output is an evaluation claim //! on the multilinear extensions â, b̂, ĉ at the protocol-derived point. //! //! Protocol shape (m = log_n, k_skip = [`K_SKIP`] = 6): -//! 1. Verifier samples `r ∈ F_{2^128}^m` (the zerocheck challenge). +//! 1. Verifier samples `r ∈ F_{2^192}^m` (the zerocheck challenge). //! 2. Prover sends `P^{AB}(λ)` and `P^C(λ)` for λ ∈ Λ, |Λ| = 2^k_skip. -//! 3. Verifier samples `z ∈ F_{2^128}` (univariate-skip fold point). +//! 3. Verifier samples `z ∈ F_{2^192}` (univariate-skip fold point). //! 4. For each of the `m - k_skip` multilinear rounds, prover sends //! `(P_r(1), P_r(∞))` and verifier samples `ρ_r`. //! 5. Prover sends final MLE evaluations `(â, b̂, ĉ)` at the resulting point. @@ -16,8 +16,9 @@ //! is tested on honest witnesses; verify also rejects byte-mutated proofs and //! shape-corrupted ones. -use pcs::{ProverState, VerifierState}; -use primitives::field::{F8, F128}; +use fiat_shamir::transcript::{ProverState, VerifierState}; +use primitives::field::{F8, F192}; + use pcs::ntt::{AdditiveNttGf8, InvNttTableByteSingleGf8}; pub mod multilinear; @@ -25,28 +26,27 @@ pub mod univariate_skip; pub mod univariate_skip_optimized; use multilinear::{ - UniSkipFoldTable, fold_and_compute_round_pair_into, fold_in_place_pair, - interpolate_at_z_combined, interpolate_at_z_on_lambda, round_pair_naive, - uni_skip_fold_and_round_pair_optimized_packed_padded, + UniSkipFoldTable, fold_and_compute_round_pair_into, fold_in_place_pair, interpolate_at_z_combined, + interpolate_at_z_on_lambda, round_pair_naive, uni_skip_fold_and_round_pair_optimized_packed_padded, }; use univariate_skip_optimized::{ - c_s_f128, medium_challenges_ghash, small_challenges_ghash, + c_s, medium_challenges, round1_shift_reduce_extract_c_packed_padded, small_challenges, }; /// Number of variables folded in round 1 via the additive-NTT univariate skip. /// |Λ| = 2^K_SKIP = 64 elements; the round-1 prover message is two length-64 -/// vectors of F128. +/// vectors of F192. pub const K_SKIP: usize = 6; const N_INNER: usize = 7; // 3 small + 4 medium fixed-constant eq dimensions /// Build the zerocheck challenge vector in the shared prover/verifier order: /// sampled skip coordinates, fixed inner coordinates, then sampled outer ones. -fn challenge_vector(m: usize, mut sample_vec: impl FnMut(usize) -> Vec) -> Vec { +fn challenge_vector(m: usize, mut sample_vec: impl FnMut(usize) -> Vec) -> Vec { let skip = sample_vec(K_SKIP); let outer = sample_vec(m - K_SKIP - N_INNER); skip.into_iter() - .chain(small_challenges_ghash()) - .chain(medium_challenges_ghash()) + .chain(small_challenges()) + .chain(medium_challenges()) .chain(outer) .collect() } @@ -82,20 +82,20 @@ pub use pcs::pack::PaddingSpec; #[derive(Clone, Debug, PartialEq, Eq)] pub struct ZerocheckClaim { /// Univariate-skip challenge sampled after round 1 (binds the K_SKIP - /// skip variables). - pub z: F128, + /// skip variables), represented directly in `F192`. + pub z: F192, /// AB sumcheck bind challenges, one per multilinear round; length = `m - K_SKIP`. - pub mlv_challenges: Vec, + pub mlv_challenges: Vec, /// Eq weights for the rest variables = the zerocheck challenge restricted /// to `r[K_SKIP..m]`. This is the *rest part of the c-claim's point*. /// Length = `m - K_SKIP`. - pub r_rest: Vec, + pub r_rest: Vec, /// `â(z, mlv_challenges)`. - pub a_eval: F128, + pub a_eval: F192, /// `b̂(z, mlv_challenges)`. - pub b_eval: F128, + pub b_eval: F192, /// `ĉ(z, r_rest)` — at a *different point* than a_eval, b_eval. - pub c_eval: F128, + pub c_eval: F192, } // (No ZerocheckProof struct: every round message rides the shared transcript @@ -126,15 +126,28 @@ pub enum VerifyError { /// ([`univariate_skip_optimized::round1_shift_reduce_extract_c_packed_padded_with_s_hat_v`]), /// which the downstream PCS open consumes to skip `fold_1b_rows` for the /// c-claim. +pub fn prove_packed_padded_capture_s_hat_v_c( + a_packed: &[u8], + b_packed: &[u8], + c_packed: &[u8], + m: usize, + padding: &PaddingSpec, + ps: &mut ProverState, +) -> (ZerocheckClaim, Vec) { + let (claim, captured) = prove_packed_padded_inner(a_packed, b_packed, c_packed, m, padding, true, ps); + (claim, captured.expect("capture=true must produce s_hat_v_c")) +} + #[allow(clippy::too_many_arguments)] -pub fn prove_packed_padded( +fn prove_packed_padded_inner( a_packed: &[u8], b_packed: &[u8], c_packed: &[u8], m: usize, padding: &PaddingSpec, - ps: &mut ProverState, -) -> (ZerocheckClaim, Vec) { + capture_s_hat_v_c: bool, + ps: &mut ProverState, +) -> (ZerocheckClaim, Option>) { let k_skip = K_SKIP; assert!( m >= k_skip + N_INNER, @@ -156,7 +169,8 @@ pub fn prove_packed_padded( // r[k_skip+3..k_skip+7] — protocol medium-eq constants β_i // r[k_skip+7..m] — sampled (the "outer" eq weights for // the URM and multilinear rounds) - let r = challenge_vector(m, |n| ps.sample_vec(n)); + // Prover and verifier use the same tower-valued challenges directly. + let r = challenge_vector(m, |n| ps.sample_vec(n).into_iter().collect()); // ---- 3. Round 1: URM (extract_c, parallel) ---- // @@ -170,20 +184,21 @@ pub fn prove_packed_padded( let ntt_s = AdditiveNttGf8::new(k_skip, F8::ZERO); let ntt_l = AdditiveNttGf8::new(k_skip, F8(1u8 << k_skip)); let inv_table = InvNttTableByteSingleGf8::new(&ntt_s, &ntt_l); - let (round1_ab_opt, round1_c_opt, s_hat_v_c) = - crate::zerocheck::univariate_skip_optimized::round1_shift_reduce_extract_c_packed_padded_with_s_hat_v( - a_packed, - b_packed, - c_packed, - m, - k_skip, - &r, - &inv_table, - padding, + let (round1_ab_opt, round1_c_opt, s_hat_v_c) = if capture_s_hat_v_c { + let (ab, c, s) = + crate::zerocheck::univariate_skip_optimized::round1_shift_reduce_extract_c_packed_padded_with_s_hat_v( + a_packed, b_packed, c_packed, m, k_skip, &r, &inv_table, padding, + ); + (ab, c, Some(s)) + } else { + let (ab, c) = round1_shift_reduce_extract_c_packed_padded( + a_packed, b_packed, c_packed, m, k_skip, &r, &inv_table, padding, ); - let c_s = c_s_f128(); - let round1_ab: Vec = round1_ab_opt.iter().map(|x| c_s * *x).collect(); - let round1_c: Vec = round1_c_opt.iter().map(|x| c_s * *x).collect(); + (ab, c, None) + }; + let c_s = c_s(); + let round1_ab: Vec = round1_ab_opt.iter().map(|x| c_s * *x).collect(); + let round1_c: Vec = round1_c_opt.iter().map(|x| c_s * *x).collect(); if zc_timing { eprintln!( "[zc-timing] round1 URM: {:.2} ms", @@ -192,8 +207,12 @@ pub fn prove_packed_padded( } // ---- 4. Transmit + bind round-1 message on the stream, sample z ---- - ps.add_scalars(&round1_ab); - ps.add_scalars(&round1_c); + for &x in round1_ab.iter() { + ps.add_scalar(x); + } + for &x in round1_c.iter() { + ps.add_scalar(x); + } let z = ps.sample(); // ---- 5. c_eval = ĉ(z, r_rest) via interpolation of round1_c at z ---- @@ -202,7 +221,7 @@ pub fn prove_packed_padded( // as its 2^k_skip evaluations on Λ. Interpolating to λ=z gives // `ĉ(z, r_rest)` directly (the eq-weighted sum collapses to the MLE // evaluation because ĉ is linear). This is **the c-claim** — at point - // `(z, r_rest)`, *not* `(z, ρ-values)`. ~64 F128 muls + Lagrange weights. + // `(z, r_rest)`, *not* `(z, ρ-values)`. ~64 F192 muls + Lagrange weights. let final_c_eval = interpolate_at_z_on_lambda(&round1_c, k_skip, z); // ---- 6. Round 2: fused fold + first multilinear message ---- @@ -212,18 +231,17 @@ pub fn prove_packed_padded( // verifier samples ρ_1 after observing this message. let t_round2 = std::time::Instant::now(); let fold_table = UniSkipFoldTable::new(k_skip, z); - let mut mlv_arg = vec![F128::ONE; n_mlv]; + let mut mlv_arg = vec![F192::ONE; n_mlv]; mlv_arg[1..].copy_from_slice(&r[k_skip + 1..]); - let (mut a_mlv, mut b_mlv, msg_1, msg_inf) = - uni_skip_fold_and_round_pair_optimized_packed_padded( - a_packed, - b_packed, - m, - k_skip, - &fold_table, - &mlv_arg, - padding, - ); + let (mut a_mlv, mut b_mlv, msg_1, msg_inf) = uni_skip_fold_and_round_pair_optimized_packed_padded( + a_packed, + b_packed, + m, + k_skip, + &fold_table, + &mlv_arg, + padding, + ); if zc_timing { eprintln!( @@ -236,14 +254,14 @@ pub fn prove_packed_padded( multilinear_msgs.push((msg_1, msg_inf)); ps.add_scalar(msg_1); ps.add_scalar(msg_inf); - let mut mlv_rhos: Vec = Vec::with_capacity(n_mlv); + let mut mlv_rhos: Vec = Vec::with_capacity(n_mlv); mlv_rhos.push(ps.sample()); // ---- 7. Rounds 3..(n_mlv + 1) — AB only (c is done) ---- // // Iter i: fold (a, b) at ρ_{i+1}, compute round (i+3) message, sample // ρ_{i+2}. Use the fused parallel path while log_n ≥ 10; below that the - // SplitEqGhash inner can't form lo_size ≥ 2, so we fall back to + // SplitEq inner can't form lo_size ≥ 2, so we fall back to // fold_in_place_pair + round_pair_naive. // // Ping-pong scratch buffers for the fused path: each fused round folds @@ -255,8 +273,8 @@ pub fn prove_packed_padded( let n_in = a_mlv.len(); let (mut a_nxt, mut b_nxt) = if n_in >= 1024 { ( - primitives::scratch::take_f128(n_in / 2), - primitives::scratch::take_f128(n_in / 2), + primitives::scratch::take_f192(n_in / 2), + primitives::scratch::take_f192(n_in / 2), ) } else { (Vec::new(), Vec::new()) @@ -269,7 +287,7 @@ pub fn prove_packed_padded( // r_next for the next round's message: length log_n_before - 1. // r_next[0] = ONE (Convention A factor); r_next[1..] are the eq // weights for the remaining variables = r[k_skip + i + 2..m]. - let mut r_next = vec![F128::ONE; log_n_before - 1]; + let mut r_next = vec![F192::ONE; log_n_before - 1]; r_next[1..].copy_from_slice(&r[k_skip + i + 2..]); let (m1, mi) = if log_n_before >= 10 { @@ -327,10 +345,10 @@ pub fn prove_packed_padded( // Recycle the four tail buffers (the two len-1 survivors still own their // full round-2 capacity) for the next phase/prove. - primitives::scratch::give_f128(a_mlv); - primitives::scratch::give_f128(b_mlv); - primitives::scratch::give_f128(a_nxt); - primitives::scratch::give_f128(b_nxt); + primitives::scratch::give_f192(a_mlv); + primitives::scratch::give_f192(b_mlv); + primitives::scratch::give_f192(a_nxt); + primitives::scratch::give_f192(b_nxt); if zc_timing { eprintln!( @@ -339,12 +357,10 @@ pub fn prove_packed_padded( ); } - let r_rest: Vec = r[k_skip..].to_vec(); - let claim = ZerocheckClaim { z, - mlv_challenges: mlv_rhos, - r_rest, + mlv_challenges: mlv_rhos.to_vec(), + r_rest: r[k_skip..].to_vec(), a_eval: final_a_eval, b_eval: final_b_eval, c_eval: final_c_eval, @@ -360,10 +376,7 @@ pub fn prove_packed_padded( /// On accept: returns the [`ZerocheckClaim`] the caller must check against /// its PCS opening of `â`, `b̂`, `ĉ`. /// On reject: returns a [`VerifyError`] indicating which check failed. -pub fn verify( - log_n: usize, - vs: &mut VerifierState<'_>, -) -> Result { +pub fn verify(log_n: usize, vs: &mut VerifierState<'_, O>) -> Result { let m = log_n; let k_skip = K_SKIP; @@ -374,11 +387,12 @@ pub fn verify( let ell = 1usize << k_skip; // ---- Re-derive r (in lockstep with prove_packed) ---- + // The verifier samples tower challenges directly, matching the prover. let r = challenge_vector(m, |n| vs.sample_vec(n)); // ---- Read + bind round-1 messages off the stream, sample z ---- - let round1_ab = vs.next_scalars(ell).map_err(VerifyError::Transcript)?; - let round1_c = vs.next_scalars(ell).map_err(VerifyError::Transcript)?; + let round1_ab: Vec = vs.next_scalars(ell).map_err(VerifyError::Transcript)?; + let round1_c: Vec = vs.next_scalars(ell).map_err(VerifyError::Transcript)?; let z = vs.sample(); // ---- Reconstruct ĉ(z, r_rest) from round1_c ---- @@ -402,11 +416,7 @@ pub fn verify( // If the prover's witness is dishonest the S-zero assumption fails, the // reconstructed c_0 is wrong, and the running-claim chain ends at a value // inconsistent with `â · b̂`. We catch that at the final sumcheck check. - let combined_at_lambda: Vec = round1_ab - .iter() - .zip(&round1_c) - .map(|(x, y)| *x + *y) - .collect(); + let combined_at_lambda: Vec = round1_ab.iter().zip(&round1_c).map(|(x, y)| *x + *y).collect(); let combined_at_z = interpolate_at_z_combined(&combined_at_lambda, k_skip, z); let p_c_at_z = interpolate_at_z_on_lambda(&round1_c, k_skip, z); let mut c_running = combined_at_z + p_c_at_z; @@ -427,14 +437,14 @@ pub fn verify( // 3. update `c_running ← G(ρ_i)`, // where `G(X) = G(0)·(1+X) + G(1)·X + G(∞)·X·(X+1)` (char-2 quadratic // interpolation through G(0), G(1), G(∞)). - let mut mlv_rhos: Vec = Vec::with_capacity(n_mlv); - let mut multilinear_rounds: Vec<(F128, F128)> = Vec::with_capacity(n_mlv); + let mut mlv_rhos: Vec = Vec::with_capacity(n_mlv); + let mut multilinear_rounds: Vec<(F192, F192)> = Vec::with_capacity(n_mlv); for i in 0..n_mlv { let msg_1 = vs.next_scalar().map_err(VerifyError::Transcript)?; let msg_inf = vs.next_scalar().map_err(VerifyError::Transcript)?; multilinear_rounds.push((msg_1, msg_inf)); let r_eq = r[k_skip + i]; - let one_plus_r_eq = F128::ONE + r_eq; + let one_plus_r_eq = F192::ONE + r_eq; let g1 = msg_1; let g_inf = msg_inf; @@ -443,9 +453,8 @@ pub fn verify( let rho = vs.sample(); mlv_rhos.push(rho); - let one_plus_rho = F128::ONE + rho; // G(ρ) = G(0)·(1+ρ) + G(1)·ρ + G(∞)·ρ·(1+ρ). - c_running = g0 * one_plus_rho + g1 * rho + g_inf * rho * one_plus_rho; + c_running = g0 + rho * (g0 + g1 + (F192::ONE + rho) * g_inf); } // ---- AB sumcheck final consistency ---- @@ -456,11 +465,11 @@ pub fn verify( // (The eq factors were absorbed round-by-round into the consistency checks, // never accumulating into the running claim.) // Read + bind the final â, b̂ claims off the stream (mirrors - // `prove_packed_padded`): binding must land before the next challenge + // `prove_packed_padded_inner`): binding must land before the next challenge // (lincheck's α) is drawn, so the α-batched reduction of these two claims is // sound. `final_c_eval` is the verifier's OWN interpolation of the // already-bound `round1_c` at `z` — never transported. - let r_rest: Vec = r[k_skip..].to_vec(); + let r_rest: Vec = r[k_skip..].to_vec(); let final_a_eval = vs.next_scalar().map_err(VerifyError::Transcript)?; let final_b_eval = vs.next_scalar().map_err(VerifyError::Transcript)?; if c_running != final_a_eval * final_b_eval { @@ -491,14 +500,8 @@ mod tests { m: usize, ps: &mut pcs::ProverState, ) -> ZerocheckClaim { - let (claim, _) = prove_packed_padded( - a_packed, - b_packed, - c_packed, - m, - &PaddingSpec::dense(m), - ps, - ); + let (claim, _) = + prove_packed_padded_capture_s_hat_v_c(a_packed, b_packed, c_packed, m, &PaddingSpec::dense(m), ps); claim } @@ -535,7 +538,8 @@ mod tests { assert_eq!(stream.len(), 2 * (1 << K_SKIP) + 2 * (m - K_SKIP) + 2, "m={m}"); assert_eq!(claim.mlv_challenges.len(), m - K_SKIP, "m={m}"); - // Claim's eval fields agree with the streamed final evals. + // Claim's eval fields agree with the streamed final evals (both are + // now tower values — the prover streams eval). assert_eq!(claim.a_eval, stream[stream.len() - 2], "m={m}"); assert_eq!(claim.b_eval, stream[stream.len() - 1], "m={m}"); } @@ -566,7 +570,7 @@ mod tests { } /// **Verify rejects byte-mutated proofs.** Walk each component of the - /// proof and flip one F128 entry; the verifier must return an `Err` + /// proof and flip one F192 entry; the verifier must return an `Err` /// (rather than panicking or silently accepting). #[test] fn verify_rejects_mutations() { @@ -595,7 +599,7 @@ mod tests { ]; for (label, word) in mutations { let mut bad = proof_t.clone(); - bad.stream[word].lo ^= 1; + bad.stream[word].c0 ^= 1; let mut ch = pcs::VerifierState::new(b"flock-test-v0", &bad, &[]); let result = verify(m, &mut ch); assert!( @@ -622,10 +626,7 @@ mod tests { let mut bad = proof_t.clone(); bad.stream.truncate(bad.stream.len() - 3); let mut ch = pcs::VerifierState::new(b"flock-test-v0", &bad, &[]); - assert!(matches!( - verify(m, &mut ch), - Err(VerifyError::Transcript(_)) - )); + assert!(matches!(verify(m, &mut ch), Err(VerifyError::Transcript(_)))); // log_n too small. let mut ch = pcs::VerifierState::new(b"flock-test-v0", &proof_t, &[]); @@ -655,10 +656,7 @@ mod tests { let mut ch_verify = pcs::VerifierState::new(b"flock-test-v0", &proof_t, &[]); let res = verify(m, &mut ch_verify); - assert!( - res.is_err(), - "verify ACCEPTED a false statement at m={m}: {res:?}" - ); + assert!(res.is_err(), "verify ACCEPTED a false statement at m={m}: {res:?}"); } } @@ -683,7 +681,7 @@ mod tests { // verifier should reject (overwhelming probability). for idx in 0..(m - K_SKIP) { let mut bad = proof_t.clone(); - bad.stream[2 * (1 << K_SKIP) + 2 * idx + 1] += F128::ONE; + bad.stream[2 * (1 << K_SKIP) + 2 * idx + 1] += F192::ONE; let mut ch = pcs::VerifierState::new(b"flock-test-v0", &bad, &[]); let res = verify(m, &mut ch); assert!(res.is_err(), "msg_inf tamper at round {idx} ACCEPTED"); @@ -708,12 +706,9 @@ mod tests { let last = m - K_SKIP - 1; let mut bad = proof_t.clone(); - bad.stream[2 * (1 << K_SKIP) + 2 * last + 1] += F128::ONE; + bad.stream[2 * (1 << K_SKIP) + 2 * last + 1] += F192::ONE; let mut ch = pcs::VerifierState::new(b"flock-test-v0", &bad, &[]); - assert!( - verify(m, &mut ch).is_err(), - "last-round msg_inf unconstrained" - ); + assert!(verify(m, &mut ch).is_err(), "last-round msg_inf unconstrained"); } /// AUDIT (Fiat–Shamir binding of the final â, b̂ claims). Regression test @@ -749,19 +744,14 @@ mod tests { // Honest verify, then capture the next challenge the transcript feeds // downstream — this is exactly the slot lincheck samples α from. let mut ch_honest = pcs::VerifierState::new(b"flock-test-v0", &proof_t, &[]); - assert!( - verify(m, &mut ch_honest).is_ok(), - "honest verify rejected" - ); + assert!(verify(m, &mut ch_honest).is_ok(), "honest verify rejected"); let alpha_honest = ch_honest.sample(); // Product-preserving tamper: â' = â·t, b̂' = b̂·t⁻¹ ⇒ â'·b̂' = â·b̂, so the // zerocheck's `c_running == â·b̂` check still holds for the tampered pair. - let t = F128 { - lo: 0x0123_4567_89ab_cdef, - hi: 0xfedc_ba98_7654_3210, - }; - assert!(t != F128::ZERO && t != F128::ONE, "t must be nontrivial"); + // The stream now carries tower (F192) values, so tamper in F192. + let t = F192::new(0x0123_4567_89ab_cdef, 0xfedc_ba98_7654_3210, 0x55aa_aa55_0123_4567); + assert!(t != F192::ZERO && t != F192::ONE, "t must be nontrivial"); // The finals are the LAST two stream words of this standalone proof. let n = proof_t.stream.len(); let mut bad = proof_t.clone(); @@ -819,10 +809,7 @@ mod tests { let proof_t = ch_prove.into_proof(); let mut ch_verify = pcs::VerifierState::new(b"flock-test-v0", &proof_t, &[]); let res = verify(m, &mut ch_verify); - assert!( - res.is_err(), - "false statement (seed={seed}) ACCEPTED: {res:?}" - ); + assert!(res.is_err(), "false statement (seed={seed}) ACCEPTED: {res:?}"); } } @@ -840,12 +827,9 @@ mod tests { let proof_t = ch_prove.into_proof(); for idx in 0..(m - K_SKIP) { let mut bad = proof_t.clone(); - bad.stream[2 * (1 << K_SKIP) + 2 * idx] += F128::ONE; + bad.stream[2 * (1 << K_SKIP) + 2 * idx] += F192::ONE; let mut ch = pcs::VerifierState::new(b"flock-test-v0", &bad, &[]); - assert!( - verify(m, &mut ch).is_err(), - "msg_1 tamper round {idx} ACCEPTED" - ); + assert!(verify(m, &mut ch).is_err(), "msg_1 tamper round {idx} ACCEPTED"); } } diff --git a/crates/flock/src/zerocheck/multilinear.rs b/crates/flock/src/zerocheck/multilinear.rs index b38576a0..478e71f6 100644 --- a/crates/flock/src/zerocheck/multilinear.rs +++ b/crates/flock/src/zerocheck/multilinear.rs @@ -1,4 +1,4 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. //! Multilinear sumcheck — rounds 2..(m − k_skip + 1) of the zerocheck protocol. //! //! After the round-1 URM and the verifier's univariate-skip fold-point `z`, the @@ -33,10 +33,10 @@ //! Verifier reconstructs `G(0)` from the running claim via //! `current_claim = (1+r_now)·G(0) + r_now·G(1)`. -use primitives::field::{F128, F256Unreduced, PHI_8_TABLE}; -pub use primitives::multilinear::eq_eval; use crate::zerocheck::PaddingSpec; -use crate::zerocheck::univariate_skip::{SplitEqGhash, build_eq, pack_bits}; +use crate::zerocheck::univariate_skip::{SplitEq, build_eq, pack_bits}; +use primitives::field::{F192, F192Unreduced, PHI_8_TABLE_192 as PHI_8_TABLE}; +pub use primitives::multilinear::eq_eval; /// Returns `(pair_in_block_mask, useful_pairs_inclusive)` for the round-2 /// fused-fold kernel. A pair (post-URM chunks `2k`, `2k+1`) is fully inside @@ -70,7 +70,7 @@ fn round2_pair_skip(padding: &PaddingSpec, k_skip: usize) -> (usize, usize) { /// /// `L_i(z) = ∏_{j ≠ i} (z + φ_8(j)) / (φ_8(i) + φ_8(j))` — the standard Lagrange /// formula, with the nodes being the F_8 elements `0..2^k_skip` embedded into -/// F_{2^128} via `φ_8`. Subtraction is XOR in characteristic 2. +/// F_{2^192} via `φ_8`. Subtraction is XOR in characteristic 2. /// pub use primitives::multilinear::lagrange_weights_naive; @@ -80,14 +80,14 @@ pub use primitives::multilinear::lagrange_weights_naive; /// /// Used to interpolate the extract_c round-1 output `round1_c` (which carries /// the polynomial `P^C` as its 2^k_skip evaluations on Λ) at the URM challenge `z`. -pub fn lagrange_weights_lambda_naive(k_skip: usize, z: F128) -> Vec { +pub fn lagrange_weights_lambda_naive(k_skip: usize, z: F192) -> Vec { let ell = 1usize << k_skip; assert!(2 * ell <= 256, "Λ ∪ S must fit in F_8 (need k_skip ≤ 7)"); - let mut weights = vec![F128::ZERO; ell]; + let mut weights = vec![F192::ZERO; ell]; for i in 0..ell { let si = PHI_8_TABLE[ell + i]; - let mut num = F128::ONE; - let mut den = F128::ONE; + let mut num = F192::ONE; + let mut den = F192::ONE; for j in 0..ell { if j == i { continue; @@ -107,11 +107,11 @@ pub fn lagrange_weights_lambda_naive(k_skip: usize, z: F128) -> Vec { /// In the extract_c protocol the prover ships `round1_c` (the `P^C` polynomial /// in Λ-form) and the verifier (or higher-level prover) needs `P^C(z) = ĉ(z, r_rest)`. /// That value is *the c-claim* at the bound point `(z, r_rest)`. -pub fn interpolate_at_z_on_lambda(values: &[F128], k_skip: usize, z: F128) -> F128 { +pub fn interpolate_at_z_on_lambda(values: &[F192], k_skip: usize, z: F192) -> F192 { let ell = 1usize << k_skip; assert_eq!(values.len(), ell); let weights = lagrange_weights_lambda_naive(k_skip, z); - let mut acc = F128::ZERO; + let mut acc = F192::ZERO; for i in 0..ell { acc += weights[i] * values[i]; } @@ -127,21 +127,21 @@ pub fn interpolate_at_z_on_lambda(values: &[F128], k_skip: usize, z: F128) -> F1 /// evaluations on Λ that the prover sends, that's `2·2^k_skip` evaluations — /// enough to interpolate the degree-`< 2·2^k_skip` polynomial uniquely. /// -/// Cost: `2·ell × (2·ell − 1)` F128 muls + `ell` inversions for the Lagrange +/// Cost: `2·ell × (2·ell − 1)` F192 muls + `ell` inversions for the Lagrange /// weights. At ell=64 that's ~16K muls + 64 inversions. Sub-millisecond /// one-time cost in the verifier. -pub fn interpolate_at_z_combined(values_on_lambda: &[F128], k_skip: usize, z: F128) -> F128 { +pub fn interpolate_at_z_combined(values_on_lambda: &[F192], k_skip: usize, z: F192) -> F192 { let ell = 1usize << k_skip; assert_eq!(values_on_lambda.len(), ell); assert!(2 * ell <= 256, "Λ ∪ S must fit in F_8 (need k_skip ≤ 7)"); let n_total = 2 * ell; - let mut acc = F128::ZERO; + let mut acc = F192::ZERO; for i in 0..ell { // i-th Λ node = node index `ell + i` in PHI_8_TABLE. let node_idx = ell + i; let si = PHI_8_TABLE[node_idx]; - let mut num = F128::ONE; - let mut den = F128::ONE; + let mut num = F192::ONE; + let mut den = F192::ONE; for j in 0..n_total { if j == node_idx { continue; @@ -162,23 +162,23 @@ pub fn interpolate_at_z_combined(values_on_lambda: &[F128], k_skip: usize, z: F1 /// Evaluate the univariate-skip polynomial at the fold point `z`, given the /// precomputed Lagrange `weights`. Returns the multilinear extension table -/// `a_mlv` of length `2^(m − k_skip)` over F_{2^128}. +/// `a_mlv` of length `2^(m − k_skip)` over F_{2^192}. /// /// `a_mlv[x_rest] = Σ_s a(s, x_rest) · L_s(z)` /// /// `a(s, x_rest)` is the witness bit at index `x_rest * 2^k_skip + s` (low /// bits = skip variable, high bits = rest variables). -pub fn fold_at_z_naive(witness: &[bool], m: usize, k_skip: usize, weights: &[F128]) -> Vec { +pub fn fold_at_z_naive(witness: &[bool], m: usize, k_skip: usize, weights: &[F192]) -> Vec { assert!(k_skip <= m); let ell = 1usize << k_skip; let n_rest = 1usize << (m - k_skip); assert_eq!(witness.len(), 1usize << m); assert_eq!(weights.len(), ell); - let mut folded = vec![F128::ZERO; n_rest]; + let mut folded = vec![F192::ZERO; n_rest]; for x_rest in 0..n_rest { let base = x_rest * ell; - let mut acc = F128::ZERO; + let mut acc = F192::ZERO; for s in 0..ell { if witness[base + s] { acc += weights[s]; @@ -197,7 +197,7 @@ pub fn fold_at_z_naive(witness: &[bool], m: usize, k_skip: usize, weights: &[F12 /// multilinear sumcheck. /// /// Inputs: -/// - `a_mlv`, `b_mlv`: F128 vectors of length `2^n` for some `n ≥ 1`. +/// - `a_mlv`, `b_mlv`: F192 vectors of length `2^n` for some `n ≥ 1`. /// - `r`: full eq challenges, length `n`. `r[0]` is the challenge for the /// variable being bound *this* round; `r[1..]` is for the remaining `n − 1` /// variables. @@ -209,7 +209,7 @@ pub fn fold_at_z_naive(witness: &[bool], m: usize, k_skip: usize, weights: &[F12 /// The `r[0]` prefactor matches the C++ `sumcheck_round_pair` convention: the /// quantity sent on the wire is `Π(1) = eq(r[0], 1) · G(1) = r[0] · G(1)`, /// where `Π(X) = eq(r[0], X) · G(X)` is the actual round polynomial. -pub fn round_pair_naive(a_mlv: &[F128], b_mlv: &[F128], r: &[F128]) -> (F128, F128) { +pub fn round_pair_naive(a_mlv: &[F192], b_mlv: &[F192], r: &[F192]) -> (F192, F192) { let n = a_mlv.len(); assert_eq!(b_mlv.len(), n); assert!(n.is_power_of_two() && n >= 2); @@ -220,8 +220,8 @@ pub fn round_pair_naive(a_mlv: &[F128], b_mlv: &[F128], r: &[F128]) -> (F128, F1 let eq_remaining = build_eq(&r[1..]); assert_eq!(eq_remaining.len(), half); - let mut g_one = F128::ZERO; - let mut g_inf = F128::ZERO; + let mut g_one = F192::ZERO; + let mut g_inf = F192::ZERO; for x_prime in 0..half { let a0 = a_mlv[2 * x_prime]; let a1 = a_mlv[2 * x_prime + 1]; @@ -257,15 +257,12 @@ pub fn uni_skip_fold_and_round_pair_naive( b: &[bool], m: usize, k_skip: usize, - z: F128, - mlv_challenges: &[F128], -) -> (Vec, Vec, F128, F128) { + z: F192, + mlv_challenges: &[F192], +) -> (Vec, Vec, F192, F192) { assert_eq!(a.len(), 1usize << m); assert_eq!(b.len(), 1usize << m); - assert!( - m > k_skip, - "need at least one multilinear variable past the skip" - ); + assert!(m > k_skip, "need at least one multilinear variable past the skip"); assert_eq!(mlv_challenges.len(), m - k_skip); let weights = lagrange_weights_naive(k_skip, z); @@ -281,7 +278,7 @@ pub fn uni_skip_fold_and_round_pair_naive( /// Precomputed fold table for the univariate-skip fold at a fixed `z`. /// -/// Storage: `n_chunks × 256` F128 entries (32 KB at `k_skip=6`). For each +/// Storage: `n_chunks × 256` F192 entries (32 KB at `k_skip=6`). For each /// byte-chunk `j ∈ 0..n_chunks` and byte value `v ∈ 0..256`: /// /// `data[j * 256 + v] = Σ_{b : bit b of v set} weights[8j + b]` @@ -294,17 +291,17 @@ pub fn uni_skip_fold_and_round_pair_naive( #[derive(Clone, Debug)] pub struct UniSkipFoldTable { pub n_chunks: usize, - pub data: Vec, + pub data: Vec, } impl UniSkipFoldTable { - pub fn new(k_skip: usize, z: F128) -> Self { + pub fn new(k_skip: usize, z: F192) -> Self { let ell = 1usize << k_skip; assert_eq!(ell % 8, 0, "k_skip must be ≥ 3 (need ell divisible by 8)"); let n_chunks = ell / 8; let weights = lagrange_weights_naive(k_skip, z); - let mut data = vec![F128::ZERO; n_chunks * 256]; + let mut data = vec![F192::ZERO; n_chunks * 256]; for j in 0..n_chunks { let basis = &weights[8 * j..8 * j + 8]; // v = 0: zero (already initialized). @@ -325,11 +322,11 @@ impl UniSkipFoldTable { } /// Scalar one-row fold: `Σ_j table[j][bytes[j]]`. Ports the NEON - /// `uni_skip_fold_one_output_ghash` in scalar form. + /// the scalar univariate-skip fold. #[inline] - pub fn fold_one_row(&self, bytes: &[u8]) -> F128 { + pub fn fold_one_row(&self, bytes: &[u8]) -> F192 { assert_eq!(bytes.len(), self.n_chunks); - let mut acc = F128::ZERO; + let mut acc = F192::ZERO; for j in 0..self.n_chunks { acc += self.data[j * 256 + bytes[j] as usize]; } @@ -337,54 +334,30 @@ impl UniSkipFoldTable { } } -/// NEON one-row fold: 8 aligned 16-byte loads + 8 XORs, hand-unrolled for -/// `n_chunks = 8` (the k_skip=6 protocol size). Returns the folded F128. -/// -/// The table is `Vec` with each entry 16-byte aligned (F128 is -/// `repr(C, align(16))`), so every `vld1q_u8` lands on an aligned address. +/// NEON one-row fold, hand-unrolled for `n_chunks = 8` (the k_skip=6 protocol +/// size). Each table entry is 24 bytes: NEON folds c0/c1 together while c2 is +/// folded in a scalar register. /// /// # Safety -/// Caller must guarantee `table_data` points to ≥ 8 × 256 × 16 valid bytes +/// Caller must guarantee `table_data` points to ≥ 8 × 256 valid F192 entries /// (an `n_chunks ≥ 8` table) and `bytes_ptr` to ≥ 8 valid bytes. #[cfg(target_arch = "aarch64")] #[inline(always)] -unsafe fn fold_one_row_neon_unchecked_8(table_data: *const u8, bytes_ptr: *const u8) -> F128 { +unsafe fn fold_one_row_neon_unchecked_8(table_data: *const F192, bytes_ptr: *const u8) -> F192 { use core::arch::aarch64::*; unsafe { - const STRIDE: usize = 256 * 16; - let mut acc = vld1q_u8(table_data.add((*bytes_ptr) as usize * 16)); - acc = veorq_u8( - acc, - vld1q_u8(table_data.add(1 * STRIDE + (*bytes_ptr.add(1)) as usize * 16)), - ); - acc = veorq_u8( - acc, - vld1q_u8(table_data.add(2 * STRIDE + (*bytes_ptr.add(2)) as usize * 16)), - ); - acc = veorq_u8( - acc, - vld1q_u8(table_data.add(3 * STRIDE + (*bytes_ptr.add(3)) as usize * 16)), - ); - acc = veorq_u8( - acc, - vld1q_u8(table_data.add(4 * STRIDE + (*bytes_ptr.add(4)) as usize * 16)), - ); - acc = veorq_u8( - acc, - vld1q_u8(table_data.add(5 * STRIDE + (*bytes_ptr.add(5)) as usize * 16)), - ); - acc = veorq_u8( - acc, - vld1q_u8(table_data.add(6 * STRIDE + (*bytes_ptr.add(6)) as usize * 16)), - ); - acc = veorq_u8( - acc, - vld1q_u8(table_data.add(7 * STRIDE + (*bytes_ptr.add(7)) as usize * 16)), - ); - let acc_u64 = vreinterpretq_u64_u8(acc); - F128 { - lo: vgetq_lane_u64::<0>(acc_u64), - hi: vgetq_lane_u64::<1>(acc_u64), + let first = &*table_data.add((*bytes_ptr) as usize); + let mut acc = vld1q_u64(&first.c0); + let mut c2 = first.c2; + for chunk in 1..8 { + let entry = &*table_data.add(chunk * 256 + (*bytes_ptr.add(chunk)) as usize); + acc = veorq_u64(acc, vld1q_u64(&entry.c0)); + c2 ^= entry.c2; + } + F192 { + c0: vgetq_lane_u64::<0>(acc), + c1: vgetq_lane_u64::<1>(acc), + c2, } } } @@ -394,7 +367,7 @@ unsafe fn fold_one_row_neon_unchecked_8(table_data: *const u8, bytes_ptr: *const /// by default** via rayon — the outer x_hi loop is distributed across workers, /// each writing to a disjoint chunk of `a_folded`/`b_folded` via `par_chunks_mut` /// and accumulating its own `(sum1_contrib, sum_inf_contrib)`. The final -/// reduce sums the per-worker contributions (commutative + associative F128 +/// reduce sums the per-worker contributions (commutative + associative F192 /// XOR/multiply). /// /// Algorithm (per worker, one x_hi): @@ -416,8 +389,8 @@ pub fn uni_skip_fold_and_round_pair_optimized_packed( m: usize, k_skip: usize, table: &UniSkipFoldTable, - mlv_challenges: &[F128], -) -> (Vec, Vec, F128, F128) { + mlv_challenges: &[F192], +) -> (Vec, Vec, F192, F192) { uni_skip_fold_and_round_pair_optimized_packed_padded( a_packed, b_packed, @@ -439,15 +412,12 @@ pub fn uni_skip_fold_and_round_pair_optimized_packed_padded( m: usize, k_skip: usize, table: &UniSkipFoldTable, - mlv_challenges: &[F128], + mlv_challenges: &[F192], padding: &PaddingSpec, -) -> (Vec, Vec, F128, F128) { +) -> (Vec, Vec, F192, F192) { use rayon::prelude::*; - assert_eq!( - k_skip, 6, - "optimized fold-and-round_pair variant is k_skip=6 only" - ); + assert_eq!(k_skip, 6, "optimized fold-and-round_pair variant is k_skip=6 only"); assert_eq!(table.n_chunks, 8); let n_chunks = table.n_chunks; let n_out = 1usize << (m - k_skip); @@ -455,14 +425,12 @@ pub fn uni_skip_fold_and_round_pair_optimized_packed_padded( assert_eq!(b_packed.len(), n_out * n_chunks); assert_eq!(mlv_challenges.len(), m - k_skip); - // Uninit alloc — the parallel loop below writes every slot (dense path) - // or explicitly writes F128::ZERO at padding holes (padded path). - // Saves ~22 ms of sequential zero-fill at m=29 (256 MB total) that would - // otherwise cap the parallel speedup of this phase at ~2.5× on 8 cores. - let mut a_folded: Vec = primitives::scratch::take_f128(n_out); - let mut b_folded: Vec = primitives::scratch::take_f128(n_out); + // The parallel loop overwrites every pooled slot (including padding holes), + // avoiding a separate sequential clear of 256 MB at m=29. + let mut a_folded: Vec = primitives::scratch::take_f192(n_out); + let mut b_folded: Vec = primitives::scratch::take_f192(n_out); - let eq = SplitEqGhash::new(&mlv_challenges[1..]); + let eq = SplitEq::new(&mlv_challenges[1..]); let lo_size = 1usize << eq.n_lo; let hi_size = 1usize << eq.n_hi; assert_eq!(lo_size * hi_size * 2, n_out); @@ -473,19 +441,19 @@ pub fn uni_skip_fold_and_round_pair_optimized_packed_padded( let (pair_in_block_mask, useful_pairs_inclusive) = round2_pair_skip(padding, k_skip); // Parallel: each worker writes one disjoint chunk of a_folded/b_folded - // and returns its (sum1, sum_inf) contribution. Reduce by F128 XOR. + // and returns its (sum1, sum_inf) contribution. Reduce by F192 XOR. let (sum1, sum_inf) = a_folded .par_chunks_mut(chunk_size) .zip(b_folded.par_chunks_mut(chunk_size)) .enumerate() .map(|(x_hi, (a_chunk, b_chunk))| { - let mut p1_acc = F256Unreduced::ZERO; - let mut pinf_acc = F256Unreduced::ZERO; + let mut p1_acc = F192Unreduced::ZERO; + let mut pinf_acc = F192Unreduced::ZERO; let pair_idx_base = x_hi * lo_size; #[cfg(target_arch = "aarch64")] unsafe { - let table_ptr = table.data.as_ptr() as *const u8; + let table_ptr = table.data.as_ptr(); let a_pkt_ptr = a_packed.as_ptr(); let b_pkt_ptr = b_packed.as_ptr(); let base = x_hi * chunk_size; @@ -496,10 +464,10 @@ pub fn uni_skip_fold_and_round_pair_optimized_packed_padded( if ((pair_idx_base + x_lo) & pair_in_block_mask) >= useful_pairs_inclusive { // Padding hole: write zero (a_folded/b_folded were alloc'd // uninit, so we have to write every slot we don't fold into). - a_chunk[x0l] = F128::ZERO; - a_chunk[x1l] = F128::ZERO; - b_chunk[x0l] = F128::ZERO; - b_chunk[x1l] = F128::ZERO; + a_chunk[x0l] = F192::ZERO; + a_chunk[x1l] = F192::ZERO; + b_chunk[x0l] = F192::ZERO; + b_chunk[x1l] = F192::ZERO; continue; } let x0g = base + 2 * x_lo; @@ -530,10 +498,10 @@ pub fn uni_skip_fold_and_round_pair_optimized_packed_padded( let x1l = x0l + 1; if ((pair_idx_base + x_lo) & pair_in_block_mask) >= useful_pairs_inclusive { // See aarch64 branch above for why this zero write is needed. - a_chunk[x0l] = F128::ZERO; - a_chunk[x1l] = F128::ZERO; - b_chunk[x0l] = F128::ZERO; - b_chunk[x1l] = F128::ZERO; + a_chunk[x0l] = F192::ZERO; + a_chunk[x1l] = F192::ZERO; + b_chunk[x0l] = F192::ZERO; + b_chunk[x1l] = F192::ZERO; continue; } let x0g = base + 2 * x_lo; @@ -560,7 +528,7 @@ pub fn uni_skip_fold_and_round_pair_optimized_packed_padded( (eq_h * p1, eq_h * pinf) }) .reduce( - || (F128::ZERO, F128::ZERO), + || (F192::ZERO, F192::ZERO), |(s1, sinf), (c1, cinf)| (s1 + c1, sinf + cinf), ); @@ -574,7 +542,7 @@ pub fn uni_skip_fold_and_round_pair_optimized_packed_padded( /// In-place fold of a single multilinear polynomial table at `challenge`. /// Pairs `(a[2x], a[2x+1])` collapse to `a[x] = a[2x] + challenge · (a[2x+1] + a[2x])`. /// After the call, `a.len()` is halved. -pub fn fold_in_place_single(a: &mut Vec, challenge: F128) { +pub fn fold_in_place_single(a: &mut Vec, challenge: F192) { let n = a.len(); assert!(n.is_power_of_two() && n >= 2); let half = n / 2; @@ -593,7 +561,7 @@ pub fn fold_in_place_single(a: &mut Vec, challenge: F128) { /// /// Used at the tail of the multilinear-round sequence where the polynomial is /// small enough that parallel/fusion overhead outweighs benefit. -pub fn fold_in_place_pair(a: &mut Vec, b: &mut Vec, challenge: F128) { +pub fn fold_in_place_pair(a: &mut Vec, b: &mut Vec, challenge: F192) { let n = a.len(); assert_eq!(b.len(), n); assert!(n.is_power_of_two() && n >= 2); @@ -621,19 +589,40 @@ pub fn fold_in_place_pair(a: &mut Vec, b: &mut Vec, challenge: F128) /// one bit of x_lo (lo_size ≥ 2). Smaller polynomials should use the /// unfused `fold_in_place_pair + round_pair_naive` pair. pub fn fold_and_compute_round_pair_optimized( - a: &[F128], - b: &[F128], - r_fold: F128, - r_next: &[F128], -) -> (Vec, Vec, F128, F128) { + a: &[F192], + b: &[F192], + r_fold: F192, + r_next: &[F192], +) -> (Vec, Vec, F192, F192) { let half = a.len() / 2; - // Uninit alloc — `_into` writes every slot of a_new/b_new. - let mut a_new = primitives::alloc_uninit_vec::(half); - let mut b_new = primitives::alloc_uninit_vec::(half); - let (m1, mi) = fold_and_compute_round_pair_into(a, b, &mut a_new, &mut b_new, r_fold, r_next); + let mut a_new = primitives::alloc_uninit(half); + let mut b_new = primitives::alloc_uninit(half); + let (m1, mi) = fold_and_compute_round_pair_into_slots(a, b, &mut a_new, &mut b_new, r_fold, r_next); + // SAFETY: the fold writes every slot of both output vectors exactly once. + let a_new = unsafe { primitives::assume_init(a_new) }; + // SAFETY: the fold writes every slot of both output vectors exactly once. + let b_new = unsafe { primitives::assume_init(b_new) }; (a_new, b_new, m1, mi) } +trait OutputSlot { + fn put(&mut self, value: F192); +} + +impl OutputSlot for F192 { + #[inline(always)] + fn put(&mut self, value: F192) { + *self = value; + } +} + +impl OutputSlot for std::mem::MaybeUninit { + #[inline(always)] + fn put(&mut self, value: F192) { + self.write(value); + } +} + /// Buffer-reusing variant of [`fold_and_compute_round_pair_optimized`]: writes /// the folded `a`/`b` into the caller-provided `a_out`/`b_out` (each length /// `a.len() / 2`) instead of allocating. Returns `(r_next[0] · G(1), G(∞))`. @@ -643,13 +632,24 @@ pub fn fold_and_compute_round_pair_optimized( /// than per round. The per-round `munmap` of the old buffer (64 MB at m=29) /// runs single-threaded and otherwise caps the tail's parallel speedup. pub fn fold_and_compute_round_pair_into( - a: &[F128], - b: &[F128], - a_out: &mut [F128], - b_out: &mut [F128], - r_fold: F128, - r_next: &[F128], -) -> (F128, F128) { + a: &[F192], + b: &[F192], + a_out: &mut [F192], + b_out: &mut [F192], + r_fold: F192, + r_next: &[F192], +) -> (F192, F192) { + fold_and_compute_round_pair_into_slots(a, b, a_out, b_out, r_fold, r_next) +} + +fn fold_and_compute_round_pair_into_slots( + a: &[F192], + b: &[F192], + a_out: &mut [O], + b_out: &mut [O], + r_fold: F192, + r_next: &[F192], +) -> (F192, F192) { use rayon::prelude::*; let n = a.len(); @@ -661,7 +661,7 @@ pub fn fold_and_compute_round_pair_into( let log_n = n.trailing_zeros() as usize; assert_eq!(r_next.len(), log_n - 1); - let eq = SplitEqGhash::new(&r_next[1..]); + let eq = SplitEq::new(&r_next[1..]); let lo_size = 1usize << eq.n_lo; let hi_size = 1usize << eq.n_hi; assert!(lo_size >= 2, "fold_and_compute requires lo_size ≥ 2"); @@ -681,8 +681,8 @@ pub fn fold_and_compute_round_pair_into( let a_in = &a[x_hi * chunk_in..(x_hi + 1) * chunk_in]; let b_in = &b[x_hi * chunk_in..(x_hi + 1) * chunk_in]; - let mut p1_acc = F256Unreduced::ZERO; - let mut pinf_acc = F256Unreduced::ZERO; + let mut p1_acc = F192Unreduced::ZERO; + let mut pinf_acc = F192Unreduced::ZERO; // Unroll 4 x_lo's per iteration when lo_size % 4 == 0 (the common // case for the fused path; falls back to 2-wide for lo_size==2 at @@ -757,22 +757,22 @@ pub fn fold_and_compute_round_pair_into( let oi_b = 2 * x_lo_b; let oi_c = 2 * x_lo_c; let oi_d = 2 * x_lo_d; - a_out[oi_a] = a0_a; - a_out[oi_a + 1] = a1_a; - b_out[oi_a] = b0_a; - b_out[oi_a + 1] = b1_a; - a_out[oi_b] = a0_b; - a_out[oi_b + 1] = a1_b; - b_out[oi_b] = b0_b; - b_out[oi_b + 1] = b1_b; - a_out[oi_c] = a0_c; - a_out[oi_c + 1] = a1_c; - b_out[oi_c] = b0_c; - b_out[oi_c + 1] = b1_c; - a_out[oi_d] = a0_d; - a_out[oi_d + 1] = a1_d; - b_out[oi_d] = b0_d; - b_out[oi_d + 1] = b1_d; + a_out[oi_a].put(a0_a); + a_out[oi_a + 1].put(a1_a); + b_out[oi_a].put(b0_a); + b_out[oi_a + 1].put(b1_a); + a_out[oi_b].put(a0_b); + a_out[oi_b + 1].put(a1_b); + b_out[oi_b].put(b0_b); + b_out[oi_b + 1].put(b1_b); + a_out[oi_c].put(a0_c); + a_out[oi_c + 1].put(a1_c); + b_out[oi_c].put(b0_c); + b_out[oi_c + 1].put(b1_c); + a_out[oi_d].put(a0_d); + a_out[oi_d + 1].put(a1_d); + b_out[oi_d].put(b0_d); + b_out[oi_d + 1].put(b1_d); // 8 independent msg muls. let eq_l_a = eq_lo[x_lo_a]; @@ -835,14 +835,14 @@ pub fn fold_and_compute_round_pair_into( let oi_a = 2 * x_lo_a; let oi_b = 2 * x_lo_b; - a_out[oi_a] = a0_a; - a_out[oi_a + 1] = a1_a; - b_out[oi_a] = b0_a; - b_out[oi_a + 1] = b1_a; - a_out[oi_b] = a0_b; - a_out[oi_b + 1] = a1_b; - b_out[oi_b] = b0_b; - b_out[oi_b + 1] = b1_b; + a_out[oi_a].put(a0_a); + a_out[oi_a + 1].put(a1_a); + b_out[oi_a].put(b0_a); + b_out[oi_a + 1].put(b1_a); + a_out[oi_b].put(a0_b); + a_out[oi_b + 1].put(a1_b); + b_out[oi_b].put(b0_b); + b_out[oi_b + 1].put(b1_b); let eq_l_a = eq_lo[x_lo_a]; let eq_l_b = eq_lo[x_lo_b]; @@ -864,7 +864,7 @@ pub fn fold_and_compute_round_pair_into( (eq_h * p1, eq_h * pinf) }) .reduce( - || (F128::ZERO, F128::ZERO), + || (F192::ZERO, F192::ZERO), |(s1, sinf), (c1, cinf)| (s1 + c1, sinf + cinf), ); @@ -881,22 +881,22 @@ fn uni_skip_fold_and_round_pair_optimized_packed_serial( m: usize, k_skip: usize, table: &UniSkipFoldTable, - mlv_challenges: &[F128], -) -> (Vec, Vec, F128, F128) { + mlv_challenges: &[F192], +) -> (Vec, Vec, F192, F192) { assert_eq!(k_skip, 6); assert_eq!(table.n_chunks, 8); let n_chunks = table.n_chunks; let n_out = 1usize << (m - k_skip); - let mut a_folded = vec![F128::ZERO; n_out]; - let mut b_folded = vec![F128::ZERO; n_out]; - let eq = SplitEqGhash::new(&mlv_challenges[1..]); + let mut a_folded = vec![F192::ZERO; n_out]; + let mut b_folded = vec![F192::ZERO; n_out]; + let eq = SplitEq::new(&mlv_challenges[1..]); let lo_size = 1usize << eq.n_lo; let hi_size = 1usize << eq.n_hi; - let mut sum1 = F128::ZERO; - let mut sum_inf = F128::ZERO; + let mut sum1 = F192::ZERO; + let mut sum_inf = F192::ZERO; for x_hi in 0..hi_size { - let mut p1_acc = F256Unreduced::ZERO; - let mut pinf_acc = F256Unreduced::ZERO; + let mut p1_acc = F192Unreduced::ZERO; + let mut pinf_acc = F192Unreduced::ZERO; let k_base = x_hi << eq.n_lo; for x_lo in 0..lo_size { let k = k_base | x_lo; @@ -932,22 +932,15 @@ pub fn uni_skip_fold_and_round_pair_optimized( b: &[bool], m: usize, k_skip: usize, - z: F128, - mlv_challenges: &[F128], -) -> (Vec, Vec, F128, F128) { + z: F192, + mlv_challenges: &[F192], +) -> (Vec, Vec, F192, F192) { assert_eq!(a.len(), 1usize << m); assert_eq!(b.len(), 1usize << m); let a_packed = pack_bits(a); let b_packed = pack_bits(b); let table = UniSkipFoldTable::new(k_skip, z); - uni_skip_fold_and_round_pair_optimized_packed( - &a_packed, - &b_packed, - m, - k_skip, - &table, - mlv_challenges, - ) + uni_skip_fold_and_round_pair_optimized_packed(&a_packed, &b_packed, m, k_skip, &table, mlv_challenges) } // --------------------------------------------------------------------------- @@ -970,10 +963,10 @@ mod tests { let mut rng = Rng::new(1); for &k_skip in &[1usize, 2, 3, 4, 5, 6] { for _ in 0..4 { - let z = rng.f128(); + let z = rng.ext(); let weights = lagrange_weights_naive(k_skip, z); - let sum: F128 = weights.iter().copied().fold(F128::ZERO, |a, b| a + b); - assert_eq!(sum, F128::ONE, "Σ L_i ≠ 1 at k_skip={k_skip}"); + let sum: F192 = weights.iter().copied().fold(F192::ZERO, |a, b| a + b); + assert_eq!(sum, F192::ONE, "Σ L_i ≠ 1 at k_skip={k_skip}"); } } } @@ -987,7 +980,7 @@ mod tests { let z = PHI_8_TABLE[i]; let weights = lagrange_weights_naive(k_skip, z); for j in 0..ell { - let expected = if j == i { F128::ONE } else { F128::ZERO }; + let expected = if j == i { F192::ONE } else { F192::ZERO }; assert_eq!(weights[j], expected, "k_skip={k_skip}, z=node{i}, j={j}"); } } @@ -999,7 +992,7 @@ mod tests { // ---------------------------------------------------------------------- /// At a node `z = φ_8(i)`, fold reduces to the witness restricted to s=i: - /// `a_mlv[x_rest] = a[x_rest · 2^k_skip + i]` (lifted to F_128). + /// `a_mlv[x_rest] = a[x_rest · 2^k_skip + i]` (lifted to F192). #[test] fn fold_at_node_recovers_witness_slice() { let m = 8; @@ -1013,15 +1006,8 @@ mod tests { let weights = lagrange_weights_naive(k_skip, z); let a_mlv = fold_at_z_naive(&a, m, k_skip, &weights); for x_rest in 0..n_rest { - let expected = if a[x_rest * ell + i] { - F128::ONE - } else { - F128::ZERO - }; - assert_eq!( - a_mlv[x_rest], expected, - "fold at node {i} mismatch at x_rest={x_rest}" - ); + let expected = if a[x_rest * ell + i] { F192::ONE } else { F192::ZERO }; + assert_eq!(a_mlv[x_rest], expected, "fold at node {i} mismatch at x_rest={x_rest}"); } } } @@ -1036,7 +1022,7 @@ mod tests { let a = rng.bits(1 << m); let aprime = rng.bits(1 << m); let a_xor: Vec = a.iter().zip(&aprime).map(|(x, y)| x ^ y).collect(); - let z = rng.f128(); + let z = rng.ext(); let weights = lagrange_weights_naive(k_skip, z); let fa = fold_at_z_naive(&a, m, k_skip, &weights); @@ -1058,15 +1044,15 @@ mod tests { let m = 6; let k_skip = 3; let mut rng = Rng::new(20); - let z = rng.f128(); - let mlv_challenges = rng.f128_vec(m - k_skip); + let z = rng.ext(); + let mlv_challenges = rng.ext_vec(m - k_skip); let zeros = vec![false; 1 << m]; let (a_mlv, b_mlv, msg_1, msg_inf) = uni_skip_fold_and_round_pair_naive(&zeros, &zeros, m, k_skip, z, &mlv_challenges); assert!(a_mlv.iter().all(|v| v.is_zero())); assert!(b_mlv.iter().all(|v| v.is_zero())); - assert_eq!(msg_1, F128::ZERO); - assert_eq!(msg_inf, F128::ZERO); + assert_eq!(msg_1, F192::ZERO); + assert_eq!(msg_inf, F192::ZERO); } #[test] @@ -1076,8 +1062,8 @@ mod tests { let mut rng = Rng::new(33); let a = rng.bits(1 << m); let b = rng.bits(1 << m); - let z = rng.f128(); - let mlv_challenges = rng.f128_vec(m - k_skip); + let z = rng.ext(); + let mlv_challenges = rng.ext_vec(m - k_skip); let o1 = uni_skip_fold_and_round_pair_naive(&a, &b, m, k_skip, z, &mlv_challenges); let o2 = uni_skip_fold_and_round_pair_naive(&a, &b, m, k_skip, z, &mlv_challenges); assert_eq!(o1, o2); @@ -1093,12 +1079,10 @@ mod tests { let mut rng = Rng::new(40); let a = rng.bits(1 << m); let b = rng.bits(1 << m); - let z = rng.f128(); - let mlv_challenges = rng.f128_vec(m - k_skip); - let (_, _, m1_ab, minf_ab) = - uni_skip_fold_and_round_pair_naive(&a, &b, m, k_skip, z, &mlv_challenges); - let (_, _, m1_ba, minf_ba) = - uni_skip_fold_and_round_pair_naive(&b, &a, m, k_skip, z, &mlv_challenges); + let z = rng.ext(); + let mlv_challenges = rng.ext_vec(m - k_skip); + let (_, _, m1_ab, minf_ab) = uni_skip_fold_and_round_pair_naive(&a, &b, m, k_skip, z, &mlv_challenges); + let (_, _, m1_ba, minf_ba) = uni_skip_fold_and_round_pair_naive(&b, &a, m, k_skip, z, &mlv_challenges); assert_eq!(m1_ab, m1_ba); assert_eq!(minf_ab, minf_ba); } @@ -1113,7 +1097,7 @@ mod tests { fn fold_one_row_neon_matches_scalar() { let k_skip = 6; let mut rng = Rng::new(70); - let z = rng.f128(); + let z = rng.ext(); let table = UniSkipFoldTable::new(k_skip, z); for _ in 0..256 { @@ -1123,9 +1107,7 @@ mod tests { } let scalar = table.fold_one_row(&bytes); // SAFETY: on aarch64; bytes has 8 entries; table has 8 chunks. - let neon = unsafe { - fold_one_row_neon_unchecked_8(table.data.as_ptr() as *const u8, bytes.as_ptr()) - }; + let neon = unsafe { fold_one_row_neon_unchecked_8(table.data.as_ptr(), bytes.as_ptr()) }; assert_eq!(scalar, neon, "fold mismatch bytes={bytes:02x?}"); } } @@ -1136,9 +1118,9 @@ mod tests { let mut rng = Rng::new(300); for &log_n in &[1usize, 2, 3, 4, 6] { let n = 1usize << log_n; - let a_orig: Vec = (0..n).map(|_| rng.f128()).collect(); - let b_orig: Vec = (0..n).map(|_| rng.f128()).collect(); - let challenge = rng.f128(); + let a_orig: Vec = (0..n).map(|_| rng.ext()).collect(); + let b_orig: Vec = (0..n).map(|_| rng.ext()).collect(); + let challenge = rng.ext(); let mut a = a_orig.clone(); let mut b = b_orig.clone(); @@ -1163,12 +1145,11 @@ mod tests { /// prover skip per-round c tracking entirely. #[test] fn c_eval_from_round1_c_matches_direct_fold() { - use primitives::field::F8; - use pcs::ntt::{AdditiveNttGf8, InvNttTableByteSingleGf8}; use crate::zerocheck::univariate_skip_optimized::{ - c_s_f128, medium_challenges_ghash, round1_shift_reduce_extract_c_packed, - small_challenges_ghash, + c_s, medium_challenges, round1_shift_reduce_extract_c_packed, small_challenges, }; + use pcs::ntt::{AdditiveNttGf8, InvNttTableByteSingleGf8}; + use primitives::field::F8; const K_SKIP: usize = 6; const N_INNER: usize = 7; @@ -1181,20 +1162,20 @@ mod tests { // Build r with protocol-fixed constants in the middle 7 dims, // matching how `prove` constructs it. - let mut r = vec![F128::ZERO; m]; + let mut r = vec![F192::ZERO; m]; for slot in r[..K_SKIP].iter_mut() { - *slot = rng.f128(); + *slot = rng.ext(); } - for (i, v) in small_challenges_ghash().iter().enumerate() { + for (i, v) in small_challenges().iter().enumerate() { r[K_SKIP + i] = *v; } - for (i, v) in medium_challenges_ghash().iter().enumerate() { + for (i, v) in medium_challenges().iter().enumerate() { r[K_SKIP + 3 + i] = *v; } for slot in r[K_SKIP + N_INNER..].iter_mut() { - *slot = rng.f128(); + *slot = rng.ext(); } - let z = rng.f128(); + let z = rng.ext(); let a_packed = pack_bits(&a); let b_packed = pack_bits(&b); @@ -1203,13 +1184,11 @@ mod tests { let ntt_s = AdditiveNttGf8::new(K_SKIP, F8::ZERO); let ntt_l = AdditiveNttGf8::new(K_SKIP, F8(1u8 << K_SKIP)); let inv_table = InvNttTableByteSingleGf8::new(&ntt_s, &ntt_l); - let (_round1_ab, round1_c) = round1_shift_reduce_extract_c_packed( - &a_packed, &b_packed, &c_packed, m, K_SKIP, &r, &inv_table, - ); + let (_round1_ab, round1_c) = + round1_shift_reduce_extract_c_packed(&a_packed, &b_packed, &c_packed, m, K_SKIP, &r, &inv_table); // Path A: interpolate round1_c at z, scale by C_s. - let c_eval_via_interpolation = - c_s_f128() * interpolate_at_z_on_lambda(&round1_c, K_SKIP, z); + let c_eval_via_interpolation = c_s() * interpolate_at_z_on_lambda(&round1_c, K_SKIP, z); // Path B: direct fold of c at z (Lagrange) then bind each // r_rest = r[K_SKIP..m] element with fold_in_place_single. @@ -1234,15 +1213,15 @@ mod tests { #[test] fn fused_round_matches_unfused() { let mut rng = Rng::new(310); - // fold_and_compute requires lo_size ≥ 2 in SplitEqGhash. eq is over + // fold_and_compute requires lo_size ≥ 2 in SplitEq. eq is over // r_next[1..] (size log_n − 2); with MAX_N_HI = 7, n_lo ≥ 1 needs // eq size ≥ 8 ⇒ log_n ≥ 10. Smaller cases use the unfused path. for &log_n in &[10usize, 11, 12] { let n = 1usize << log_n; - let a: Vec = (0..n).map(|_| rng.f128()).collect(); - let b: Vec = (0..n).map(|_| rng.f128()).collect(); - let r_fold = rng.f128(); - let r_next = rng.f128_vec(log_n - 1); + let a: Vec = (0..n).map(|_| rng.ext()).collect(); + let b: Vec = (0..n).map(|_| rng.ext()).collect(); + let r_fold = rng.ext(); + let r_next = rng.ext_vec(log_n - 1); // Fused path. let (a_fused, b_fused, m1_fused, minf_fused) = @@ -1262,7 +1241,7 @@ mod tests { } /// Parallel `uni_skip_fold_and_round_pair_optimized_packed` produces - /// byte-identical output to the serial version. F128 XOR + multiply sum + /// byte-identical output to the serial version. F192 XOR + multiply sum /// is commutative + associative, so worker scheduling order doesn't /// affect the result. #[test] @@ -1275,20 +1254,14 @@ mod tests { let mut rng = Rng::new(200 + m as u64); let a = rng.bits(1 << m); let b = rng.bits(1 << m); - let z = rng.f128(); - let mlv_challenges = rng.f128_vec(m - k_skip); + let z = rng.ext(); + let mlv_challenges = rng.ext_vec(m - k_skip); let a_packed = pack_bits(&a); let b_packed = pack_bits(&b); let table = UniSkipFoldTable::new(k_skip, z); - let par = uni_skip_fold_and_round_pair_optimized_packed( - &a_packed, - &b_packed, - m, - k_skip, - &table, - &mlv_challenges, - ); + let par = + uni_skip_fold_and_round_pair_optimized_packed(&a_packed, &b_packed, m, k_skip, &table, &mlv_challenges); let ser = uni_skip_fold_and_round_pair_optimized_packed_serial( &a_packed, &b_packed, @@ -1315,8 +1288,7 @@ mod tests { #[test] fn uni_skip_fold_round_pair_padded_matches_dense() { const K_SKIP: usize = 6; - let cases: &[(usize, usize, usize)] = - &[(17, 14, 15_409), (18, 15, 31_401), (19, 16, 42_560)]; + let cases: &[(usize, usize, usize)] = &[(17, 14, 15_409), (18, 15, 31_401), (19, 16, 42_560)]; for &(m, k_log, useful_bits) in cases { let mut rng = Rng::new(0xFADE_F00D_u64.wrapping_add((k_log * 31 + m) as u64)); let total_bits = 1usize << m; @@ -1336,22 +1308,16 @@ mod tests { let a_packed = pack_bits(&a); let b_packed = pack_bits(&b); - let z = rng.f128(); - let mlv_challenges = rng.f128_vec(m - K_SKIP); + let z = rng.ext(); + let mlv_challenges = rng.ext_vec(m - K_SKIP); let table = UniSkipFoldTable::new(K_SKIP, z); let padding = PaddingSpec { k_log, useful_bits_per_block: useful_bits, }; - let dense = uni_skip_fold_and_round_pair_optimized_packed( - &a_packed, - &b_packed, - m, - K_SKIP, - &table, - &mlv_challenges, - ); + let dense = + uni_skip_fold_and_round_pair_optimized_packed(&a_packed, &b_packed, m, K_SKIP, &table, &mlv_challenges); let padded = uni_skip_fold_and_round_pair_optimized_packed_padded( &a_packed, &b_packed, @@ -1361,22 +1327,10 @@ mod tests { &mlv_challenges, &padding, ); - assert_eq!( - dense.0, padded.0, - "a_mlv: m={m}, k_log={k_log}, useful={useful_bits}" - ); - assert_eq!( - dense.1, padded.1, - "b_mlv: m={m}, k_log={k_log}, useful={useful_bits}" - ); - assert_eq!( - dense.2, padded.2, - "msg_1: m={m}, k_log={k_log}, useful={useful_bits}" - ); - assert_eq!( - dense.3, padded.3, - "msg_inf: m={m}, k_log={k_log}, useful={useful_bits}" - ); + assert_eq!(dense.0, padded.0, "a_mlv: m={m}, k_log={k_log}, useful={useful_bits}"); + assert_eq!(dense.1, padded.1, "b_mlv: m={m}, k_log={k_log}, useful={useful_bits}"); + assert_eq!(dense.2, padded.2, "msg_1: m={m}, k_log={k_log}, useful={useful_bits}"); + assert_eq!(dense.3, padded.3, "msg_inf: m={m}, k_log={k_log}, useful={useful_bits}"); } } @@ -1386,7 +1340,7 @@ mod tests { let m = 8; let k_skip = 3; let mut rng = Rng::new(60); - let z = rng.f128(); + let z = rng.ext(); let a = rng.bits(1 << m); let weights = lagrange_weights_naive(k_skip, z); let table = UniSkipFoldTable::new(k_skip, z); @@ -1398,7 +1352,7 @@ mod tests { for x_rest in 0..(1usize << (m - k_skip)) { let direct = { - let mut acc = F128::ZERO; + let mut acc = F192::ZERO; for s in 0..(1usize << k_skip) { if a[x_rest * (1usize << k_skip) + s] { acc += weights[s]; @@ -1406,8 +1360,7 @@ mod tests { } acc }; - let via_table = - table.fold_one_row(&a_packed[x_rest * n_chunks..(x_rest + 1) * n_chunks]); + let via_table = table.fold_one_row(&a_packed[x_rest * n_chunks..(x_rest + 1) * n_chunks]); assert_eq!(via_table, direct, "x_rest={x_rest}"); } } @@ -1426,11 +1379,10 @@ mod tests { let mut rng = Rng::new(100 + m as u64); let a = rng.bits(1 << m); let b = rng.bits(1 << m); - let z = rng.f128(); - let mlv_challenges = rng.f128_vec(m - k_skip); + let z = rng.ext(); + let mlv_challenges = rng.ext_vec(m - k_skip); - let (a_n, b_n, m1_n, minf_n) = - uni_skip_fold_and_round_pair_naive(&a, &b, m, k_skip, z, &mlv_challenges); + let (a_n, b_n, m1_n, minf_n) = uni_skip_fold_and_round_pair_naive(&a, &b, m, k_skip, z, &mlv_challenges); let (a_o, b_o, m1_o, minf_o) = uni_skip_fold_and_round_pair_optimized(&a, &b, m, k_skip, z, &mlv_challenges); @@ -1455,8 +1407,8 @@ mod tests { let mut rng = Rng::new(55); let a = rng.bits(1 << m); let b = rng.bits(1 << m); - let z = rng.f128(); - let r = rng.f128_vec(m - k_skip); + let z = rng.ext(); + let r = rng.ext_vec(m - k_skip); let weights = lagrange_weights_naive(k_skip, z); let a_mlv = fold_at_z_naive(&a, m, k_skip, &weights); @@ -1467,9 +1419,9 @@ mod tests { let eq_remaining = build_eq(&r[1..]); // G(0), G(1), G(∞) by direct definition. - let mut g0 = F128::ZERO; - let mut g1 = F128::ZERO; - let mut g_inf = F128::ZERO; + let mut g0 = F192::ZERO; + let mut g1 = F192::ZERO; + let mut g_inf = F192::ZERO; for x_prime in 0..half { let a0 = a_mlv[2 * x_prime]; let a1 = a_mlv[2 * x_prime + 1]; @@ -1489,9 +1441,9 @@ mod tests { // Degree-2 check: G(X) reconstructed through (G(0), G(1), G(∞)) must // agree with the direct multilinear evaluation at a fresh point X. // Char-2 interpolation: G(X) = G(0) + X·(G(0)+G(1)) + X·(X+1)·G(∞). - let x = rng.f128(); - let g_via_poly = g0 + x * (g0 + g1) + x * (x + F128::ONE) * g_inf; - let mut g_via_sum = F128::ZERO; + let x = rng.ext(); + let g_via_poly = g0 + x * (g0 + g1) + x * (x + F192::ONE) * g_inf; + let mut g_via_sum = F192::ZERO; for x_prime in 0..half { let a0 = a_mlv[2 * x_prime]; let a1 = a_mlv[2 * x_prime + 1]; diff --git a/crates/flock/src/zerocheck/univariate_skip.rs b/crates/flock/src/zerocheck/univariate_skip.rs index 25b60183..8d4784eb 100644 --- a/crates/flock/src/zerocheck/univariate_skip.rs +++ b/crates/flock/src/zerocheck/univariate_skip.rs @@ -1,8 +1,8 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. //! Round-1 prover message (univariate skip). //! //! The round-1 message is `(P^{AB}, P^C)`, each a length-`2^k_skip` vector -//! of F128 values. They are evaluations on the NTT domain `Λ` of the +//! of F192 values. They are evaluations on the NTT domain `Λ` of the //! polynomial (over λ) defined by //! //! P^{AB}(λ) = Σ_{x ∈ {0,1}^{m-k_skip}} eq(r_rest, x) · φ₈(â(λ, x) · b̂(λ, x)) @@ -19,14 +19,14 @@ //! [`super::univariate_skip_optimized`] drops a constant F₈ factor //! `C_s = φ₈(0x1C)` from the eq-on-S weights; this one keeps it. -use primitives::field::{F8, F128, mul_by_x, phi8}; use pcs::ntt::{AdditiveNttGf8, InvNttTableByteSingleGf8}; +use primitives::field::{F8, F192, phi8_192 as phi8}; // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- -pub use primitives::multilinear::build_eq; +pub use primitives::multilinear::eq_table as build_eq; // --------------------------------------------------------------------------- // Naive round-1 prover message (extract_c form) @@ -36,7 +36,7 @@ pub use primitives::multilinear::build_eq; /// inner, no deferred reduction — direct algorithmic translation of the /// protocol formula). /// -/// Returns `(p_ab, p_c)`, each a length-`2^k_skip` F128 vector of evaluations +/// Returns `(p_ab, p_c)`, each a length-`2^k_skip` F192 vector of evaluations /// on Λ. /// /// Preconditions: @@ -47,14 +47,7 @@ pub use primitives::multilinear::build_eq; /// Index convention: for index `i ∈ 0..2^m`, the low `k_skip` bits address /// the *skip* variables (`y_skip ∈ S`), the high `m - k_skip` bits address /// the *rest* variables (`y_rest`). -pub fn round1_naive( - a: &[bool], - b: &[bool], - c: &[bool], - m: usize, - k_skip: usize, - r: &[F128], -) -> (Vec, Vec) { +pub fn round1_naive(a: &[bool], b: &[bool], c: &[bool], m: usize, k_skip: usize, r: &[F192]) -> (Vec, Vec) { assert!(k_skip <= m, "k_skip must be ≤ m"); assert_eq!(a.len(), 1usize << m); assert_eq!(b.len(), 1usize << m); @@ -72,8 +65,8 @@ pub fn round1_naive( // (the skip portion r[0..k_skip] is consumed by the verifier later). let eq_full = build_eq(&r[k_skip..]); - let mut p_ab = vec![F128::ZERO; ell]; - let mut p_c = vec![F128::ZERO; ell]; + let mut p_ab = vec![F192::ZERO; ell]; + let mut p_c = vec![F192::ZERO; ell]; let mut a_col = vec![F8::ZERO; ell]; let mut b_col = vec![F8::ZERO; ell]; @@ -127,43 +120,41 @@ pub fn pack_bits(bits: &[bool]) -> Vec { let mut out = vec![0u8; n_bytes]; // Each output byte depends on 8 contiguous input bits — disjoint, so // process bytes in parallel. - out.par_chunks_mut(1) - .enumerate() - .for_each(|(byte_idx, slot)| { - let mut byte = 0u8; - let base = byte_idx * 8; - for j in 0..8 { - let bit_idx = base + j; - if bit_idx < bits.len() && bits[bit_idx] { - byte |= 1u8 << j; - } + out.par_chunks_mut(1).enumerate().for_each(|(byte_idx, slot)| { + let mut byte = 0u8; + let base = byte_idx * 8; + for j in 0..8 { + let bit_idx = base + j; + if bit_idx < bits.len() && bits[bit_idx] { + byte |= 1u8 << j; } - slot[0] = byte; - }); + } + slot[0] = byte; + }); out } /// Eq table split into a lo half (large, L2-resident) and a hi half (small, /// kept in registers across the inner loop). #[derive(Clone, Debug)] -pub struct SplitEqGhash { +pub struct SplitEq { pub n_lo: usize, pub n_hi: usize, - pub lo: Vec, - pub hi: Vec, + pub lo: Vec, + pub hi: Vec, } -impl SplitEqGhash { - /// C++-default cap on the hi half size — keeps outer F128 muls cheap. +impl SplitEq { + /// C++-default cap on the hi half size — keeps outer F192 muls cheap. pub const MAX_N_HI: usize = 7; - pub fn new(r: &[F128]) -> Self { + pub fn new(r: &[F192]) -> Self { let n = r.len(); let n_hi = n.min(Self::MAX_N_HI); Self::with_n_hi(r, n_hi) } - pub fn with_n_hi(r: &[F128], n_hi: usize) -> Self { + pub fn with_n_hi(r: &[F192], n_hi: usize) -> Self { let n = r.len(); let n_hi = n_hi.min(n); let n_lo = n - n_hi; @@ -176,40 +167,34 @@ impl SplitEqGhash { } } -/// Extend a length-`ell` F128 vector from the input domain S to the extension -/// domain Λ using bit-plane decomposition: for each of the 128 bit positions -/// of F128, run the bit-input NTT (`inv_NTT_S` then `fwd_NTT_Λ` via the +/// Extend a length-`ell` F192 vector from the input domain S to the extension +/// domain Λ using bit-plane decomposition: for each of the 192 bit positions +/// of F192, run the bit-input NTT (`inv_NTT_S` then `fwd_NTT_Λ` via the /// precomputed table) on that bit-plane, scale by γ^b, and accumulate. /// -/// Ports `ntt_extend_f128_vec_ghash` (scalar form). The NTT is F_2-linear and +/// Ports `ntt_extend_vec` (scalar form). The NTT is F_2-linear and /// φ_8 commutes with that linearity, which is what makes the bit-by-bit /// decomposition equal to the direct F_8-valued NTT extension. -pub fn ntt_extend_f128_vec_ghash(in_s: &[F128], inv_table: &InvNttTableByteSingleGf8) -> Vec { +pub fn ntt_extend_vec(in_s: &[F192], inv_table: &InvNttTableByteSingleGf8) -> Vec { let ell = inv_table.ell; assert_eq!(in_s.len(), ell); assert_eq!(ell, 1usize << inv_table.k); - let mut out = vec![F128::ZERO; ell]; + let mut out = vec![F192::ZERO; ell]; let n_chunks = inv_table.n_chunks; - // γ^b for b ∈ [0, 128). - let mut gamma_pow = [F128::ZERO; 128]; - gamma_pow[0] = F128::ONE; - for b in 1..128 { - gamma_pow[b] = mul_by_x(gamma_pow[b - 1]); - } - let mut input_bits = vec![0u8; n_chunks]; let mut out_bytes = vec![F8::ZERO; ell]; - for b in 0..128 { + for b in 0..192 { // Pack bit b of each in_s[z] into z-indexed LSB-first byte form. input_bits.iter_mut().for_each(|x| *x = 0); for z in 0..ell { - let bit = if b < 64 { - (in_s[z].lo >> b) & 1 - } else { - (in_s[z].hi >> (b - 64)) & 1 + let bit = match b / 64 { + 0 => (in_s[z].c0 >> b) & 1, + 1 => (in_s[z].c1 >> (b - 64)) & 1, + 2 => (in_s[z].c2 >> (b - 128)) & 1, + _ => unreachable!(), }; if bit != 0 { input_bits[z / 8] |= 1u8 << (z % 8); @@ -219,9 +204,14 @@ pub fn ntt_extend_f128_vec_ghash(in_s: &[F128], inv_table: &InvNttTableByteSingl // Bit-input NTT. inv_table.apply(&input_bits, &mut out_bytes); - let g_b = gamma_pow[b]; + let basis = match b / 64 { + 0 => F192::new(1u64 << b, 0, 0), + 1 => F192::new(0, 1u64 << (b - 64), 0), + 2 => F192::new(0, 0, 1u64 << (b - 128)), + _ => unreachable!(), + }; for lambda in 0..ell { - out[lambda] += g_b * phi8(out_bytes[lambda]); + out[lambda] += basis * phi8(out_bytes[lambda]); } } @@ -231,7 +221,7 @@ pub fn ntt_extend_f128_vec_ghash(in_s: &[F128], inv_table: &InvNttTableByteSingl /// Round-1 prover message (extract_c form, scalar, algorithmically optimized /// but without the geometric-eq shift_reduce trick). /// -/// Output: `(res_AB, res_C_lifted)`, each length `2^k_skip` F128 vector. +/// Output: `(res_AB, res_C_lifted)`, each length `2^k_skip` F192 vector. /// Both are evaluations on Λ. Output equals `round1_naive(..)` byte-for-byte /// (no C_s factor — see module-level comment). pub fn round1_extract_c( @@ -240,9 +230,9 @@ pub fn round1_extract_c( c: &[bool], m: usize, k_skip: usize, - r: &[F128], + r: &[F192], inv_table: &InvNttTableByteSingleGf8, -) -> (Vec, Vec) { +) -> (Vec, Vec) { assert_eq!(a.len(), 1usize << m); assert_eq!(b.len(), 1usize << m); assert_eq!(c.len(), 1usize << m); @@ -262,9 +252,9 @@ pub fn round1_extract_c_packed( c_packed: &[u8], m: usize, k_skip: usize, - r: &[F128], + r: &[F192], inv_table: &InvNttTableByteSingleGf8, -) -> (Vec, Vec) { +) -> (Vec, Vec) { assert!(k_skip <= m); let total_bytes = (1usize << m) / 8; assert_eq!(a_packed.len(), total_bytes); @@ -276,23 +266,23 @@ pub fn round1_extract_c_packed( let ell = 1usize << k_skip; let n_chunks = ell / 8; - let eq = SplitEqGhash::new(&r[k_skip..]); + let eq = SplitEq::new(&r[k_skip..]); let lo_size = 1usize << eq.n_lo; let hi_size = 1usize << eq.n_hi; - let mut res_ab = vec![F128::ZERO; ell]; + let mut res_ab = vec![F192::ZERO; ell]; // C accumulator stays in S-domain; we NTT-extend once at the end. - let mut res_c_s = vec![F128::ZERO; ell]; + let mut res_c_s = vec![F192::ZERO; ell]; - let mut partial_ab = vec![F128::ZERO; ell]; - let mut partial_c = vec![F128::ZERO; ell]; + let mut partial_ab = vec![F192::ZERO; ell]; + let mut partial_c = vec![F192::ZERO; ell]; let mut a_col = vec![F8::ZERO; ell]; let mut b_col = vec![F8::ZERO; ell]; for x_hi in 0..hi_size { - partial_ab.iter_mut().for_each(|p| *p = F128::ZERO); - partial_c.iter_mut().for_each(|p| *p = F128::ZERO); + partial_ab.iter_mut().for_each(|p| *p = F192::ZERO); + partial_c.iter_mut().for_each(|p| *p = F192::ZERO); for x_lo in 0..lo_size { let x_rest = (x_hi << eq.n_lo) | x_lo; @@ -327,7 +317,7 @@ pub fn round1_extract_c_packed( } // Lift C from S to Λ via bit-plane NTT extension. - let res_c_lifted = ntt_extend_f128_vec_ghash(&res_c_s, inv_table); + let res_c_lifted = ntt_extend_vec(&res_c_s, inv_table); (res_ab, res_c_lifted) } @@ -369,9 +359,9 @@ pub fn round1_extract_c_packed_with_s_hat_v( c_packed: &[u8], m: usize, k_skip: usize, - r: &[F128], + r: &[F192], inv_table: &InvNttTableByteSingleGf8, -) -> (Vec, Vec, Vec) { +) -> (Vec, Vec, Vec) { assert!(k_skip <= m); let total_bytes = (1usize << m) / 8; assert_eq!(a_packed.len(), total_bytes); @@ -383,27 +373,27 @@ pub fn round1_extract_c_packed_with_s_hat_v( let ell = 1usize << k_skip; let n_chunks = ell / 8; - let eq = SplitEqGhash::new(&r[k_skip..]); + let eq = SplitEq::new(&r[k_skip..]); let lo_size = 1usize << eq.n_lo; let hi_size = 1usize << eq.n_hi; - let mut res_ab = vec![F128::ZERO; ell]; + let mut res_ab = vec![F192::ZERO; ell]; // Two C banks, one per value of bit 0 of `x_rest` = bit `k_skip` of the // flat witness index (= `b_7` in ring-switch's parlance). - let mut res_c_s_0 = vec![F128::ZERO; ell]; - let mut res_c_s_1 = vec![F128::ZERO; ell]; + let mut res_c_s_0 = vec![F192::ZERO; ell]; + let mut res_c_s_1 = vec![F192::ZERO; ell]; - let mut partial_ab = vec![F128::ZERO; ell]; - let mut partial_c_0 = vec![F128::ZERO; ell]; - let mut partial_c_1 = vec![F128::ZERO; ell]; + let mut partial_ab = vec![F192::ZERO; ell]; + let mut partial_c_0 = vec![F192::ZERO; ell]; + let mut partial_c_1 = vec![F192::ZERO; ell]; let mut a_col = vec![F8::ZERO; ell]; let mut b_col = vec![F8::ZERO; ell]; for x_hi in 0..hi_size { - partial_ab.iter_mut().for_each(|p| *p = F128::ZERO); - partial_c_0.iter_mut().for_each(|p| *p = F128::ZERO); - partial_c_1.iter_mut().for_each(|p| *p = F128::ZERO); + partial_ab.iter_mut().for_each(|p| *p = F192::ZERO); + partial_c_0.iter_mut().for_each(|p| *p = F192::ZERO); + partial_c_1.iter_mut().for_each(|p| *p = F192::ZERO); for x_lo in 0..lo_size { let x_rest = (x_hi << eq.n_lo) | x_lo; @@ -424,13 +414,9 @@ pub fn round1_extract_c_packed_with_s_hat_v( // C on S — route into bank 0 or bank 1 based on b_7. The eq // factor `eq(r[k_skip], b_7)` is implicit in eq_lo because the - // SplitEqGhash builds the tensor for r[k_skip..]; we strip that + // SplitEq builds the tensor for r[k_skip..]; we strip that // factor out at the end via division. - let target = if b_7 == 0 { - &mut partial_c_0 - } else { - &mut partial_c_1 - }; + let target = if b_7 == 0 { &mut partial_c_0 } else { &mut partial_c_1 }; for s in 0..ell { let c_bit = (c_packed[chunk_offset + s / 8] >> (s % 8)) & 1; if c_bit != 0 { @@ -450,18 +436,18 @@ pub fn round1_extract_c_packed_with_s_hat_v( // Wire output: combined bank sum = original res_c_s. (The eq(r[k_skip], 0) // factor (= 1 + r[k_skip]) is baked into bank 0, eq(r[k_skip], 1) (= r[k_skip]) // into bank 1. Summing reconstitutes the eq(r[k_skip..m], x_rest) sum.) - let mut res_c_s = vec![F128::ZERO; ell]; + let mut res_c_s = vec![F192::ZERO; ell]; for s in 0..ell { res_c_s[s] = res_c_s_0[s] + res_c_s_1[s]; } - let res_c_lifted = ntt_extend_f128_vec_ghash(&res_c_s, inv_table); + let res_c_lifted = ntt_extend_vec(&res_c_s, inv_table); // s_hat_v_c: strip the eq(r[k_skip], ·) factor from each bank by dividing // by 1 + r[k_skip] (bank 0) and r[k_skip] (bank 1). No NTT extension — // lanes are already boolean indices, which is what ring-switch consumes. - let inv_zero = (F128::ONE + r[k_skip]).inv(); + let inv_zero = (F192::ONE + r[k_skip]).inv(); let inv_one = r[k_skip].inv(); - let mut s_hat_v_c = vec![F128::ZERO; 2 * ell]; + let mut s_hat_v_c = vec![F192::ZERO; 2 * ell]; for lane in 0..ell { s_hat_v_c[lane] = res_c_s_0[lane] * inv_zero; s_hat_v_c[ell + lane] = res_c_s_1[lane] * inv_one; @@ -488,8 +474,8 @@ pub fn round1_evals_on_s( c: &[bool], m: usize, k_skip: usize, - r: &[F128], -) -> (Vec, Vec) { + r: &[F192], +) -> (Vec, Vec) { assert!(k_skip <= m); assert_eq!(a.len(), 1usize << m); assert_eq!(b.len(), 1usize << m); @@ -500,8 +486,8 @@ pub fn round1_evals_on_s( let n_chunks_x = 1usize << (m - k_skip); let eq_full = build_eq(&r[k_skip..]); - let mut p_ab = vec![F128::ZERO; ell]; - let mut p_c = vec![F128::ZERO; ell]; + let mut p_ab = vec![F192::ZERO; ell]; + let mut p_c = vec![F192::ZERO; ell]; for x_rest in 0..n_chunks_x { let base = x_rest * ell; @@ -531,23 +517,20 @@ mod tests { #[test] fn build_eq_basic() { // Empty r → table = [1]. - assert_eq!(build_eq(&[]), vec![F128::ONE]); + assert_eq!(build_eq(&[]), vec![F192::ONE]); // Single r = [r0] → table = [(1+r0), r0]. - let r0 = F128 { - lo: 0xCAFEBABE, - hi: 0x12345678, - }; + let r0 = F192::new(0xCAFEBABE, 0x12345678, 0x87654321); let t = build_eq(&[r0]); assert_eq!(t.len(), 2); - assert_eq!(t[0], F128::ONE + r0); + assert_eq!(t[0], F192::ONE + r0); assert_eq!(t[1], r0); // Sum of all eq values is 1 (a defining property of the multilinear eq). let n = 5; let mut rng = Rng::new(99); - let r = rng.f128_vec(n); + let r = rng.ext_vec(n); let t = build_eq(&r); - let sum: F128 = t.iter().copied().fold(F128::ZERO, |a, b| a + b); - assert_eq!(sum, F128::ONE, "Σ_x eq(r, x) should be 1"); + let sum: F192 = t.iter().copied().fold(F192::ZERO, |a, b| a + b); + assert_eq!(sum, F192::ONE, "Σ_x eq(r, x) should be 1"); } #[test] @@ -559,7 +542,7 @@ mod tests { let a = rng.bits(1 << m); let b = rng.bits(1 << m); let c = rng.bits(1 << m); - let r = rng.f128_vec(m); + let r = rng.ext_vec(m); let (p_ab, p_c) = round1_naive(&a, &b, &c, m, k_skip, &r); assert_eq!(p_ab.len(), ell); assert_eq!(p_c.len(), ell); @@ -570,7 +553,7 @@ mod tests { let m = 7; let k_skip = 3; let mut rng = Rng::new(2); - let r = rng.f128_vec(m); + let r = rng.ext_vec(m); let zeros = vec![false; 1 << m]; let (p_ab, p_c) = round1_naive(&zeros, &zeros, &zeros, m, k_skip, &r); assert!(p_ab.iter().all(|v| v.is_zero())); @@ -585,7 +568,7 @@ mod tests { let a = rng.bits(1 << m); let b = rng.bits(1 << m); let c = rng.bits(1 << m); - let r = rng.f128_vec(m); + let r = rng.ext_vec(m); let out1 = round1_naive(&a, &b, &c, m, k_skip, &r); let out2 = round1_naive(&a, &b, &c, m, k_skip, &r); assert_eq!(out1, out2); @@ -602,7 +585,7 @@ mod tests { let c1 = rng.bits(1 << m); let c2 = rng.bits(1 << m); let c_sum: Vec = c1.iter().zip(&c2).map(|(x, y)| x ^ y).collect(); - let r = rng.f128_vec(m); + let r = rng.ext_vec(m); let (ab1, pc1) = round1_naive(&a, &b, &c1, m, k_skip, &r); let (ab2, pc2) = round1_naive(&a, &b, &c2, m, k_skip, &r); @@ -636,13 +619,13 @@ mod tests { let b = rng.bits(1 << m); // Honest c: c = a AND b for every i. let c: Vec = a.iter().zip(&b).map(|(x, y)| *x & *y).collect(); - let r = rng.f128_vec(m); + let r = rng.ext_vec(m); let (p_ab_s, p_c_s) = round1_evals_on_s(&a, &b, &c, m, k_skip, &r); for s in 0..p_ab_s.len() { assert_eq!( p_ab_s[s] + p_c_s[s], - F128::ZERO, + F192::ZERO, "P at S should be 0 for honest witness, but failed at s={s}" ); } @@ -659,10 +642,10 @@ mod tests { let a = rng.bits(1 << m); let b = rng.bits(1 << m); let c = rng.bits(1 << m); - let r = rng.f128_vec(m); + let r = rng.ext_vec(m); let (p_ab_s, p_c_s) = round1_evals_on_s(&a, &b, &c, m, k_skip, &r); - let combined: Vec = p_ab_s.iter().zip(&p_c_s).map(|(x, y)| *x + *y).collect(); + let combined: Vec = p_ab_s.iter().zip(&p_c_s).map(|(x, y)| *x + *y).collect(); let nonzero = combined.iter().any(|v| !v.is_zero()); assert!(nonzero, "P at S should be nonzero for a random witness"); } @@ -681,7 +664,7 @@ mod tests { let a = rng.bits(1 << m); let b = rng.bits(1 << m); let c = rng.bits(1 << m); - let r = rng.f128_vec(m); + let r = rng.ext_vec(m); let table = make_inv_table(k_skip); let (ab, c_l) = round1_extract_c(&a, &b, &c, m, k_skip, &r, &table); assert_eq!(ab.len(), 1usize << k_skip); @@ -696,7 +679,7 @@ mod tests { let a = rng.bits(1 << m); let b = rng.bits(1 << m); let c = rng.bits(1 << m); - let r = rng.f128_vec(m); + let r = rng.ext_vec(m); let table = make_inv_table(k_skip); let out1 = round1_extract_c(&a, &b, &c, m, k_skip, &r, &table); let out2 = round1_extract_c(&a, &b, &c, m, k_skip, &r, &table); @@ -713,12 +696,11 @@ mod tests { let a = pack_bits(&rng.bits(1 << m)); let b = pack_bits(&rng.bits(1 << m)); let c = pack_bits(&rng.bits(1 << m)); - let r = rng.f128_vec(m); + let r = rng.ext_vec(m); let table = make_inv_table(k_skip); let (ab_old, c_old) = round1_extract_c_packed(&a, &b, &c, m, k_skip, &r, &table); - let (ab_new, c_new, _) = - round1_extract_c_packed_with_s_hat_v(&a, &b, &c, m, k_skip, &r, &table); + let (ab_new, c_new, _) = round1_extract_c_packed_with_s_hat_v(&a, &b, &c, m, k_skip, &r, &table); assert_eq!(ab_old, ab_new, "res_ab mismatch m={m} k_skip={k_skip}"); assert_eq!(c_old, c_new, "res_c_lifted mismatch m={m} k_skip={k_skip}"); @@ -731,10 +713,10 @@ mod tests { #[test] fn extract_c_with_s_hat_v_matches_fold_1b_rows() { use pcs::pack::pack_witness; - use pcs::ring_switch::fold_1b_rows_naive; + use pcs::ring_switch::fold_1b_rows; // K_SKIP = 6 is the production setup (LOG_PACKING = 7, so 2 · 2^K_SKIP // = 128 matches s_hat_v's length). The kernel needs m >= K_SKIP + 1 = - // 7 for pack_witness, plus the SplitEqGhash's n_lo + n_hi machinery + // 7 for pack_witness, plus the SplitEq's n_lo + n_hi machinery // wants m - k_skip >= some floor — tested at m=8..11. const K_SKIP: usize = 6; for &m in &[8usize, 9, 10, 11] { @@ -743,21 +725,25 @@ mod tests { let a = pack_bits(&rng.bits(1 << m)); let b = pack_bits(&rng.bits(1 << m)); let c = pack_bits(&z_bits); - let r = rng.f128_vec(m); + let r = rng.ext_vec(m); let table = make_inv_table(K_SKIP); - let (_, _, s_hat_v_c) = - round1_extract_c_packed_with_s_hat_v(&a, &b, &c, m, K_SKIP, &r, &table); + let (_, _, s_hat_v_c) = round1_extract_c_packed_with_s_hat_v(&a, &b, &c, m, K_SKIP, &r, &table); // Reference: fold_1b_rows on the packed C-witness against the // suffix tensor built from r[k_skip + 1 ..]. let packed_c = pack_witness(&z_bits, m); - let suffix = &r[K_SKIP + 1..]; + let suffix = &r[K_SKIP..]; let suffix_tensor = build_eq(suffix); - let want = fold_1b_rows_naive(&packed_c, &suffix_tensor); + let want = fold_1b_rows(&packed_c, &suffix_tensor); + + let c = r[K_SKIP]; + let folded: Vec<_> = (0..pcs::pack::PACKING_WIDTH) + .map(|i| (F192::ONE + c) * s_hat_v_c[i] + c * s_hat_v_c[i + pcs::pack::PACKING_WIDTH]) + .collect(); - assert_eq!(s_hat_v_c.len(), want.len(), "length mismatch at m={m}"); - assert_eq!(s_hat_v_c, want, "s_hat_v_c mismatch at m={m}"); + assert_eq!(folded.len(), want.len(), "length mismatch at m={m}"); + assert_eq!(folded, want, "s_hat_v_c mismatch at m={m}"); } } @@ -771,7 +757,7 @@ mod tests { let a = rng.bits(1 << m); let b = rng.bits(1 << m); let c = rng.bits(1 << m); - let r = rng.f128_vec(m); + let r = rng.ext_vec(m); let table = make_inv_table(k_skip); let (naive_ab, naive_c) = round1_naive(&a, &b, &c, m, k_skip, &r); @@ -794,7 +780,7 @@ mod tests { let a = rng.bits(1 << m); let b = rng.bits(1 << m); let c: Vec = a.iter().zip(&b).map(|(x, y)| *x & *y).collect(); - let r = rng.f128_vec(m); + let r = rng.ext_vec(m); let table = make_inv_table(k_skip); let (naive_ab, naive_c) = round1_naive(&a, &b, &c, m, k_skip, &r); @@ -811,9 +797,9 @@ mod tests { // values as the full eq table when indexed appropriately. let mut rng = Rng::new(300); let n = 6; - let r = rng.f128_vec(n); + let r = rng.ext_vec(n); let full = build_eq(&r); - let eq = SplitEqGhash::new(&r); + let eq = SplitEq::new(&r); assert_eq!(eq.n_lo + eq.n_hi, n); for x in 0..(1 << n) { let x_lo = x & ((1 << eq.n_lo) - 1); @@ -824,7 +810,7 @@ mod tests { #[test] fn ntt_extend_round_trips_naive_c_path() { - // Sanity for the F128 NTT extension: build a length-ell F128 vector by + // Sanity for the F192 NTT extension: build a length-ell F192 vector by // applying the naive (eq-weighted) C accumulation at S, then // NTT-extending it. Compare to running the naive C path (which does // the NTT-extend per row). These must agree because both are linear. @@ -834,7 +820,7 @@ mod tests { let a = rng.bits(1 << m); // unused for C let b = rng.bits(1 << m); // unused for C let c = rng.bits(1 << m); - let r = rng.f128_vec(m); + let r = rng.ext_vec(m); let table = make_inv_table(k_skip); let _ = (&a, &b); // silence unused-var lints @@ -857,7 +843,7 @@ mod tests { let a = rng.bits(1 << m); let b = rng.bits(1 << m); let c = rng.bits(1 << m); - let r = rng.f128_vec(m); + let r = rng.ext_vec(m); let (p_ab, p_c) = round1_naive(&a, &b, &c, m, k_skip, &r); assert!(p_ab.iter().any(|v| !v.is_zero())); diff --git a/crates/flock/src/zerocheck/univariate_skip_optimized.rs b/crates/flock/src/zerocheck/univariate_skip_optimized.rs index 0c4ed0aa..606f6516 100644 --- a/crates/flock/src/zerocheck/univariate_skip_optimized.rs +++ b/crates/flock/src/zerocheck/univariate_skip_optimized.rs @@ -1,16 +1,16 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. //! Round-1 prover message — fully optimized (shift_reduce + extract_c, scalar). //! //! Scalar Rust implementation (no NEON). Three layered optimizations on top of -//! the [`super::round1_extract_c`] scaffold: +//! the `round1_extract_c` scaffold: //! //! 1. **Geometric small-eq + shift_reduce inner** (3 inner-most rest-dims). //! Protocol fixes the three small challenges to //! `r[k_skip..k_skip+3] = φ_8([0xF7, 0x53, 0xB5])`, which makes -//! `eq_small[K] = C_s · α^K` (geometric in α, the AES root in GHASH). +//! `eq_small[K] = C_s · α^K` (geometric in the embedded AES root α). //! The shift_reduce trick computes //! `Σ_K eq_small[K] · φ_8(y_K) = C_s · φ_8(reduce(Σ_K y_K << K))`, -//! replacing 8 F128 mults per lane with 8 u16 XOR-shifts + one F_8 +//! replacing 8 F192 mults per lane with 8 u16 XOR-shifts + one F_8 //! reduction. //! //! 2. **Geometric medium-eq + convert table** (4 next rest-dims). @@ -18,7 +18,7 @@ //! `β_i = γ^{2^{i-1}} / (1 + γ^{2^{i-1}})`, which makes //! `eq_med[b] = γ^b / D` for `D = ∏(1+γ^{2^{i-1}})`. //! Precomputed table `convert[b][v] = γ^b · φ_8(v)` (64 KB) reduces the -//! per-lane medium-eq sum from 16 F128 mults to 16 lookups + 16 XORs. +//! per-lane medium-eq sum from 16 F192 mults to 16 lookups + 16 XORs. //! //! 3. **D⁻¹ absorbed into eq_lo.** //! Pre-scale `eq_lo[i] ← eq_lo[i] · D⁻¹` once before the loop; this cancels @@ -33,12 +33,12 @@ use std::sync::OnceLock; -use primitives::field::gf2_8::gf8_reduce; -use primitives::field::{F8, F128, PHI_8_TABLE, mul_by_x, phi8}; use pcs::ntt::InvNttTableByteSingleGf8; +use primitives::field::gf2_8::gf8_reduce; +use primitives::field::{F8, F192, PHI_8_TABLE_192 as PHI_8_TABLE, phi8_192 as phi8}; use super::PaddingSpec; -use super::univariate_skip::{SplitEqGhash, ntt_extend_f128_vec_ghash, pack_bits}; +use super::univariate_skip::{SplitEq, ntt_extend_vec, pack_bits}; // --------------------------------------------------------------------------- // Protocol constants — fixed by the optimization design. @@ -56,8 +56,8 @@ const N_MEDIUM: usize = 4; /// Choosing these specific values is what makes `eq_small[K] = C_s · α^K`. /// /// **Soundness dependency.** These three constants — together with the -/// four medium constants returned by [`medium_challenges_ghash`] — must be -/// **F₂-linearly independent** in F₁₂₈. Zerocheck soundness relies on this +/// four medium constants returned by [`medium_challenges`] — must be +/// **F₂-linearly independent** in F₁₉₂. Zerocheck soundness relies on this /// (a witness aligned with the friendly subspace would otherwise let the /// prover cancel the URM message), and so does Ligerito's L0 list-collapse /// argument (the SZ bound `(m−7)/|F|` for MLE collisions at `r` requires @@ -68,16 +68,16 @@ pub const SMALL_CHAL_F8: [u8; 3] = [0xF7, 0x53, 0xB5]; /// `C_s` as an F_8 value. Verified empirically by the C++ project. pub const C_S_F8: u8 = 0x1C; -/// The constant `C_s = φ_8(0x1C) ∈ F_{2^128}` — the relative scaling factor +/// The constant `C_s = φ_8(0x1C) ∈ F_{2^192}` — the relative scaling factor /// between this optimized output and the naive output. -pub fn c_s_f128() -> F128 { +pub fn c_s() -> F192 { phi8(F8(C_S_F8)) } -/// The three F_128 small challenges (embeddings of [`SMALL_CHAL_F8`]) — caller +/// The three F192 small challenges (embeddings of [`SMALL_CHAL_F8`]) — caller /// must place these at `r[k_skip..k_skip+3]` for the naive cross-check to /// produce a result related to the optimized output by exactly `C_s`. -pub fn small_challenges_ghash() -> [F128; 3] { +pub fn small_challenges() -> [F192; 3] { [ phi8(F8(SMALL_CHAL_F8[0])), phi8(F8(SMALL_CHAL_F8[1])), @@ -85,34 +85,27 @@ pub fn small_challenges_ghash() -> [F128; 3] { ] } -/// The four F_128 medium challenges `β_i = γ^{2^{i-1}} / (1 + γ^{2^{i-1}})`. +/// The four F192 medium challenges `β_i = γ^{2^{i-1}} / (1 + γ^{2^{i-1}})`. /// Caller must place these at `r[k_skip+3..k_skip+7]` for the naive /// cross-check. -pub fn medium_challenges_ghash() -> [F128; 4] { - let g1 = F128 { - lo: 1u64 << 1, - hi: 0, - }; // γ^1 - let g2 = F128 { - lo: 1u64 << 2, - hi: 0, - }; // γ^2 - let g4 = F128 { - lo: 1u64 << 4, - hi: 0, - }; // γ^4 - let g8 = F128 { - lo: 1u64 << 8, - hi: 0, - }; // γ^8 +pub fn medium_challenges() -> [F192; 4] { + let g1 = medium_generator(); + let g2 = g1.square(); + let g4 = g2.square(); + let g8 = g4.square(); [ - g1 * (F128::ONE + g1).inv(), - g2 * (F128::ONE + g2).inv(), - g4 * (F128::ONE + g4).inv(), - g8 * (F128::ONE + g8).inv(), + g1 * (F192::ONE + g1).inv(), + g2 * (F192::ONE + g2).inv(), + g4 * (F192::ONE + g4).inv(), + g8 * (F192::ONE + g8).inv(), ] } +/// Protocol medium-coordinate generator in the tower basis. +const fn medium_generator() -> F192 { + F192::new(0x243f_6a88_85a3_08d3, 0x1319_8a2e_0370_7344, 0xa409_3822_299f_31d0) +} + /// `C_2 = (1+r_2)(1+r_3)` where `r_2 = φ_8(0x53)` (= `α^2/(1+α^2)`), /// `r_3 = φ_8(0xB5)` (= `α^4/(1+α^4)`). This is the residual small-eq /// constant after the first small friendly bit (`b_3[0]`, indexed by @@ -125,64 +118,52 @@ pub fn medium_challenges_ghash() -> [F128; 4] { /// Used in [`round1_shift_reduce_extract_c_packed_padded_with_s_hat_v`] to /// post-scale the raw bank values into canonical `s_hat_v_c` (which /// `ring_switch::fold_1b_rows` would produce against suffix `r[k_skip+1..m]`). -pub fn c_2_small_f128() -> F128 { +pub fn c_2_small() -> F192 { let r_2 = phi8(F8(SMALL_CHAL_F8[1])); let r_3 = phi8(F8(SMALL_CHAL_F8[2])); - (F128::ONE + r_2) * (F128::ONE + r_3) + (F192::ONE + r_2) * (F192::ONE + r_3) } -/// `α⁻¹` in F_128, as a subfield-embedded F_8 element. Used to strip the +/// `α⁻¹` in F192, as a subfield-embedded F_8 element. Used to strip the /// extra `α` factor from `s_hat_v_c`'s bank 1 (the K-odd lattice's raw /// contribution is `α · α^{2 b_3[1] + 4 b_3[2]}`; canonical wants just /// `α^{2 b_3[1] + 4 b_3[2]}`). -pub fn alpha_inv_f128() -> F128 { +pub fn alpha_inv() -> F192 { // α in F_8 = byte 0x02 (the polynomial generator). Its inverse is α^254; // F8::inv computes it via the standard extended Euclidean / power table. phi8(F8(0x02).inv()) } /// `D = (1+γ)(1+γ^2)(1+γ^4)(1+γ^8)`; `D⁻¹` cancels the medium-eq normalization. -fn compute_d_inv() -> F128 { - let g1 = F128 { - lo: 1u64 << 1, - hi: 0, - }; - let g2 = F128 { - lo: 1u64 << 2, - hi: 0, - }; - let g4 = F128 { - lo: 1u64 << 4, - hi: 0, - }; - let g8 = F128 { - lo: 1u64 << 8, - hi: 0, - }; - ((F128::ONE + g1) * (F128::ONE + g2) * (F128::ONE + g4) * (F128::ONE + g8)).inv() +fn compute_d_inv() -> F192 { + let g1 = medium_generator(); + let g2 = g1.square(); + let g4 = g2.square(); + let g8 = g4.square(); + ((F192::ONE + g1) * (F192::ONE + g2) * (F192::ONE + g4) * (F192::ONE + g8)).inv() } -static D_INV_CACHE: OnceLock = OnceLock::new(); -fn d_inv() -> F128 { +static D_INV_CACHE: OnceLock = OnceLock::new(); +fn d_inv() -> F192 { *D_INV_CACHE.get_or_init(compute_d_inv) } // --------------------------------------------------------------------------- // Convert table: γ^b · φ_8(v) for b ∈ [0, 16), v ∈ [0, 256). -// 16 × 256 × 16 bytes = 64 KB. Computed once, cached via OnceLock. +// 16 × 256 × 24 bytes = 96 KB. Computed once, cached via OnceLock. // --------------------------------------------------------------------------- const CONVERT_TABLE_SIZE: usize = 16 * 256; -static CONVERT_TABLE_CACHE: OnceLock> = OnceLock::new(); +static CONVERT_TABLE_CACHE: OnceLock> = OnceLock::new(); -fn build_convert_table() -> Vec { - let mut gamma_pow = [F128::ZERO; 16]; - gamma_pow[0] = F128::ONE; +fn build_convert_table() -> Vec { + let mut gamma_pow = [F192::ZERO; 16]; + gamma_pow[0] = F192::ONE; for b in 1..16 { - gamma_pow[b] = mul_by_x(gamma_pow[b - 1]); + gamma_pow[b] = gamma_pow[b - 1] * medium_generator(); } - let mut table = vec![F128::ZERO; CONVERT_TABLE_SIZE]; + let mut table = vec![F192::ZERO; CONVERT_TABLE_SIZE]; for b in 0..16 { let g_b = gamma_pow[b]; for v in 0..256 { @@ -192,7 +173,7 @@ fn build_convert_table() -> Vec { table } -fn convert_table() -> &'static [F128] { +fn convert_table() -> &'static [F192] { CONVERT_TABLE_CACHE.get_or_init(build_convert_table) } @@ -227,8 +208,8 @@ fn shift_reduce_inner_ab_neon( a_col: &mut [F8], b_col: &mut [F8], ) { - use primitives::field::gf2_8::neon::{gf8_mul_vec16, gf8_reduce_vec16}; use core::arch::aarch64::*; + use primitives::field::gf2_8::neon::{gf8_mul_vec16, gf8_reduce_vec16}; let byte_base_b = chunk_byte_base + b_med * N_CHUNKS * 8; @@ -308,6 +289,9 @@ fn shift_reduce_inner_ab_neon( // --------------------------------------------------------------------------- #[cfg(target_arch = "aarch64")] +// `0 ^ BH` is the i = 0 case of the `i ^ BH` row-select pattern below; spelling +// it out keeps the four loads visibly parallel. +#[allow(clippy::identity_op)] #[inline(always)] unsafe fn xor_apply_byte_into_8_regs( table_base: *const u8, @@ -376,8 +360,8 @@ unsafe fn fused_apply_one_k( acc3_lo: &mut core::arch::aarch64::uint16x8_t, acc3_hi: &mut core::arch::aarch64::uint16x8_t, ) { - use primitives::field::gf2_8::neon::gf8_mul_vec16; use core::arch::aarch64::*; + use primitives::field::gf2_8::neon::gf8_mul_vec16; unsafe { // b = 0: identity permutation — plain load of the 4 chunks. let ra0 = table_base.add(*a_row as usize * 64); @@ -512,8 +496,8 @@ fn shift_reduce_inner_ab_fused_neon( b_med: usize, out: &mut [u8; 64], ) { - use primitives::field::gf2_8::neon::gf8_reduce_vec16; use core::arch::aarch64::*; + use primitives::field::gf2_8::neon::gf8_reduce_vec16; let byte_base_b = chunk_byte_base + b_med * N_CHUNKS * 8; let table_base = inv_table.data_ptr(); @@ -586,46 +570,16 @@ fn shift_reduce_inner_ab( #[cfg(target_arch = "aarch64")] { let _ = (a_col, b_col); // unused in the fused path - shift_reduce_inner_ab_fused_neon( - a_packed, - b_packed, - inv_table, - chunk_byte_base, - b_med, - out, - ); + shift_reduce_inner_ab_fused_neon(a_packed, b_packed, inv_table, chunk_byte_base, b_med, out); } #[cfg(all(target_arch = "x86_64", target_feature = "gfni"))] { // SAFETY: gfni is statically enabled at compile time. - unsafe { - shift_reduce_inner_ab_gfni( - a_packed, - b_packed, - inv_table, - chunk_byte_base, - b_med, - out, - a_col, - b_col, - ) - }; + unsafe { shift_reduce_inner_ab_gfni(a_packed, b_packed, inv_table, chunk_byte_base, b_med, out, a_col, b_col) }; } - #[cfg(not(any( - target_arch = "aarch64", - all(target_arch = "x86_64", target_feature = "gfni") - )))] + #[cfg(not(any(target_arch = "aarch64", all(target_arch = "x86_64", target_feature = "gfni"))))] { - shift_reduce_inner_ab_scalar( - a_packed, - b_packed, - inv_table, - chunk_byte_base, - b_med, - out, - a_col, - b_col, - ); + shift_reduce_inner_ab_scalar(a_packed, b_packed, inv_table, chunk_byte_base, b_med, out, a_col, b_col); } } @@ -743,15 +697,15 @@ fn shift_reduce_inner_ab_scalar( /// Compute the round-1 prover message via the full shift_reduce + extract_c /// optimization, in scalar Rust. /// -/// Output relative to [`super::round1_naive`]: +/// Output relative to `round1_naive`: /// `C_s · (res_AB[i] + res_C_lifted[i]) = naive_p_ab[i] + naive_p_c[i]` /// /// Preconditions: /// - `k_skip == K_SKIP` (= 6) /// - `m >= k_skip + N_INNER` (= 13) /// - `r.len() == m`. `r[k_skip..k_skip+7]` must hold the protocol-fixed small -/// and medium constants (see [`small_challenges_ghash`] / -/// [`medium_challenges_ghash`]) for the naive cross-check to line up. Only +/// and medium constants (see [`small_challenges`] / +/// [`medium_challenges`]) for the naive cross-check to line up. Only /// `r[k_skip+7..m]` is used internally. /// - `inv_table.k == k_skip`. pub fn round1_shift_reduce_extract_c( @@ -760,9 +714,9 @@ pub fn round1_shift_reduce_extract_c( c: &[bool], m: usize, k_skip: usize, - r: &[F128], + r: &[F192], inv_table: &InvNttTableByteSingleGf8, -) -> (Vec, Vec) { +) -> (Vec, Vec) { assert_eq!(a.len(), 1usize << m); assert_eq!(b.len(), 1usize << m); assert_eq!(c.len(), 1usize << m); @@ -784,100 +738,33 @@ pub fn round1_shift_reduce_extract_c( // // --------------------------------------------------------------------------- -/// Gather-and-accumulate the φ_8 convert tables for one chunk, across all -/// `ELL` lanes. -/// -/// Four lanes are swept per pass. Each lane's accumulator is a serial XOR -/// chain of `n_b_med` dependent table loads, so a one-lane-at-a-time sweep -/// keeps only three such chains in flight and the loop stalls on the gather -/// latency. Four lanes give twelve independent chains — enough to cover it — -/// and let the `b_med` table base be hoisted out of all twelve index -/// computations. -/// -/// `ELL` is 64, so the four-lane step divides it exactly. -/// -/// # Safety -/// `convert` must hold at least `n_b_med * 256` entries; every gather index is -/// `b_med * 256 + byte` with `b_med < n_b_med` and `byte` a `u8`. -#[cfg(target_arch = "aarch64")] -#[inline] -unsafe fn accumulate_convert_lanes( - convert: &[F128], - chunk_ab_bytes: &[[u8; 64]; 1 << N_MEDIUM], - chunk_c_bytes: &[[u8; 64]; 1 << N_MEDIUM], - n_b_med: usize, - eq_lo_val: F128, - partial_ab: &mut [F128; ELL], - partial_c_0: &mut [F128; ELL], - partial_c_1: &mut [F128; ELL], -) { - use core::arch::aarch64::*; - - const { assert!(ELL.is_multiple_of(4)) }; - - // SAFETY: caller guarantees the convert table covers every gather index. - unsafe { - let convert_ptr = convert.as_ptr() as *const u8; - let to_f128 = |v: uint8x16_t| { - let w = vreinterpretq_u64_u8(v); - F128 { - lo: vgetq_lane_u64::<0>(w), - hi: vgetq_lane_u64::<1>(w), - } - }; - - for lane in (0..ELL).step_by(4) { - let mut ab = [vdupq_n_u8(0); 4]; - let mut c_0 = [vdupq_n_u8(0); 4]; - let mut c_1 = [vdupq_n_u8(0); 4]; - for b_med in 0..n_b_med { - let table = convert_ptr.add(b_med * 256 * 16); - let ab_row = &chunk_ab_bytes[b_med]; - let c_row = &chunk_c_bytes[b_med]; - for j in 0..4 { - let v_ab = ab_row[lane + j] as usize; - let v_c = c_row[lane + j] as usize; - ab[j] = veorq_u8(ab[j], vld1q_u8(table.add(v_ab * 16))); - c_0[j] = veorq_u8(c_0[j], vld1q_u8(table.add((v_c & 0x55) * 16))); - c_1[j] = veorq_u8(c_1[j], vld1q_u8(table.add((v_c & 0xAA) * 16))); - } - } - for j in 0..4 { - partial_ab[lane + j] += to_f128(ab[j]) * eq_lo_val; - partial_c_0[lane + j] += to_f128(c_0[j]) * eq_lo_val; - partial_c_1[lane + j] += to_f128(c_1[j]) * eq_lo_val; - } - } - } -} - /// Per-worker scratch and local accumulators, with C split into its two banks. struct WorkerState { - partial_ab: [F128; ELL], - partial_c_0: [F128; ELL], - partial_c_1: [F128; ELL], + partial_ab: [F192; ELL], + partial_c_0: [F192; ELL], + partial_c_1: [F192; ELL], chunk_ab_bytes: [[u8; 64]; 1 << N_MEDIUM], chunk_c_bytes: [[u8; 64]; 1 << N_MEDIUM], a_col: [F8; ELL], b_col: [F8; ELL], - local_res_ab: [F128; ELL], - local_res_c_s_0: [F128; ELL], - local_res_c_s_1: [F128; ELL], + local_res_ab: [F192; ELL], + local_res_c_s_0: [F192; ELL], + local_res_c_s_1: [F192; ELL], } impl WorkerState { fn new() -> Self { Self { - partial_ab: [F128::ZERO; ELL], - partial_c_0: [F128::ZERO; ELL], - partial_c_1: [F128::ZERO; ELL], + partial_ab: [F192::ZERO; ELL], + partial_c_0: [F192::ZERO; ELL], + partial_c_1: [F192::ZERO; ELL], chunk_ab_bytes: [[0u8; 64]; 1 << N_MEDIUM], chunk_c_bytes: [[0u8; 64]; 1 << N_MEDIUM], a_col: [F8::ZERO; ELL], b_col: [F8::ZERO; ELL], - local_res_ab: [F128::ZERO; ELL], - local_res_c_s_0: [F128::ZERO; ELL], - local_res_c_s_1: [F128::ZERO; ELL], + local_res_ab: [F192::ZERO; ELL], + local_res_c_s_0: [F192::ZERO; ELL], + local_res_c_s_1: [F192::ZERO; ELL], } } } @@ -895,14 +782,14 @@ fn process_one_x_hi( b_packed: &[u8], c_packed: &[u8], inv_table: &InvNttTableByteSingleGf8, - eq_lo_scaled: &[F128], - eq_hi_val: F128, - convert: &[F128], + eq_lo_scaled: &[F192], + eq_hi_val: F192, + convert: &[F192], state: &mut WorkerState, ) { - state.partial_ab.iter_mut().for_each(|p| *p = F128::ZERO); - state.partial_c_0.iter_mut().for_each(|p| *p = F128::ZERO); - state.partial_c_1.iter_mut().for_each(|p| *p = F128::ZERO); + state.partial_ab.iter_mut().for_each(|p| *p = F192::ZERO); + state.partial_c_0.iter_mut().for_each(|p| *p = F192::ZERO); + state.partial_c_1.iter_mut().for_each(|p| *p = F192::ZERO); let n_lo = n_lo_and_inner - N_INNER; @@ -936,38 +823,20 @@ fn process_one_x_hi( bit_transpose_64bytes(c_in, &mut state.chunk_c_bytes[b_med]); } - #[cfg(target_arch = "aarch64")] - // SAFETY: `convert` holds 16·256 F128 entries, and every gather - // index below is `b_med · 256 + byte` with `b_med < 1 << N_MEDIUM`. - unsafe { - accumulate_convert_lanes( - convert, - &state.chunk_ab_bytes, - &state.chunk_c_bytes, - 1 << N_MEDIUM, - eq_lo_val, - &mut state.partial_ab, - &mut state.partial_c_0, - &mut state.partial_c_1, - ); - } - #[cfg(not(target_arch = "aarch64"))] - { - for lane in 0..ELL { - let mut cf_ab = F128::ZERO; - let mut cf_c_0 = F128::ZERO; - let mut cf_c_1 = F128::ZERO; - for b_med in 0..(1 << N_MEDIUM) { - let v_ab = state.chunk_ab_bytes[b_med][lane] as usize; - let v_c = state.chunk_c_bytes[b_med][lane] as usize; - cf_ab += convert[b_med * 256 + v_ab]; - cf_c_0 += convert[b_med * 256 + (v_c & 0x55)]; - cf_c_1 += convert[b_med * 256 + (v_c & 0xAA)]; - } - state.partial_ab[lane] += cf_ab * eq_lo_val; - state.partial_c_0[lane] += cf_c_0 * eq_lo_val; - state.partial_c_1[lane] += cf_c_1 * eq_lo_val; + for lane in 0..ELL { + let mut cf_ab = F192::ZERO; + let mut cf_c_0 = F192::ZERO; + let mut cf_c_1 = F192::ZERO; + for b_med in 0..(1 << N_MEDIUM) { + let v_ab = state.chunk_ab_bytes[b_med][lane] as usize; + let v_c = state.chunk_c_bytes[b_med][lane] as usize; + cf_ab += convert[b_med * 256 + v_ab]; + cf_c_0 += convert[b_med * 256 + (v_c & 0x55)]; + cf_c_1 += convert[b_med * 256 + (v_c & 0xAA)]; } + state.partial_ab[lane] += cf_ab * eq_lo_val; + state.partial_c_0[lane] += cf_c_0 * eq_lo_val; + state.partial_c_1[lane] += cf_c_1 * eq_lo_val; } } else { for b_med in 0..n_b_med { @@ -988,38 +857,20 @@ fn process_one_x_hi( bit_transpose_64bytes(c_in, &mut state.chunk_c_bytes[b_med]); } - #[cfg(target_arch = "aarch64")] - // SAFETY: `convert` holds 16·256 F128 entries, and every gather - // index below is `b_med · 256 + byte` with `b_med < 1 << N_MEDIUM`. - unsafe { - accumulate_convert_lanes( - convert, - &state.chunk_ab_bytes, - &state.chunk_c_bytes, - n_b_med, - eq_lo_val, - &mut state.partial_ab, - &mut state.partial_c_0, - &mut state.partial_c_1, - ); - } - #[cfg(not(target_arch = "aarch64"))] - { - for lane in 0..ELL { - let mut cf_ab = F128::ZERO; - let mut cf_c_0 = F128::ZERO; - let mut cf_c_1 = F128::ZERO; - for b_med in 0..n_b_med { - let v_ab = state.chunk_ab_bytes[b_med][lane] as usize; - let v_c = state.chunk_c_bytes[b_med][lane] as usize; - cf_ab += convert[b_med * 256 + v_ab]; - cf_c_0 += convert[b_med * 256 + (v_c & 0x55)]; - cf_c_1 += convert[b_med * 256 + (v_c & 0xAA)]; - } - state.partial_ab[lane] += cf_ab * eq_lo_val; - state.partial_c_0[lane] += cf_c_0 * eq_lo_val; - state.partial_c_1[lane] += cf_c_1 * eq_lo_val; + for lane in 0..ELL { + let mut cf_ab = F192::ZERO; + let mut cf_c_0 = F192::ZERO; + let mut cf_c_1 = F192::ZERO; + for b_med in 0..n_b_med { + let v_ab = state.chunk_ab_bytes[b_med][lane] as usize; + let v_c = state.chunk_c_bytes[b_med][lane] as usize; + cf_ab += convert[b_med * 256 + v_ab]; + cf_c_0 += convert[b_med * 256 + (v_c & 0x55)]; + cf_c_1 += convert[b_med * 256 + (v_c & 0xAA)]; } + state.partial_ab[lane] += cf_ab * eq_lo_val; + state.partial_c_0[lane] += cf_c_0 * eq_lo_val; + state.partial_c_1[lane] += cf_c_1 * eq_lo_val; } } } @@ -1075,7 +926,7 @@ fn build_b_med_counts(padding: &PaddingSpec) -> (usize, Vec) { /// Packed-input variant of [`round1_shift_reduce_extract_c`]. **Parallel by /// default** via rayon — the outer x_hi loop is distributed across workers, /// each with its own scratch + local accumulator. Reduction is a per-lane -/// F128 XOR across workers (commutative + associative). +/// F192 XOR across workers (commutative + associative). /// /// To run single-threaded for debugging, set `RAYON_NUM_THREADS=1`. pub fn round1_shift_reduce_extract_c_packed( @@ -1084,9 +935,9 @@ pub fn round1_shift_reduce_extract_c_packed( c_packed: &[u8], m: usize, k_skip: usize, - r: &[F128], + r: &[F192], inv_table: &InvNttTableByteSingleGf8, -) -> (Vec, Vec) { +) -> (Vec, Vec) { round1_shift_reduce_extract_c_packed_padded( a_packed, b_packed, @@ -1109,10 +960,10 @@ pub fn round1_shift_reduce_extract_c_packed_padded( c_packed: &[u8], m: usize, k_skip: usize, - r: &[F128], + r: &[F192], inv_table: &InvNttTableByteSingleGf8, padding: &PaddingSpec, -) -> (Vec, Vec) { +) -> (Vec, Vec) { let (ab, c, _) = round1_shift_reduce_extract_c_packed_padded_with_s_hat_v( a_packed, b_packed, c_packed, m, k_skip, r, inv_table, padding, ); @@ -1140,10 +991,10 @@ pub fn round1_shift_reduce_extract_c_packed_padded_with_s_hat_v( c_packed: &[u8], m: usize, k_skip: usize, - r: &[F128], + r: &[F192], inv_table: &InvNttTableByteSingleGf8, padding: &PaddingSpec, -) -> (Vec, Vec, Vec) { +) -> (Vec, Vec, Vec) { use rayon::prelude::*; assert_eq!(k_skip, K_SKIP, "optimized variant is k_skip=6 only"); @@ -1159,13 +1010,13 @@ pub fn round1_shift_reduce_extract_c_packed_padded_with_s_hat_v( assert_eq!(r.len(), m); assert_eq!(inv_table.k, k_skip); - let eq = SplitEqGhash::new(&r[k_skip + N_INNER..]); + let eq = SplitEq::new(&r[k_skip + N_INNER..]); let big_lo_size = 1usize << eq.n_lo; let hi_size = 1usize << eq.n_hi; let n_lo_and_inner = eq.n_lo + N_INNER; let d_inv_val = d_inv(); - let eq_lo_scaled: Vec = eq.lo.iter().map(|v| *v * d_inv_val).collect(); + let eq_lo_scaled: Vec = eq.lo.iter().map(|v| *v * d_inv_val).collect(); let convert = convert_table(); let eq_hi = &eq.hi; @@ -1194,7 +1045,7 @@ pub fn round1_shift_reduce_extract_c_packed_padded_with_s_hat_v( }) .map(|s| (s.local_res_ab, s.local_res_c_s_0, s.local_res_c_s_1)) .reduce( - || ([F128::ZERO; ELL], [F128::ZERO; ELL], [F128::ZERO; ELL]), + || ([F192::ZERO; ELL], [F192::ZERO; ELL], [F192::ZERO; ELL]), |(mut ab1, mut c0_1, mut c1_1), (ab2, c0_2, c1_2)| { for i in 0..ELL { ab1[i] += ab2[i]; @@ -1207,18 +1058,18 @@ pub fn round1_shift_reduce_extract_c_packed_padded_with_s_hat_v( // Wire output: bank_0 + bank_1 reconstructs the original `res_c_s` (by // F_2-linearity of φ_8 over the masked-byte sum). - let mut res_c_s_combined = [F128::ZERO; ELL]; + let mut res_c_s_combined = [F192::ZERO; ELL]; for i in 0..ELL { res_c_s_combined[i] = res_c_s_0[i] + res_c_s_1[i]; } - let res_c_lifted = ntt_extend_f128_vec_ghash(&res_c_s_combined, inv_table); + let res_c_lifted = ntt_extend_vec(&res_c_s_combined, inv_table); // s_hat_v_c canonical form: apply residual C_2 (small-eq constant for // r[k_skip+1..k_skip+3]) and α⁻¹ (strips bank 1's extra α factor). - let c_2 = c_2_small_f128(); - let alpha_inv = alpha_inv_f128(); + let c_2 = c_2_small(); + let alpha_inv = alpha_inv(); let c_2_alpha_inv = c_2 * alpha_inv; - let mut s_hat_v_c = vec![F128::ZERO; 2 * ELL]; + let mut s_hat_v_c = vec![F192::ZERO; 2 * ELL]; for lane in 0..ELL { s_hat_v_c[lane] = c_2 * res_c_s_0[lane]; s_hat_v_c[ELL + lane] = c_2_alpha_inv * res_c_s_1[lane]; @@ -1238,9 +1089,9 @@ fn round1_shift_reduce_extract_c_packed_serial( c_packed: &[u8], m: usize, k_skip: usize, - r: &[F128], + r: &[F192], inv_table: &InvNttTableByteSingleGf8, -) -> (Vec, Vec) { +) -> (Vec, Vec) { assert_eq!(k_skip, K_SKIP); assert!(m >= k_skip + N_INNER); let total_bytes = (1usize << m) / 8; @@ -1250,13 +1101,13 @@ fn round1_shift_reduce_extract_c_packed_serial( assert_eq!(r.len(), m); assert_eq!(inv_table.k, k_skip); - let eq = SplitEqGhash::new(&r[k_skip + N_INNER..]); + let eq = SplitEq::new(&r[k_skip + N_INNER..]); let big_lo_size = 1usize << eq.n_lo; let hi_size = 1usize << eq.n_hi; let n_lo_and_inner = eq.n_lo + N_INNER; let d_inv_val = d_inv(); - let eq_lo_scaled: Vec = eq.lo.iter().map(|v| *v * d_inv_val).collect(); + let eq_lo_scaled: Vec = eq.lo.iter().map(|v| *v * d_inv_val).collect(); let convert = convert_table(); let (within_outer_mask, b_med_counts) = build_b_med_counts(&PaddingSpec::dense(m)); @@ -1280,13 +1131,13 @@ fn round1_shift_reduce_extract_c_packed_serial( ); } - let res_c_s: Vec = state + let res_c_s: Vec = state .local_res_c_s_0 .iter() .zip(state.local_res_c_s_1) .map(|(a, b)| *a + b) .collect(); - let res_c_lifted = ntt_extend_f128_vec_ghash(&res_c_s, inv_table); + let res_c_lifted = ntt_extend_vec(&res_c_s, inv_table); (state.local_res_ab.to_vec(), res_c_lifted) } @@ -1294,8 +1145,8 @@ fn round1_shift_reduce_extract_c_packed_serial( mod tests { use super::*; use crate::test_rng::Rng; - use pcs::ntt::AdditiveNttGf8; use crate::zerocheck::univariate_skip::round1_naive; + use pcs::ntt::AdditiveNttGf8; #[cfg(all(target_arch = "x86_64", target_feature = "gfni"))] #[test] @@ -1319,13 +1170,27 @@ mod tests { let mut out_scalar = [0u8; 64]; shift_reduce_inner_ab_scalar( - &a_packed, &b_packed, &inv_table, 0, 0, &mut out_scalar, &mut a_col, &mut b_col, + &a_packed, + &b_packed, + &inv_table, + 0, + 0, + &mut out_scalar, + &mut a_col, + &mut b_col, ); let mut out_gfni = [0u8; 64]; // SAFETY: cfg-gated on gfni. unsafe { shift_reduce_inner_ab_gfni( - &a_packed, &b_packed, &inv_table, 0, 0, &mut out_gfni, &mut a_col, &mut b_col, + &a_packed, + &b_packed, + &inv_table, + 0, + 0, + &mut out_gfni, + &mut a_col, + &mut b_col, ) }; assert_eq!(out_scalar, out_gfni); @@ -1337,7 +1202,7 @@ mod tests { /// L0 both depend on the seven "friendly" constants — three small /// (`φ_8(SMALL_CHAL_F8[k])`, k ∈ 0..3) and four medium /// (`γ^{2^i}/(1+γ^{2^i})`, i ∈ 0..4) — being **F₂-linearly independent** - /// in F₁₂₈. + /// in F₁₉₂. /// /// Zerocheck needs this so that the prover's URM message can't be /// trivially canceled by a malicious witness aligned with the friendly @@ -1349,36 +1214,34 @@ mod tests { /// cheating prover could engineer their witness so two candidates' /// MLEs agree at the friendly point with probability 1. /// - /// The check: form the 7×128 binary matrix whose rows are the bit + /// The check: form the 7×192 binary matrix whose rows are the bit /// representations of the seven constants, Gauss-eliminate over F₂, /// assert rank = 7. #[test] fn friendly_challenges_f2_independent() { - // Pack each F₁₂₈ element into a u128 (lo, hi → 128 bits). - let mut basis: Vec = small_challenges_ghash() + let mut basis: Vec<[u64; 3]> = small_challenges() .iter() - .chain(medium_challenges_ghash().iter()) - .map(|f| ((f.hi as u128) << 64) | (f.lo as u128)) + .chain(medium_challenges().iter()) + .map(|f| [f.c0, f.c1, f.c2]) .collect(); - assert_eq!( - basis.len(), - 7, - "expected 3 small + 4 medium friendly values" - ); + assert_eq!(basis.len(), 7, "expected 3 small + 4 medium friendly values"); // Row-reduce over F₂. For each column from MSB to LSB, find a row // with that bit set (a pivot), swap it into place, and XOR it into // every other row to clear that column. Final rank = number of // pivots placed. let mut rank = 0usize; - for col in (0..128).rev() { - let mask = 1u128 << col; - let pivot = (rank..basis.len()).find(|&i| basis[i] & mask != 0); + for col in (0..192).rev() { + let limb = col / 64; + let mask = 1u64 << (col % 64); + let pivot = (rank..basis.len()).find(|&i| basis[i][limb] & mask != 0); if let Some(p) = pivot { basis.swap(rank, p); for i in 0..basis.len() { - if i != rank && basis[i] & mask != 0 { - basis[i] ^= basis[rank]; + if i != rank && basis[i][limb] & mask != 0 { + for limb in 0..3 { + basis[i][limb] ^= basis[rank][limb]; + } } } rank += 1; @@ -1386,7 +1249,7 @@ mod tests { } assert_eq!( rank, 7, - "friendly challenges must be F₂-linearly independent in F₁₂₈; \ + "friendly challenges must be F₂-linearly independent in F₁₉₂; \ zerocheck and Ligerito L0 soundness depend on it" ); } @@ -1394,14 +1257,14 @@ mod tests { /// Build the full `r` vector with the protocol-fixed constants in the /// small/medium slots. Only `r[k_skip + N_INNER..]` is the actual /// randomness fed to the optimized URM. - fn build_protocol_r(m: usize, outer: &[F128]) -> Vec { + fn build_protocol_r(m: usize, outer: &[F192]) -> Vec { assert_eq!(outer.len(), m - K_SKIP - N_INNER); - let mut r = vec![F128::ZERO; m]; + let mut r = vec![F192::ZERO; m]; // r[0..K_SKIP]: not used by either function — can be anything. - for (i, &small) in small_challenges_ghash().iter().enumerate() { + for (i, &small) in small_challenges().iter().enumerate() { r[K_SKIP + i] = small; } - for (i, &med) in medium_challenges_ghash().iter().enumerate() { + for (i, &med) in medium_challenges().iter().enumerate() { r[K_SKIP + 3 + i] = med; } for (i, &x) in outer.iter().enumerate() { @@ -1423,7 +1286,7 @@ mod tests { let a = rng.bits(1 << m); let b = rng.bits(1 << m); let c = rng.bits(1 << m); - let outer = rng.f128_vec(m - K_SKIP - N_INNER); + let outer = rng.ext_vec(m - K_SKIP - N_INNER); let r = build_protocol_r(m, &outer); let table = make_inv_table(); @@ -1439,7 +1302,7 @@ mod tests { let a = rng.bits(1 << m); let b = rng.bits(1 << m); let c = rng.bits(1 << m); - let outer = rng.f128_vec(m - K_SKIP - N_INNER); + let outer = rng.ext_vec(m - K_SKIP - N_INNER); let r = build_protocol_r(m, &outer); let table = make_inv_table(); @@ -1454,13 +1317,13 @@ mod tests { /// pre-scaling. #[test] fn matches_naive_with_c_s_factor() { - let c_s = c_s_f128(); + let c_s = c_s(); for &m in &[13usize, 14, 15] { let mut rng = Rng::new(100 + m as u64); let a = rng.bits(1 << m); let b = rng.bits(1 << m); let c = rng.bits(1 << m); - let outer = rng.f128_vec(m - K_SKIP - N_INNER); + let outer = rng.ext_vec(m - K_SKIP - N_INNER); let r = build_protocol_r(m, &outer); let table = make_inv_table(); @@ -1490,33 +1353,18 @@ mod tests { fn small_and_medium_challenges_sanity() { // Reach into the constants and verify their structural identities. // Medium: β_i · (1 + γ^{2^{i-1}}) == γ^{2^{i-1}}. - let med = medium_challenges_ghash(); - let powers = [1u64 << 1, 1u64 << 2, 1u64 << 4, 1u64 << 8]; - for (i, &p) in powers.iter().enumerate() { - let g = F128 { lo: p, hi: 0 }; - assert_eq!(med[i] * (F128::ONE + g), g, "β_{i} identity"); + let med = medium_challenges(); + let g1 = medium_generator(); + let powers = [g1, g1.square(), g1.square().square(), g1.square().square().square()]; + for (i, &g) in powers.iter().enumerate() { + assert_eq!(med[i] * (F192::ONE + g), g, "β_{i} identity"); } // D · D_inv == 1. let d_inv_val = d_inv(); - let g1 = F128 { - lo: 1u64 << 1, - hi: 0, - }; - let g2 = F128 { - lo: 1u64 << 2, - hi: 0, - }; - let g4 = F128 { - lo: 1u64 << 4, - hi: 0, - }; - let g8 = F128 { - lo: 1u64 << 8, - hi: 0, - }; - let d = (F128::ONE + g1) * (F128::ONE + g2) * (F128::ONE + g4) * (F128::ONE + g8); - assert_eq!(d * d_inv_val, F128::ONE); + let [g1, g2, g4, g8] = powers; + let d = (F192::ONE + g1) * (F192::ONE + g2) * (F192::ONE + g4) * (F192::ONE + g8); + assert_eq!(d * d_inv_val, F192::ONE); } #[test] @@ -1524,7 +1372,7 @@ mod tests { use crate::zerocheck::univariate_skip::pack_bits; // At small m the parallel overhead dominates, but the *output* must - // still match the serial version bit-for-bit. F128 XOR-sum reduction + // still match the serial version bit-for-bit. F192 XOR-sum reduction // is commutative + associative, so any thread-scheduling order yields // the same result. for &m in &[13usize, 14, 15] { @@ -1532,18 +1380,15 @@ mod tests { let a = rng.bits(1 << m); let b = rng.bits(1 << m); let c = rng.bits(1 << m); - let outer = rng.f128_vec(m - K_SKIP - N_INNER); + let outer = rng.ext_vec(m - K_SKIP - N_INNER); let r = build_protocol_r(m, &outer); let table = make_inv_table(); let a_p = pack_bits(&a); let b_p = pack_bits(&b); let c_p = pack_bits(&c); - let (par_ab, par_c) = - round1_shift_reduce_extract_c_packed(&a_p, &b_p, &c_p, m, K_SKIP, &r, &table); - let (ser_ab, ser_c) = round1_shift_reduce_extract_c_packed_serial( - &a_p, &b_p, &c_p, m, K_SKIP, &r, &table, - ); + let (par_ab, par_c) = round1_shift_reduce_extract_c_packed(&a_p, &b_p, &c_p, m, K_SKIP, &r, &table); + let (ser_ab, ser_c) = round1_shift_reduce_extract_c_packed_serial(&a_p, &b_p, &c_p, m, K_SKIP, &r, &table); assert_eq!(par_ab, ser_ab, "parallel AB ≠ serial AB at m={m}"); assert_eq!(par_c, ser_c, "parallel C ≠ serial C at m={m}"); @@ -1599,22 +1444,20 @@ mod tests { } } - let outer = rng.f128_vec(m - K_SKIP - N_INNER); + let outer = rng.ext_vec(m - K_SKIP - N_INNER); let r = build_protocol_r(m, &outer); let table = make_inv_table(); let a_p = pack_bits(&a); let b_p = pack_bits(&b); let c_p = pack_bits(&c); - let (dense_ab, dense_c) = - round1_shift_reduce_extract_c_packed(&a_p, &b_p, &c_p, m, K_SKIP, &r, &table); + let (dense_ab, dense_c) = round1_shift_reduce_extract_c_packed(&a_p, &b_p, &c_p, m, K_SKIP, &r, &table); let padding = PaddingSpec { k_log, useful_bits_per_block: useful_bits, }; - let (padded_ab, padded_c) = round1_shift_reduce_extract_c_packed_padded( - &a_p, &b_p, &c_p, m, K_SKIP, &r, &table, &padding, - ); + let (padded_ab, padded_c) = + round1_shift_reduce_extract_c_packed_padded(&a_p, &b_p, &c_p, m, K_SKIP, &r, &table, &padding); assert_eq!( dense_ab, padded_ab, @@ -1627,7 +1470,6 @@ mod tests { } } - #[cfg(target_arch = "aarch64")] #[test] fn neon_fused_inner_matches_scalar_inner() { @@ -1661,14 +1503,7 @@ mod tests { &mut a_col, &mut b_col, ); - shift_reduce_inner_ab_fused_neon( - &a_packed, - &b_packed, - &table, - chunk_byte_base, - b_med, - &mut out_fused, - ); + shift_reduce_inner_ab_fused_neon(&a_packed, &b_packed, &table, chunk_byte_base, b_med, &mut out_fused); assert_eq!( out_scalar, out_fused, "fused-neon disagrees with scalar at (base={chunk_byte_base}, b_med={b_med})" @@ -1733,13 +1568,13 @@ mod tests { fn convert_table_structure() { // convert[b][v] == γ^b · φ_8(v); check at a handful of (b, v). let t = convert_table(); - let mut g_pow = F128::ONE; + let mut g_pow = F192::ONE; for b in 0..16 { for &v in &[0u8, 1, 0x57, 0xFF] { let expected = g_pow * PHI_8_TABLE[v as usize]; assert_eq!(t[b * 256 + v as usize], expected, "b={b}, v={v}"); } - g_pow = mul_by_x(g_pow); + g_pow *= medium_generator(); } } @@ -1755,21 +1590,21 @@ mod tests { let a = pack_bits(&rng.bits(1 << m)); let b = pack_bits(&rng.bits(1 << m)); let c = pack_bits(&rng.bits(1 << m)); - let mut r = vec![F128::ZERO; m]; + let mut r = vec![F192::ZERO; m]; // Friendly inner constants must match the optimization's - // expectations: 3 small + 4 medium ghash. + // expectations: 3 small + 4 medium coordinates. for i in 0..3 { r[K_SKIP + i] = phi8(F8(SMALL_CHAL_F8[i])); } - let medium = crate::zerocheck::univariate_skip_optimized::medium_challenges_ghash(); + let medium = crate::zerocheck::univariate_skip_optimized::medium_challenges(); for i in 0..4 { r[K_SKIP + 3 + i] = medium[i]; } for i in 0..K_SKIP { - r[i] = rng.f128(); + r[i] = rng.ext(); } for i in (K_SKIP + N_INNER)..m { - r[i] = rng.f128(); + r[i] = rng.ext(); } let inv_table = { @@ -1779,21 +1614,19 @@ mod tests { }; // Scalar oracle (canonical s_hat_v_c). - let (_, _, oracle_s_hat_v) = - round1_extract_c_packed_with_s_hat_v(&a, &b, &c, m, K_SKIP, &r, &inv_table); + let (_, _, oracle_s_hat_v) = round1_extract_c_packed_with_s_hat_v(&a, &b, &c, m, K_SKIP, &r, &inv_table); // System under test. - let (_, _, got_s_hat_v) = - round1_shift_reduce_extract_c_packed_padded_with_s_hat_v( - &a, - &b, - &c, - m, - K_SKIP, - &r, - &inv_table, - &PaddingSpec::dense(m), - ); + let (_, _, got_s_hat_v) = round1_shift_reduce_extract_c_packed_padded_with_s_hat_v( + &a, + &b, + &c, + m, + K_SKIP, + &r, + &inv_table, + &PaddingSpec::dense(m), + ); assert_eq!(got_s_hat_v.len(), 2 * ELL, "s_hat_v length at m={m}"); assert_eq!( diff --git a/crates/flock/tests/blake3_batch.rs b/crates/flock/tests/blake3_batch.rs index 1f0e3446..483bf1e5 100644 --- a/crates/flock/tests/blake3_batch.rs +++ b/crates/flock/tests/blake3_batch.rs @@ -1,32 +1,39 @@ //! Standalone batch BLAKE3 proving, isolated from the VM. //! -//! This exercises ONLY the flock "BLAKE3 stuff" over `N` compressions — -//! witness-gen → commit → [`Blake3Setup::prove_validity_stacked`] (zerocheck + -//! lincheck reduction, then the stacked Ligerito open) → verify — with no -//! leanVM execute / bus / constraints around it. Much faster to iterate on -//! than the full xmss benchmark when optimizing the flock reduction / PCS. +//! This exercises only Flock's BLAKE3 path over `N` compressions: witness +//! generation, F64 commitment, zerocheck + lincheck reduction, the stacked +//! ring-switch/Ligerito opening, and verification. Circuit construction is +//! outside the timed region, matching the VM's warmed-setup convention. //! -//! `Blake3Setup::new` (circuit construction, one-time preprocessing -//! independent of the witness) runs OUTSIDE the timed region, matching how -//! `cpu::prove` warms it off the critical path. The Ligerito configuration is -//! the one leanVM-b commits with, so the numbers are comparable to the -//! `[open]` / `commit` stages of the xmss benchmark. -//! -//! Run (N = number of compressions; the xmss n=820 workload is ~120k = 181 + 146·820): +//! Run with the XMSS-sized workload: //! ```text -//! RAYON_NUM_THREADS=11 FLOCK_N=131072 cargo test --release -p flock --test blake3_batch -- --nocapture +//! RAYON_NUM_THREADS=11 FLOCK_N_LOG=17 cargo test --release -p flock --test blake3_batch -- --ignored --nocapture //! ``` use std::time::Instant; +use fiat_shamir::transcript::{ProverState, VerifierState}; use flock::blake3::{ - Blake3Setup, Compression, K_LOG, generate_witness_with_ab_packed_and_lincheck, + Blake3Setup, Compression, K_LOG, PackedWitnessClaims, generate_witness_with_ab_packed_and_lincheck, min_n_blocks_log, pinned_compression, }; -use pcs::{Commitment, LOG_PACKING, PcsParams, ProverState, VerifierState}; +use flock::proof::ZClaim; +use pcs::ligerito::{INITIAL_FOLDING_FACTOR, LOG_INV_RATE_0}; +use pcs::ligerito::{commit, configs_for}; +use pcs::pack::{LOG_PACKING, PACKING_WIDTH}; +use pcs::stack_open::{ + RingSwitchClaim, RingSwitchOpen, RingSwitchVerify, open_batch_mixed_ligerito_stacked, + verify_opening_batch_mixed_ligerito_stacked, +}; +use primitives::multilinear::lagrange_weights_naive; +use primitives::{ + field::{F64, F192}, + pretty_integer, +}; -/// Tiny deterministic xorshift RNG — no `rand` dep, reproducible inputs. +/// Tiny deterministic xorshift RNG: reproducible inputs without another dep. struct Rng(u64); + impl Rng { fn next_u32(&mut self) -> u32 { let mut x = self.0; @@ -38,91 +45,130 @@ impl Rng { } } +/// Split the two K coefficients of each packed tower element. Flock's fused +/// witness generator uses 128-bit containers; the PCS commits 64 bits/word. +fn flatten_packed(packed: &[F192]) -> Vec { + let mut out = Vec::with_capacity(2 * packed.len()); + for value in packed { + out.push(F64(value.c0)); + out.push(F64(value.c1)); + } + out +} + +/// Adapt one Flock evaluation claim to the 64-bit ring switch. Lincheck +/// captures its 64 slices directly; the fused zerocheck kernel captures two +/// banks around the first suffix coordinate, which are folded here. +fn ring_claim(z: &ZClaim, captured: Option<&[F192]>, qpkd_vars: usize) -> RingSwitchClaim { + let mut suffix_point = z.point.x_inner_rest.clone(); + suffix_point.extend_from_slice(&z.point.x_outer); + assert_eq!(suffix_point.len(), qpkd_vars); + + let s_hat_v = captured.and_then(|s| match s.len() { + PACKING_WIDTH => Some(s.to_vec()), + n if n == 2 * PACKING_WIDTH && !z.point.x_inner_rest.is_empty() => { + let c = z.point.x_inner_rest[0]; + Some( + (0..PACKING_WIDTH) + .map(|i| (F192::ONE + c) * s[i] + c * s[i + PACKING_WIDTH]) + .collect(), + ) + } + _ => None, + }); + + RingSwitchClaim { + prefix_weights: lagrange_weights_naive(LOG_PACKING, z.point.z_skip), + suffix_point, + value: z.value, + s_hat_v, + } +} + +fn prover_ring(reduced: &PackedWitnessClaims, qpkd_vars: usize) -> RingSwitchOpen { + RingSwitchOpen { + offset: 0, + qpkd_vars, + claims: vec![ + ring_claim(&reduced.ab.claim, reduced.ab.s_hat_v.as_deref(), qpkd_vars), + ring_claim(&reduced.c.claim, reduced.c.s_hat_v.as_deref(), qpkd_vars), + ], + } +} + +fn verifier_ring(ab: &ZClaim, c: &ZClaim, qpkd_vars: usize) -> RingSwitchVerify { + RingSwitchVerify { + offset: 0, + qpkd_vars, + claims: vec![ring_claim(ab, None, qpkd_vars), ring_claim(c, None, qpkd_vars)], + } +} + #[test] +#[ignore = "manual release benchmark; needs a large-stack worker and substantial memory"] fn blake3_batch_prove_verify() { - // Number of compressions to prove. Default is quick-but-meaningful; set - // FLOCK_N=131072 to mirror the xmss n=820 BLAKE3 workload (~2^17). - let n: usize = std::env::var("FLOCK_N").ok().and_then(|s| s.parse().ok()).unwrap_or(8192); - assert!(n >= 1, "FLOCK_N must be ≥ 1"); + // The XMSS n=820 workload executes about 2^17 BLAKE3 compressions. + let requested_n_log: usize = std::env::var("FLOCK_N_LOG") + .ok() + .map(|s| s.parse().expect("FLOCK_N_LOG must be an integer")) + .unwrap_or(13); + let n = 1usize + .checked_shl(requested_n_log as u32) + .expect("FLOCK_N_LOG exceeds the platform usize width"); let n_log = min_n_blocks_log(n); - // Committed q_pkd log-size; the Secure ladder needs some room. let mu = K_LOG + n_log - LOG_PACKING; - assert!(mu >= 15, "FLOCK_N too small — need ≥ 2^8 compressions (mu ≥ 15)"); + assert!( + mu >= 15, + "FLOCK_N_LOG too small: need a committed witness with mu >= 15" + ); - // Deterministic sample compressions (arbitrary messages; the prover does - // the same work regardless of values). cv/counter/blen/flags are pinned by - // the circuit's constant rows. let mut rng = Rng(0x9E37_79B9_7F4A_7C15 ^ n as u64); let blocks: Vec = (0..n) - .map(|_| { - let m: [u32; 16] = std::array::from_fn(|_| rng.next_u32()); - pinned_compression(m) - }) + .map(|_| pinned_compression(std::array::from_fn(|_| rng.next_u32()))) .collect(); - // Circuit construction (one-time preprocessing) — OUTSIDE the timed region, - // like cpu::prove's background warm. Warms the CSC lincheck circuit and the - // prover scratch. let t = Instant::now(); let setup = Blake3Setup::new(n); let setup_ms = t.elapsed().as_secs_f64() * 1e3; - // The committed stack is q_pkd itself (offset 0, no other columns). let t = Instant::now(); - let (q_pkd, a_packed, b_packed, z_lincheck) = - generate_witness_with_ab_packed_and_lincheck(&blocks, n_log); + let (z_packed, a_packed, b_packed, z_lincheck) = generate_witness_with_ab_packed_and_lincheck(&blocks, n_log); + let q_pkd = flatten_packed(&z_packed); let witness_ms = t.elapsed().as_secs_f64() * 1e3; assert_eq!(q_pkd.len(), 1 << mu); - let params = PcsParams { - m: mu + LOG_PACKING, - log_inv_rate: pcs::ligerito::LOG_INV_RATE_0, - log_batch_size: pcs::ligerito::INITIAL_FOLDING_FATOR, - }; - - let mut ps = ProverState::new(b"flock-blake3-batch", &[]); + let (prover_config, verifier_config) = configs_for(mu).expect("Ligerito configuration"); + let mut ps = ProverState::<()>::new(b"flock-blake3-batch", &[]); let t_prove = Instant::now(); + let t = Instant::now(); - let (commitment, prover_data) = pcs::commit(&q_pkd, ¶ms); + let (commitment, prover_data) = commit(&q_pkd, INITIAL_FOLDING_FACTOR, LOG_INV_RATE_0); ps.add_scalars(&pcs::merkle::hash_to_scalars(&commitment.root)); let commit_ms = t.elapsed().as_secs_f64() * 1e3; - // Reduction (zerocheck + lincheck) + the one stacked Ligerito open. let t = Instant::now(); - let reduced = setup.prove_reduction_precomputed( - &q_pkd, - &a_packed, - &b_packed, - &z_lincheck, - &mut ps, - ); - drop((a_packed, b_packed, z_lincheck)); - let proof = setup.discharge_reduction_stacked( - &q_pkd, - &reduced, - &q_pkd, - 0, - &prover_data, - &commitment, - &[], - &mut ps, - ); + let reduced = setup.prove_reduction_precomputed(&z_packed, &a_packed, &b_packed, &z_lincheck, &mut ps); + drop((z_packed, a_packed, b_packed, z_lincheck)); + let ring = prover_ring(&reduced, mu); + let opening = open_batch_mixed_ligerito_stacked(ps.sponge_mut(), &q_pkd, &prover_data, &prover_config, &[], &ring); let open_ms = t.elapsed().as_secs_f64() * 1e3; let prove_s = t_prove.elapsed().as_secs_f64(); - let bundle = ps.into_proof(); + let transcript = ps.into_proof(); - // Verify (correctness gate + a verify timing for reference). let t = Instant::now(); - let mut vs = VerifierState::new(b"flock-blake3-batch", &bundle, &[]); - let root = pcs::merkle::scalars_to_hash(&vs.next_scalars(2).expect("root scalars")); - let commitment_v = Commitment { root, params }; - setup - .verify_validity_stacked(&commitment_v, 0, &[], &proof, &mut vs) - .expect("flock BLAKE3 batch proof must verify"); - vs.finish().expect("stream fully consumed"); + let mut vs = VerifierState::<()>::new(b"flock-blake3-batch", &transcript, &[]); + let root = pcs::merkle::scalars_to_hash(&vs.next_scalars(2).expect("commitment root")); + let replay = setup.verify_reduction(&mut vs).expect("Flock reduction verifies"); + let ring = verifier_ring(&replay.ab, &replay.c, mu); + verify_opening_batch_mixed_ligerito_stacked(vs.sponge_mut(), &verifier_config, mu, &root, &[], &ring, &opening) + .expect("stacked PCS opening verifies"); + vs.finish().expect("transcript fully consumed"); let verify_ms = t.elapsed().as_secs_f64() * 1e3; - println!("\nflock BLAKE3 batch proving, {n} compressions (2^{n_log} slots)"); + println!( + "\nFlock BLAKE3 batch proving, {} compressions (2^{n_log} slots)", + pretty_integer(n) + ); println!(" setup (preprocessing, excluded) : {setup_ms:>8.1} ms"); println!(" witness-gen : {witness_ms:>8.1} ms"); println!(" commit : {commit_ms:>8.1} ms"); @@ -130,6 +176,13 @@ fn blake3_batch_prove_verify() { println!(" ------------------------------------------"); println!(" prove TOTAL (witness excluded) : {:>8.1} ms", prove_s * 1e3); println!(" verify : {verify_ms:>8.1} ms"); - println!(" throughput : {:>10.0} compressions/s", n as f64 / prove_s); - println!(" (~{:.0} XMSS/s equiv @ 146 compressions/sig)", n as f64 / prove_s / 146.0); + let compressions_per_second = (n as f64 / prove_s).round() as u64; + println!( + " throughput : {:>14} compressions/s", + pretty_integer(compressions_per_second) + ); + println!( + " (~{:.1} XMSS/s equivalent at 146 compressions/signature)", + n as f64 / prove_s / 146.0 + ); } diff --git a/crates/lean_compiler/Cargo.toml b/crates/lean_compiler/Cargo.toml index 6fc79e24..dbcc8f1e 100644 --- a/crates/lean_compiler/Cargo.toml +++ b/crates/lean_compiler/Cargo.toml @@ -12,4 +12,5 @@ lean_vm.workspace = true [dev-dependencies] blake3.workspace = true +rand.workspace = true bincode.workspace = true diff --git a/crates/lean_compiler/src/ast.rs b/crates/lean_compiler/src/ast.rs index 466047ec..a1ce53e0 100644 --- a/crates/lean_compiler/src/ast.rs +++ b/crates/lean_compiler/src/ast.rs @@ -1,12 +1,12 @@ //! The surface AST produced by the parser: expressions, statements, functions. +use primitives::field::F192; + /// An expression. Arithmetic is the field's own: `+` is `XOR`, `*` is `MUL`. #[derive(Clone, Debug)] pub enum Expr { - /// Integer / field literal: a `u128` taken as the field element's 128 bits, - /// `F128::new(n_lo, n_hi)`. Small values behave like integers (`5` is - /// `F128::new(5, 0)`); a full 128-bit value names an arbitrary field - /// constant (e.g. a Fibonacci result computed in the exponent). + /// Integer / field literal: the source syntax provides a raw 128-bit value, + /// embedded into the low two limbs of the 192-bit tower element (`c2 = 0`). Lit(u128), /// The generator `g` — written `GEN` in source. A logical index `i` is /// carried "in the exponent" as `gⁱ`, so `GEN` is the unit step and @@ -47,9 +47,9 @@ pub enum Expr { /// `a · b⁻¹`. Lowered to one `MUL` whose quotient operand is unset, so the /// write-once back-solve fills it with `a · b⁻¹` and the `MUL` constraint /// pins `quotient · b == a` (§range-check trick). No hint: the inverse is - /// nondeterministic but the constraint binds it. `b == 0` is rejected - /// (unless `a == 0` too, the undefined `0/0`); `1 / b` therefore also - /// enforces `b != 0`. Distinct from the compile-time `//` ([`Expr::Div`]). + /// nondeterministic but the constraint binds it. `b == 0` is rejected, + /// including `0 / 0`; `1 / b` therefore also enforces `b != 0`. Distinct + /// from the compile-time `//` ([`Expr::Div`]). FieldDiv(Box, Box), /// Single-return function call in expression position. Call(String, Vec), @@ -64,8 +64,8 @@ pub enum Expr { HeapBufDyn(Box), /// `StackBuf(n)` — allocate `n` *consecutive* frame (stack) cells, bound as a /// stack value. Its cells `sa[0..n]` are written/read directly (no heap deref), - /// and a size-2 `StackBuf` is a valid `blake3` operand (the two 128-bit words - /// of a 256-bit value live in the two consecutive cells). See [`FnLower`]. + /// and a size-2 `StackBuf` is a valid `blake3` operand (the four 64-bit hash + /// words live as two lanes in each of two consecutive 128-bit cells). StackBuf(u64), /// `arr[idx]` — read a cell. For a heap `arr` (a pointer): `m[arr·idx]` (idx a /// g-power). For a [`Expr::StackBuf`]: the frame cell `base + idx` (idx a @@ -97,15 +97,15 @@ pub enum Stmt { /// `assert a != b` — a proof-enforced inequality. Lowers to a conditional /// `JUMP` on `a + b`: when the sides differ (nonzero) execution skips to the /// continuation; when they are equal it falls through to a jump to the - /// poison pc `g^-1` ([`KVal::Poison`]), which no valid trace can continue - /// past. See [`FnLower::lower_assert_ne`]. No prover hint (unlike the + /// poison pc `g^-1` (`KVal::Poison`), which no valid trace can continue + /// past. See `FnLower::lower_assert_ne`. No prover hint (unlike the /// `(a-b)·inv == 1` idiom it replaces). AssertNe(Expr, Expr), /// `assert log X < log Y` (also `assert log X < k` with an integer /// exponent) — a *range check in the exponent*: with `X = g^x`, proves /// `x < k`, i.e. `X ∈ {g^0, g^1, …, g^{k-1}}`. The bound `Y = g^k` is a /// compile-time power of `GEN` with `1 ≤ k ≤ 2^MIN_LOG_MEM`; see - /// [`FnLower::lower_assert_lt`] for the 3-cycle gadget (leanVM's DEREF + /// `FnLower::lower_assert_lt` for the 3-cycle gadget (leanVM's DEREF /// range-check trick, transported to g-powers). AssertLt(Expr, u64), /// `f(args)` as a statement (returns discarded). @@ -126,7 +126,7 @@ pub enum Stmt { /// One conditional `JUMP` on the XOR of the two sides; bindings made /// inside a branch are local to it — branches communicate through /// write-once memory (only one branch executes, so both may write the - /// same cell). See [`FnLower::lower_if`]. + /// same cell). See `FnLower::lower_if`. If { eq: bool, lhs: Expr, @@ -139,7 +139,7 @@ pub enum Stmt { /// runs case `j`). Dispatched through a trampoline table in the bytecode /// (doc §ISA programming / Match statements); the scrutinee must be known /// to lie in `[0, n)` — range-check a hinted value first. Case bodies are - /// branch-local, like [`Stmt::If`] branches. See [`FnLower::lower_match`]. + /// branch-local, like [`Stmt::If`] branches. See `FnLower::lower_match`. Match { x: Expr, cases: Vec> }, /// `names = match_range(log(x), range(a, b), lambda i: expr, …)` — a /// [`Stmt::Match`] with generated arms (leanVM's `match_range`): arm `j` @@ -199,6 +199,27 @@ pub enum ForBound { Runtime(Expr), } +/// Compile-time representation of one source-level return value. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ReturnShape { + /// One ordinary field element or address cell. Heap buffers use this shape: + /// allocation happens in the callee and only their pointer is returned. + Scalar, + /// A compile-time-sized run of consecutive frame cells. + StackBuf(u32), +} + +impl ReturnShape { + /// Number of physical call-frame return cells occupied by this source-level + /// return value. + pub(crate) fn cells(self) -> u32 { + match self { + Self::StackBuf(n) => n, + Self::Scalar => 1, + } + } +} + /// A function definition. `main` is the entry point. #[derive(Clone, Debug)] pub struct Func { @@ -208,9 +229,13 @@ pub struct Func { /// a `Const` parameter is a *template*: it is never lowered itself — each /// call site with a distinct constant tuple queues a monomorphized copy /// with the parameter substituted by its literal (see - /// [`FnLower::specialize`]). + /// `FnLower::specialize`). pub const_params: Vec, + /// Number of source-level return values (tuple arity). pub n_ret: usize, + /// Compile-time shape of each source-level return value. Stack buffers use + /// multiple physical ABI cells; everything else uses one cell. + pub return_shapes: Vec, pub body: Vec, /// `@inline` decorator: expand this function at each call site instead of /// emitting a real call — no frame, no argument/return plumbing (the @@ -226,10 +251,10 @@ pub struct Func { pub struct Ast { pub funcs: Vec, /// Top-level constant arrays `NAME = [a, b, c]` (declaration order). Each - /// element is a `u128` (a field value `F128::new(lo,hi)` where used as a + /// element is a `u128` (a field value `extension-field::new(lo,hi)` where used as a /// value, or a small integer where used as a compile-time index / bound / /// `unroll` count). Indexed `NAME[i]` and measured `len(NAME)` at compile /// time only (`i` a literal / constant / `unroll` var). Not textually /// substituted (unlike scalar constants) — resolved at lowering. - pub const_arrays: Vec<(String, Vec)>, + pub const_arrays: Vec<(String, Vec)>, } diff --git a/crates/lean_compiler/src/cse.rs b/crates/lean_compiler/src/cse.rs new file mode 100644 index 00000000..1ea5291a --- /dev/null +++ b/crates/lean_compiler/src/cse.rs @@ -0,0 +1,252 @@ +//! Value numbering over one function's lowered code. +//! +//! leanVM's frame cells are **write-once**: a cell holds one value for the whole +//! run, so "which value does `fp[k]` hold" needs no dataflow analysis — the +//! defining instruction is the only one there will ever be. That makes common +//! subexpression elimination on the pure operations (`SET` of a constant, `XOR`, +//! `MUL`) a local rewrite: if an identical `(op, operands)` was already computed +//! in this basic block, later reads can use the earlier cell and the duplicate +//! instruction goes away. +//! +//! Why the lowerer leaves duplicates behind: it emits each expression +//! independently (an address `ptr·index` recomputed per access, a loop counter +//! advanced once for the body and once for the recursive call, the constant `1` +//! materialized per use inside a fresh frame). Roughly 5% of the recursion +//! guest's instructions were exact repeats. +//! +//! The three rules that keep this sound: +//! 1. **Only pure ops are eliminated.** `DEREF` unifies two memory cells (and +//! bumps the bus read counts), `BLAKE3`/`PACK64X2` carry bus effects, `JUMP` +//! is control flow — all are left alone. They still get their operands +//! rewritten. +//! 2. **Only single-write targets.** An instruction is a candidate only if its +//! destination cell is written exactly once in the function, counting hint +//! writes. That protects the assert idiom (`XOR fp[t] = a^b` followed by +//! `SET fp[t] = 0`, which panics as a write-once conflict when `a != b`): +//! both instructions write `fp[t]`, so neither is touched and neither is +//! offered as a replacement. It also means every cell in the substitution map +//! is written by exactly one (dropped) instruction, so every *other* +//! occurrence of it is a read — which is why operand rewriting can be blanket. +//! 3. **Locals only.** A candidate's destination must be a local temporary, not +//! an argument or return slot: the caller writes the arguments and reads the +//! returns straight out of this frame, so a write there is live even though no +//! instruction in this function reads it. +//! 4. **Block-local.** The value map is cleared at every jump target and after +//! every `JUMP`, so a duplicate is only ever folded into an earlier +//! computation from the same straight-line run. Folding across a branch could +//! redirect a read to a cell that the taken path never wrote, and an unwritten +//! cell is a prover-chosen free variable — a soundness hole, not just a bug. + +use super::ir::{Hint, KVal, LInstr, LOp, Off}; +use std::collections::HashMap; + +/// A pure operation's identity: the opcode plus its operand cells (commutative +/// operands sorted, so `a*b` and `b*a` share an entry), or a constant's bits. +#[derive(PartialEq, Eq, Hash)] +enum Key { + Xor(Off, Off), + Mul(Off, Off), + Const([u64; 3]), +} + +/// Rewrite `code` in place, dropping redundant pure instructions. `abi_end` is +/// the first purely-local frame cell (see [`super::ir::Lowered::abi_end`]); +/// writes below it are visible to the caller and are never eliminated. Returns +/// the number of instructions dropped (for the `DBG_CSE` report). +pub(crate) fn cse(code: &mut Vec, abi_end: Off) -> usize { + let writes = write_counts(code); + let labels = label_targets(code); + + let mut subst: HashMap = HashMap::new(); + let mut seen: HashMap = HashMap::new(); + let mut drop = vec![false; code.len()]; + let mut ends_block = false; + + for (i, ins) in code.iter_mut().enumerate() { + // A jump target starts a new block; so does the instruction after a jump. + if ends_block || labels.contains(&(i as u32)) { + seen.clear(); + } + ends_block = matches!(ins.op, LOp::Jump { .. }); + + rewrite_reads(ins, &subst); + + // Candidates: a pure op whose destination is a local written exactly once. + let (key, dst) = match &ins.op { + LOp::Xor { a, b, c } => (Key::Xor(*a.min(b), *a.max(b)), *c), + LOp::Mul { a, b, c } => (Key::Mul(*a.min(b), *a.max(b)), *c), + LOp::Set { o, k: KVal::Const(k) } => (Key::Const([k.c0, k.c1, k.c2]), *o), + _ => continue, + }; + // A write to an argument or return slot is read by the CALLER, which + // this pass cannot see: `walk` returning a flag as `SET fp[6] = 0` looks + // dead here but is the function's result. + if dst < abi_end || writes.get(&dst).copied().unwrap_or(0) != 1 { + continue; + } + // Hints execute immediately before their instruction. Keeping the + // instruction is the only generally safe way to preserve that control- + // flow position: moving a branch-local hint to the next textual + // instruction could move it past the branch join. + if !ins.hints.is_empty() { + seen.entry(key).or_insert(dst); + continue; + } + match seen.get(&key) { + // The value is already in `canon`: point later reads there and drop + // this instruction. + Some(&canon) => { + subst.insert(dst, canon); + drop[i] = true; + } + None => { + seen.insert(key, dst); + } + } + } + + let dropped = drop.iter().filter(|d| **d).count(); + if dropped > 0 { + compact(code, &drop); + } + dropped +} + +/// How many times each frame cell is written, instructions and hints together. +/// Over-counting is safe (it only forgoes an optimization), so ambiguous cases +/// count as writes. +fn write_counts(code: &[LInstr]) -> HashMap { + let mut w: HashMap = HashMap::new(); + let mut bump = |o: Off| *w.entry(o).or_default() += 1; + for ins in code { + match &ins.op { + LOp::Set { o, .. } => bump(*o), + LOp::Xor { c, .. } | LOp::Mul { c, .. } | LOp::Pack64x2 { c, .. } => bump(*c), + // A `Cell` deref unifies `m[p·g^beta]` with `fp[gamma]`, which writes + // `fp[gamma]` when it acts as a load. The `Pc`/`Fp` modes take their + // source from the machine state and leave `gamma` unused. + LOp::Deref { gamma, mode, .. } => { + if matches!(mode, super::DerefMode::Cell) { + bump(*gamma); + } + } + // The 32-byte digest lands in two consecutive cells. + LOp::Blake3 { c, .. } => { + bump(*c); + bump(*c + 1); + } + LOp::Jump { .. } => {} + } + for h in &ins.hints { + match h { + Hint::AllocFrame { ptr, .. } + | Hint::AllocFrameMax { ptr, .. } + | Hint::AllocBuffer { ptr, .. } + | Hint::AllocBufferDyn { ptr, .. } => bump(*ptr), + Hint::WitnessStack { base, len, .. } | Hint::FieldLimbs { base, len, .. } => { + for k in 0..*len { + bump(*base + k); + } + } + Hint::Log2Ceil { dst, .. } => bump(*dst), + // These write HEAP cells through a pointer, not frame cells. + Hint::WitnessHeap { .. } + | Hint::BitDecompose { .. } + | Hint::BitDecomposeExp { .. } + | Hint::Print { .. } => {} + } + } + } + w +} + +/// Instruction indices that are jump targets ([`KVal::Local`] destinations). +fn label_targets(code: &[LInstr]) -> std::collections::HashSet { + code.iter() + .filter_map(|ins| match &ins.op { + LOp::Set { k: KVal::Local(i), .. } => Some(*i), + _ => None, + }) + .collect() +} + +/// Point every operand read at its canonical cell. Blanket-rewriting every +/// `Off` field is safe: a cell in `subst` is written exactly once, by the +/// instruction that was dropped, so no remaining field that names it is a write +/// (see rule 2 in the module docs). +fn rewrite_reads(ins: &mut LInstr, subst: &HashMap) { + if subst.is_empty() { + return; + } + let map = |o: &mut Off| { + if let Some(&c) = subst.get(o) { + *o = c; + } + }; + match &mut ins.op { + LOp::Set { .. } => {} + LOp::Xor { a, b, .. } | LOp::Mul { a, b, .. } | LOp::Pack64x2 { a, b, .. } => { + map(a); + map(b); + } + LOp::Deref { alpha, gamma, mode, .. } => { + map(alpha); + if matches!(mode, super::DerefMode::Cell) { + map(gamma); + } + } + LOp::Jump { oc, od, of } => { + map(oc); + map(od); + map(of); + } + LOp::Blake3 { ins: chunks, cv, .. } => { + for chunk in chunks.iter_mut() { + map(chunk); + } + map(cv); + } + } + for h in &mut ins.hints { + match h { + Hint::AllocBufferDyn { size, .. } => map(size), + Hint::WitnessHeap { ptr, .. } => map(ptr), + Hint::Log2Ceil { bits_ptr, .. } => map(bits_ptr), + Hint::BitDecompose { value, bits_ptr, .. } | Hint::BitDecomposeExp { value, bits_ptr, .. } => { + map(value); + map(bits_ptr); + } + Hint::FieldLimbs { value, .. } => map(value), + Hint::Print { cell, .. } => map(cell), + Hint::AllocFrame { .. } | Hint::AllocFrameMax { .. } | Hint::AllocBuffer { .. } => {} + Hint::WitnessStack { .. } => {} + } + } +} + +/// Remove the dropped instructions and renumber the intra-function jump targets +/// ([`KVal::Local`]) to the new indices. A label is a block start and a block's +/// first instruction is never dropped (the value map is empty there), so every +/// target survives; the `saturating` fallback keeps the mapping total anyway. +fn compact(code: &mut Vec, drop: &[bool]) { + let mut new_index = Vec::with_capacity(code.len() + 1); + let mut next = 0u32; + for d in drop { + new_index.push(next); + if !*d { + next += 1; + } + } + new_index.push(next); + let mut kept: Vec = code + .drain(..) + .zip(drop) + .filter_map(|(ins, d)| (!*d).then_some(ins)) + .collect(); + for ins in &mut kept { + if let LOp::Set { k: KVal::Local(i), .. } = &mut ins.op { + *i = new_index[*i as usize]; + } + } + *code = kept; +} diff --git a/crates/lean_compiler/src/ir.rs b/crates/lean_compiler/src/ir.rs index 7ba6d1f7..da6f84e1 100644 --- a/crates/lean_compiler/src/ir.rs +++ b/crates/lean_compiler/src/ir.rs @@ -8,7 +8,9 @@ pub(crate) type Off = u32; /// once entry program counters are fixed. #[derive(Clone, Debug)] pub(crate) enum KVal { - Const(F128), + /// A 192-bit machine-word constant. Source literals fill only c0/c1, while + /// compiler-generated constants may use the full field. + Const(F192), Entry(String), /// The halt sentinel pc `g^{B-1}` (last bytecode slot), fixed once the /// padded bytecode size `B` is known. `main` jumps here to terminate. @@ -60,13 +62,21 @@ pub(crate) enum LOp { od: Off, of: Off, }, - /// `BLAKE3`: the four input words `ins` are addressed independently (`fp+ins[i]`); - /// the 32-bit output `c = (c, c+1)` occupies two CONSECUTIVE frame cells. + /// Pack two K-valued source cells into one canonical 128-bit cell. The VM + /// memory bus enforces that both inputs have zero extension limbs. + Pack64x2 { + a: Off, + b: Off, + c: Off, + }, + /// `BLAKE3`: the four 128-bit input chunks `ins` are addressed independently, + /// one frame cell each. The 32-byte output occupies the two consecutive + /// 128-bit cells `c, c+1`. Blake3 { ins: [Off; 4], cv: Off, c: Off, - metadata: F128, + metadata: F192, }, } @@ -94,7 +104,12 @@ pub(crate) enum Hint { /// buffer `m[fp·g^bits_ptr]`, reconstruct their integer value, and write /// `g^max(log2_ceil(value), floor)` into `m[fp·g^dst]`. Nondeterministic /// (prover-side); the emitting code re-verifies the result in-circuit. - Log2Ceil { bits_ptr: Off, dst: Off, nbits: u32, floor: u32 }, + Log2Ceil { + bits_ptr: Off, + dst: Off, + nbits: u32, + floor: u32, + }, /// Prover-side debug print of `fp+cell` (witness generation only). Print { label: String, cell: Off }, /// Computed advice: write the `nbits` bits of the value in `m[fp+value]` @@ -105,12 +120,22 @@ pub(crate) enum Hint { /// (recovered by a bounded discrete log at witness generation), into the /// buffer `m[fp·g^bits_ptr]`. The emitting code re-checks it in-circuit. BitDecomposeExp { value: Off, bits_ptr: Off, nbits: u32 }, + /// Computed advice: write the first `len` K-coordinate limbs of an F192 + /// value into consecutive frame cells. The guest must constrain them. + FieldLimbs { value: Off, base: Off, len: u32 }, } pub(crate) struct Lowered { pub(crate) name: String, pub(crate) code: Vec, pub(crate) frame_size: u32, + /// One past the last frame cell the CALLER touches: `2 + n_args + + /// n_ret_cells` (retpc/retfp, the arguments, then the flattened return + /// area). Cells below it cross the frame boundary — the caller writes the + /// arguments and reads the returns — so a write to one of them is + /// observable outside this function even when no instruction here reads it. + /// Everything at or above it is a local temporary. + pub(crate) abi_end: u32, } /// A resolved 2-cell `blake3` operand: a frame (stack) run used in place, or a diff --git a/crates/lean_compiler/src/lib.rs b/crates/lean_compiler/src/lib.rs index 7d166788..e86db72a 100644 --- a/crates/lean_compiler/src/lib.rs +++ b/crates/lean_compiler/src/lib.rs @@ -1,5 +1,5 @@ //! A compiler from a Python-like zkDSL (see `zkDSL.md`) to the ISA (`cpu::Op`). -//! Produces a [`cpu::Program`] — bytecode plus the prover's allocation hints. +//! Produces a [`lean_vm::cpu::Program`] — bytecode plus the prover's allocation hints. //! //! ## Calling convention //! @@ -10,9 +10,14 @@ //! | 0 | `retpc` — return program counter | //! | 1 | `retfp` — caller frame pointer | //! | 2 .. 2+nargs | arguments | -//! | 2+nargs .. 2+nargs+nret | return values | +//! | 2+nargs .. 2+nargs+nretcells | flattened return cells | //! | rest | locals / temporaries / frame-pointer hints | //! +//! A scalar or `HeapBuf` pointer occupies one return cell. A returned +//! `StackBuf(n)` occupies `n` consecutive cells and is copied into a consecutive +//! run in the caller; source-level tuple arity therefore differs from physical +//! return-cell count when a tuple contains a stack buffer. +//! //! A **call** is `DEREF`-then-`JUMP`: the callee frame pointer is a fresh //! prover-hinted cell; the args and `retfp` are stored with `DEREF`(`Cell`/`Fp`), //! then `DEREF`(`Pc`) stores the return address `g²·pc` (the resume point after the @@ -25,11 +30,12 @@ use std::collections::HashMap; use lean_vm::cpu::hints::RHint; use lean_vm::cpu::{DerefMode, Op, Program}; use primitives::{ - field::{F128, g_pow}, + field::{F64, F192, g_pow}, pretty_integer, }; mod ast; +mod cse; mod ir; mod lower; mod parser; @@ -64,10 +70,11 @@ pub fn compile(ast: &Ast) -> Program { // Definitions by name, for Const-parameter specialization at call sites. let defs: HashMap = ast.funcs.iter().map(|f| (f.name.clone(), f.clone())).collect(); // Constant arrays by name, resolved at lowering (`NAME[i]`, `len(NAME)`). - let const_arrays: HashMap> = ast.const_arrays.iter().cloned().collect(); + let const_arrays: HashMap> = ast.const_arrays.iter().cloned().collect(); let dbg_lower = std::env::var("DBG_LOWER").is_ok(); let mut loop_ctr = 0usize; + let mut cse_dropped = 0usize; let mut lowered: Vec = Vec::new(); let mut i = 0; while i < queue.len() { @@ -79,16 +86,35 @@ pub fn compile(ast: &Ast) -> Program { if f.const_params.contains(&true) || f.inline { continue; } - let low = lower_func(&f, &mut queue, &mut loop_ctr, &defs, &const_arrays); + let mut low = lower_func(&f, &mut queue, &mut loop_ctr, &defs, &const_arrays); + // Fold away the pure instructions the lowerer emitted twice. Runs before + // entry pcs are assigned, so only this function's own `KVal::Local` + // targets need renumbering (`cse::compact` does that). + let dropped = if std::env::var("DBG_NO_CSE").is_ok() { + 0 + } else { + cse::cse(&mut low.code, low.abi_end) + }; + cse_dropped += dropped; if dbg_lower { eprintln!("== fn {} (frame {}) ==", low.name, pretty_integer(low.frame_size)); for (i, ins) in low.code.iter().enumerate() { - eprintln!(" {:>3}: {:?}", pretty_integer(i), ins.op); + let index = pretty_integer(i); + eprintln!(" {index:>5}: {:?}", ins.op); } } lowered.push(low); } + if std::env::var("DBG_CSE").is_ok() { + let kept: usize = lowered.iter().map(|l| l.code.len()).sum(); + eprintln!( + "== DBG_CSE: dropped {} redundant pure instructions, {} remain", + pretty_integer(cse_dropped), + pretty_integer(kept) + ); + } + // Assign entry program counters and frame sizes. let mut entry = HashMap::new(); let mut frame_size = HashMap::new(); @@ -145,7 +171,12 @@ pub fn compile(ast: &Ast) -> Program { label: label.clone(), cell: *cell, }, - Hint::Log2Ceil { bits_ptr, dst, nbits, floor } => RHint::Log2Ceil { + Hint::Log2Ceil { + bits_ptr, + dst, + nbits, + floor, + } => RHint::Log2Ceil { bits_ptr: *bits_ptr, dst: *dst, nbits: *nbits, @@ -161,6 +192,11 @@ pub fn compile(ast: &Ast) -> Program { bits_ptr: *bits_ptr, nbits: *nbits, }, + Hint::FieldLimbs { value, base, len } => RHint::FieldLimbs { + value: *value, + base: *base, + len: *len, + }, }) .collect(); hints.insert(here, rhs); @@ -170,9 +206,12 @@ pub fn compile(ast: &Ast) -> Program { } // Pad the bytecode to `B` (the sentinel slot g^{B-1} must exist for execution). - prog.resize(bytecode_size, Op::Set { o: 0, k: F128::ZERO }); + prog.resize(bytecode_size, Op::Set { o: 0, k: F192::ZERO }); let mut program = Program::assemble(prog, 0, 0, hints, frame_size["main"]); - program.fn_ranges = lowered.iter().map(|l| (l.name.clone(), entry[&l.name], l.code.len() as u32)).collect(); + program.fn_ranges = lowered + .iter() + .map(|l| (l.name.clone(), entry[&l.name], l.code.len() as u32)) + .collect(); program } @@ -181,15 +220,18 @@ pub fn compile(ast: &Ast) -> Program { /// constants that are small g-powers (code addresses, indices) show as `gʲ`. pub fn disassemble(prog: &[Op]) -> String { // Reverse index for small g-powers, to pretty-print code addresses/indices. - let mut gmap: HashMap = HashMap::new(); - let mut acc = F128::ONE; + let mut gmap: HashMap = HashMap::new(); + let mut acc = F64::ONE; for j in 0..(prog.len() + 512) { gmap.entry(acc).or_insert(j); acc *= primitives::field::G; } - let kfmt = |k: F128| match gmap.get(&k) { + // A machine word is 192-bit; K-valued immediates (both high limbs zero) may be small + // g-powers (code addresses, indices), shown as `gʲ`. + let kfmt = |k: F192| match (k.c1 == 0 && k.c2 == 0).then(|| gmap.get(&F64(k.c0))).flatten() { Some(j) => format!("g^{j}"), - None => format!("0x{:016x}{:016x}", k.hi, k.lo), + None if k.c1 == 0 && k.c2 == 0 => format!("0x{:016x}", k.c0), + None => format!("0x{:016x}{:016x}{:016x}", k.c2, k.c1, k.c0), }; let mut out = String::new(); @@ -214,10 +256,17 @@ pub fn disassemble(prog: &[Op]) -> String { Op::Jump { oc, od, of } => { format!("JUMP if fp[{oc}]≠0: pc=fp[{od}], fp=fp[{of}]") } + Op::Pack64x2 { a, b, c } => { + format!("PACK64X2 fp[{c}] = pack64(fp[{a}], fp[{b}])") + } Op::Blake3 { ins, cv, out, metadata } => { format!( "BLAKE3 fp[{out}..]= compress(cv=fp[{cv}..], m=fp[{}],fp[{}],fp[{}],fp[{}], meta={})", - ins[0], ins[1], ins[2], ins[3], kfmt(*metadata) + ins[0], + ins[1], + ins[2], + ins[3], + kfmt(*metadata) ) } }; @@ -226,11 +275,16 @@ pub fn disassemble(prog: &[Op]) -> String { out } +/// Embed a `u128` source literal into the low 128 bits of a 192-bit machine word. +pub(crate) fn lit_field(n: u128) -> F192 { + F192::new(n as u64, (n >> 64) as u64, 0) +} + /// `g^e` for a `u128` exponent (square-and-multiply). `field::g_pow` only takes /// a `usize`; an index carried in the exponent (a Fibonacci number, say) can -/// exceed 64 bits. -fn g_pow_u128(mut e: u128) -> F128 { - let mut result = F128::ONE; +/// exceed 64 bits (`ord(g) = 2^64 − 1`, so the exponent wraps mod that). +fn g_pow_u128(mut e: u128) -> F64 { + let mut result = F64::ONE; let mut base = primitives::field::G; while e > 0 { if e & 1 == 1 { @@ -243,12 +297,16 @@ fn g_pow_u128(mut e: u128) -> F128 { } fn resolve(op: &LOp, entry: &HashMap, sentinel: u32, base: u32) -> Op { - let resolve_kval = |kv: &KVal| match kv { - KVal::Const(c) => *c, - KVal::Entry(name) => g_pow(entry[name] as usize), - KVal::EndSentinel => g_pow(sentinel as usize), - KVal::Local(i) => g_pow((base + i) as usize), - KVal::Poison => primitives::field::G.inv(), // g^-1: outside the bytecode cube + let resolve_kval = |kv: &KVal| -> F192 { + match kv { + KVal::Const(c) => *c, + // Address / entry / sentinel constants are K-valued g-powers; + // embed them canonically as (c0, 0, 0). + KVal::Entry(name) => g_pow(entry[name] as usize).into(), + KVal::EndSentinel => g_pow(sentinel as usize).into(), + KVal::Local(i) => g_pow((base + i) as usize).into(), + KVal::Poison => primitives::field::G.inv().into(), // g^-1: outside the bytecode cube + } }; match op { LOp::Set { o, k: kv } => Op::Set { @@ -273,6 +331,7 @@ fn resolve(op: &LOp, entry: &HashMap, sentinel: u32, base: u32) -> od: *od, of: *of, }, + LOp::Pack64x2 { a, b, c } => Op::Pack64x2 { a: *a, b: *b, c: *c }, LOp::Blake3 { ins, cv, c, metadata } => Op::Blake3 { ins: *ins, cv: *cv, diff --git a/crates/lean_compiler/src/lower.rs b/crates/lean_compiler/src/lower.rs index 7bf84bad..ab3ac089 100644 --- a/crates/lean_compiler/src/lower.rs +++ b/crates/lean_compiler/src/lower.rs @@ -26,7 +26,10 @@ fn gmul(a: GAddr, b: GAddr) -> Option { (None, x) | (x, None) => x, (Some(_), Some(_)) => return None, }; - Some(GAddr { base, exp: a.exp.checked_add(b.exp)? }) + Some(GAddr { + base, + exp: a.exp.checked_add(b.exp)?, + }) } /// Cap on a `β`-folded exponent: the operand g-power table is sized to the @@ -36,8 +39,9 @@ const FOLD_MAX: u128 = 1 << 16; /// A deferred stack-cell store: the cell is a copy of another cell, or a zero. /// Recorded instead of emitting the `MUL`/`SET`, and forwarded to the source at -/// each use ([`FnLower::word_src`]) — so `BLAKE3`, which now addresses its four -/// input words independently, reads them in place without assembling copies. +/// each use ([`FnLower::word_src`]) — so `BLAKE3`, +/// which addresses its four two-cell input chunks independently, reads them in +/// place without assembling copies. #[derive(Clone, Copy, PartialEq, Eq)] enum Alias { Cell(Off), @@ -45,8 +49,8 @@ enum Alias { /// holding that value (`const_cell`), so a constant stored into a /// `blake3` operand cell — the `obs`/`squeeze` tag words, padding /// halves — costs ONE `SET` per distinct value per function, not one - /// per store. `Const(0, 0)` routes through the zero pool. - Const(u64, u64), + /// per store. A zero constant routes through the zero pool. + Const(F192), } /// How an inlined `@inline` tail-return value binds into the caller @@ -76,9 +80,12 @@ struct FnLower<'a> { consts: HashMap, next: Off, n_args: u32, + /// Source-level return shapes for this function. Their physical cell widths + /// determine the reserved return area immediately after the arguments. + return_shapes: Vec, is_main: bool, code: Vec, - const_pool: HashMap<(u64, u64), Off>, + one_off: Option, /// Declared size of each `HeapBuf`, keyed by its pointer cell. Shifted /// aliases resolve to the same base cell through their gaddr, so a /// compile-time index checks against the ORIGINAL buffer's bound. @@ -87,13 +94,20 @@ struct FnLower<'a> { /// ([`Self::self_fp`]) — local (`if`/`else`) jumps reload the frame /// pointer on the taken branch. self_fp_off: Option, + /// Range-check product-target cells: bound `k` → the frame cell holding + /// `g^{k-1}`, set lazily once and shared by every check of that bound. + bounds: HashMap, + /// Constant cells: field value (as bits) → the frame cell holding it, SET + /// lazily once per distinct constant ([`Self::const_cell`]). Cells are + /// write-once and read-many, so one `SET` serves every use in scope. + const_cells: HashMap<[u64; 3], Off>, /// Variables bound to a symbolic g-address ([`GAddr`]) — index cursors and /// shifted pointers, kept virtual so their offsets fold into `DEREF`'s `β`. gaddrs: HashMap, /// Variables bound to a compile-time *field* constant that isn't a g-power /// (e.g. a running weight `CHAIN_LENGTH^i`). Kept virtual — folded through /// constant field arithmetic and materialized (one `SET`) only when used. - fconsts: HashMap, + fconsts: HashMap, /// While inlining an `@inline` call ([`Self::try_inline`]), the destination /// cells its tail `return` binds into instead of emitting a return jump. /// `None` outside an inlined body. @@ -105,8 +119,14 @@ struct FnLower<'a> { inline_stack_ret: Option>, /// Deferred stack-cell copies/zeros ([`Alias`]), forwarded at use. alias: HashMap, + /// A cached frame cell holding `0` (for forwarded zero words), set lazily. + zero_off: Option, /// Hints queued to attach to the next emitted instruction. pending: Vec, + /// Active `@inline` expansion stack. Nested inline helpers are allowed, + /// but direct or indirect recursion would otherwise recurse forever in + /// the compiler. + inline_calls: Vec, /// Two consecutive frame cells holding the standard BLAKE3 IV, emitted /// lazily at the first dominating default-IV compression in this /// control-flow scope. [`Self::scoped`] restores this cache at branch joins. @@ -117,12 +137,16 @@ struct FnLower<'a> { tail_call: bool, queue: &'a mut Vec, loop_ctr: &'a mut usize, + /// Name of the function being lowered. Only used to attribute a lowered + /// `for` loop to its source site under `DBG_LOOPS` (the profile prints bare + /// `__loopN` names, which are otherwise opaque). + fn_name: String, /// The program's function definitions by name, for `Const`-parameter /// specialization at call sites ([`Self::specialize`]). defs: &'a HashMap, /// Top-level constant arrays, resolved at compile time: `NAME[i]` yields the /// element (a field value or an index), `len(NAME)` its length. - const_arrays: &'a HashMap>, + const_arrays: &'a HashMap>, } impl FnLower<'_> { @@ -137,35 +161,68 @@ impl FnLower<'_> { self.code.push(LInstr { op, hints }); } - /// Materialize a field constant into a frame cell, pooled per function: - /// frames are write-once, so one cell per distinct constant serves every - /// use site (`main` alone had ~57k duplicated constant `SET`s before this). - fn const_cell(&mut self, v: F128) -> Off { - if let Some(&o) = self.const_pool.get(&(v.lo, v.hi)) { + /// A frame cell holding `1` (always-taken `JUMP` condition), set lazily once. + fn one(&mut self) -> Off { + if let Some(o) = self.one_off { + return o; + } + let o = self.fresh(); + self.emit(LOp::Set { + o, + k: KVal::Const(F192::ONE), + }); + self.one_off = Some(o); + o + } + + /// A frame cell holding the constant `v`, SET lazily once per distinct + /// constant and shared by every read of it in scope (`1` shares + /// [`Self::one`]'s cell; `main` alone had ~57k duplicated constant `SET`s + /// before pooling). Branch-local like the other lazy cells: a cache entry + /// made inside an `if`/`match` arm reverts at the join. + fn const_cell(&mut self, v: F192) -> Off { + if v == F192::ONE { + return self.one(); + } + let key = [v.c0, v.c1, v.c2]; + if let Some(&o) = self.const_cells.get(&key) { return o; } let o = self.fresh(); self.emit(LOp::Set { o, k: KVal::Const(v) }); - self.const_pool.insert((v.lo, v.hi), o); + self.const_cells.insert(key, o); o } - /// A frame cell holding `1` (always-taken `JUMP` condition), set lazily once. - fn one(&mut self) -> Off { - self.const_cell(F128::ONE) + /// A frame cell holding `0`, set lazily once — the source for forwarded zero + /// words (a `BLAKE3` padding half). + fn zero(&mut self) -> Off { + if let Some(o) = self.zero_off { + return o; + } + let o = self.fresh(); + self.emit(LOp::Set { + o, + k: KVal::Const(F192::ZERO), + }); + self.zero_off = Some(o); + o } + /// Two consecutive frame cells holding the standard BLAKE3 IV, emitted + /// lazily at the first dominating default-IV compression in this + /// control-flow scope. [`Self::scoped`] restores this cache at branch joins. fn default_blake3_cv(&mut self) -> Off { if let Some(o) = self.blake3_iv { return o; } let o = self.alloc_stack(2); - for (k, value) in lean_vm::blake3_flock::IV.into_iter().enumerate() { + for (k, value) in lean_vm::blake3_flock::IV_CELLS.into_iter().enumerate() { self.emit(LOp::Set { o: o + k as u32, k: KVal::Const(value), }); - self.const_pool.insert((value.lo, value.hi), o + k as u32); + self.const_cells.insert([value.c0, value.c1, value.c2], o + k as u32); } self.blake3_iv = Some(o); o @@ -183,7 +240,7 @@ impl FnLower<'_> { let (base, _) = self.stack_of(arr)?; Some(Alias::Cell(base + self.try_const_index(idx)?)) } - _ => self.try_field_const(val).map(|c| Alias::Const(c.lo, c.hi)), + _ => self.try_field_const(val).map(Alias::Const), } } @@ -264,9 +321,11 @@ impl FnLower<'_> { self.stacks.clone(), self.consts.clone(), self.self_fp_off, + self.bounds.clone(), + self.const_cells.clone(), self.gaddrs.clone(), self.fconsts.clone(), - self.const_pool.clone(), + self.zero_off, self.blake3_iv, ); f(self); @@ -289,7 +348,7 @@ impl FnLower<'_> { let o = self.fresh(); self.emit(LOp::Set { o, - k: KVal::Const(F128::ZERO), + k: KVal::Const(F192::ZERO), }); } ( @@ -297,9 +356,11 @@ impl FnLower<'_> { self.stacks, self.consts, self.self_fp_off, + self.bounds, + self.const_cells, self.gaddrs, self.fconsts, - self.const_pool, + self.zero_off, self.blake3_iv, ) = saved; self.alias = saved_aliases; @@ -341,6 +402,16 @@ impl FnLower<'_> { /// cells shared by every arm (write-once: exactly one arm executes); /// `names` bind to those cells at the join. fn lower_match_range(&mut self, names: &[String], x: &Expr, arms: &[Expr]) { + for arm in arms { + if let Expr::Call(f, _) = arm + && self + .defs + .get(f) + .is_some_and(|d| !d.inline && d.return_shapes.iter().any(|s| matches!(s, ReturnShape::StackBuf(_)))) + { + panic!("a normal function's StackBuf return cannot cross a match_range join; bind it with `let`"); + } + } // Fusion: when every arm is a direct call to the same function with // identical runtime args (differing only in `Const` args — the usual // `lambda k: f(a, b, k)`), set up one shared callee frame and dispatch @@ -469,7 +540,11 @@ impl FnLower<'_> { self.emit(LOp::Mul { a: xo, b: xo, c: x2 }); let d = self.fresh(); self.emit(LOp::Mul { a: kcell, b: x2, c: d }); - self.emit(LOp::Jump { oc: one, od: d, of: sfp }); + self.emit(LOp::Jump { + oc: one, + od: d, + of: sfp, + }); // Trampoline: slot j enters `callees[j]` with fp = nfp; the callee's own // `return` jumps to retpc (the join) in the caller frame. @@ -480,7 +555,11 @@ impl FnLower<'_> { o: c, k: KVal::Entry(callee.clone()), }); - self.emit(LOp::Jump { oc: one, od: c, of: nfp }); + self.emit(LOp::Jump { + oc: one, + od: c, + of: nfp, + }); } // Join: read the return values (written by whichever callee ran). @@ -585,9 +664,17 @@ impl FnLower<'_> { } return; } - let (la, lb) = (self.expr(lhs), self.expr(rhs)); - let x = self.fresh(); - self.emit(LOp::Xor { a: la, b: lb, c: x }); // x = lhs + rhs: nonzero ⇔ != + // `x != 0` needs no XOR: the cell itself is the JUMP's nonzero test. + let x = if self.try_lit(rhs) == Some(0) { + self.expr(lhs) + } else if self.try_lit(lhs) == Some(0) { + self.expr(rhs) + } else { + let (la, lb) = (self.expr(lhs), self.expr(rhs)); + let x = self.fresh(); + self.emit(LOp::Xor { a: la, b: lb, c: x }); // x = lhs + rhs: nonzero ⇔ != + x + }; // Hoisted on purpose: these SETs must dominate the join. let sfp = self.self_fp(); let one = self.one(); @@ -649,19 +736,39 @@ impl FnLower<'_> { // a != b: skip the poison and continue at the join (patched below). let cont = self.fresh(); let cset = self.code.len(); - self.emit(LOp::Set { o: cont, k: KVal::Local(0) }); - self.emit(LOp::Jump { oc: x, od: cont, of: sfp }); + self.emit(LOp::Set { + o: cont, + k: KVal::Local(0), + }); + self.emit(LOp::Jump { + oc: x, + od: cont, + of: sfp, + }); // a == b: fall through to the poison jump (g^-1, an unreachable pc). let pd = self.fresh(); self.emit(LOp::Set { o: pd, k: KVal::Poison }); - self.emit(LOp::Jump { oc: one, od: pd, of: sfp }); + self.emit(LOp::Jump { + oc: one, + od: pd, + of: sfp, + }); self.patch_local(cset, self.code.len()); } /// The frame cell holding `g^{k-1}` — the range-check product target — set /// lazily once per distinct bound `k` and shared by that bound's checks. fn bound_cell(&mut self, k: u64) -> Off { - self.const_cell(g_pow_u128((k - 1) as u128)) + if let Some(&o) = self.bounds.get(&k) { + return o; + } + let o = self.fresh(); + self.emit(LOp::Set { + o, + k: KVal::Const(g_pow_u128((k - 1) as u128).into()), + }); + self.bounds.insert(k, o); + o } /// `hint_witness(dest, "name")` — resolve `dest` to a run of cells and @@ -727,8 +834,8 @@ impl FnLower<'_> { /// unknown operand (leanVM's ADD deduction, multiplicatively); /// 3. `DEREF` through `y` — proves `y = g^f` with `f < 2^h`. /// - /// Then `e + f ≡ k-1 (mod 2^128-1)` with `e, f < 2^h`, and since a negative - /// `k-1-e` wraps to `≈ 2^128 ≫ 2^h`, this forces `e ≤ k-1` — for ANY memory + /// Then `e + f ≡ k-1 (mod 2^64-1)` with `e, f < 2^h`, and since a negative + /// `k-1-e` wraps to `≈ 2^64 ≫ 2^h`, this forces `e ≤ k-1` — for ANY memory /// size the prover announces, provided `k ≤ 2^MIN_LOG_MEM`. The two `DEREF` /// target cells are unconstrained touches (only the address matters), /// back-filled at the end of execution; the constant cell is one amortized @@ -761,11 +868,17 @@ impl FnLower<'_> { } fn expr(&mut self, e: &Expr) -> Off { + if matches!(e, Expr::Call(f, _) if f == "f192") { + return self.const_cell(self.try_field_const(e).expect("f192 needs three literal u64 limbs")); + } match e { - Expr::Lit(n) => self.const_cell(F128::new(*n as u64, (*n >> 64) as u64)), - Expr::Gen => self.const_cell(g_pow(1)), - Expr::GPow(k) => self.const_cell(g_pow_u128(*k)), - Expr::GenPow(e) => self.const_cell(g_pow_u128(self.gpow_exp(e))), + Expr::Lit(n) => self.const_cell(lit_field(*n)), + Expr::Gen => self.const_cell(g_pow(1).into()), + Expr::GPow(k) => self.const_cell(g_pow_u128(*k).into()), + Expr::GenPow(e) => { + let k = self.gpow_exp(e); + self.const_cell(g_pow_u128(k).into()) + } Expr::Pow(b, e) => self.pow_expr(b, e), Expr::Var(v) => { if self.stacks.contains_key(v) { @@ -784,10 +897,10 @@ impl FnLower<'_> { // (and, being a constant, has no side effect to preserve), so // `x + 0` lowers to just `x` — no cell, no XOR. Kills the // `acc = 0; acc = acc + t` accumulator seed and similar. - if self.try_field_const(a) == Some(F128::ZERO) { + if self.try_field_const(a) == Some(F192::ZERO) { return self.expr(b); } - if self.try_field_const(b) == Some(F128::ZERO) { + if self.try_field_const(b) == Some(F192::ZERO) { return self.expr(a); } let (la, lb) = (self.expr(a), self.expr(b)); @@ -799,10 +912,10 @@ impl FnLower<'_> { // Identity fold: a compile-time 1 operand is a no-op multiply, // so `x * 1` lowers to just `x`. Kills the `acc = GEN ** 0` // (= 1) accumulator seed's first `1 * f` in every product loop. - if self.try_field_const(a) == Some(F128::ONE) { + if self.try_field_const(a) == Some(F192::ONE) { return self.expr(b); } - if self.try_field_const(b) == Some(F128::ONE) { + if self.try_field_const(b) == Some(F192::ONE) { return self.expr(a); } let (la, lb) = (self.expr(a), self.expr(b)); @@ -831,12 +944,25 @@ impl FnLower<'_> { let nbits = self.const_index(&args[1]); let floor = self.const_index(&args[2]); let dst = self.fresh(); - self.pending.push(Hint::Log2Ceil { bits_ptr, dst, nbits, floor }); + self.pending.push(Hint::Log2Ceil { + bits_ptr, + dst, + nbits, + floor, + }); dst } + Expr::Call(f, args) if f == "pack64x2" => { + assert_eq!(args.len(), 2, "pack64x2(a, b) takes two scalar cells"); + let a = self.expr(&args[0]); + let b = self.expr(&args[1]); + let c = self.fresh(); + self.emit(LOp::Pack64x2 { a, b, c }); + c + } Expr::Call(f, args) => { if let Some(n) = self.const_len(e) { - self.const_cell(F128::new(n as u64, 0)) + self.const_cell(F192::new(n as u64, 0, 0)) } else { let d = self.call(f, args, 1)[0]; self.take_inline_ret_cell(d) @@ -866,7 +992,7 @@ impl FnLower<'_> { Expr::Index(arr, idx) => { // Constant-array element `NAME[i]`: a compile-time field value. if let Some(elem) = self.const_array_elem(e) { - return self.const_cell(F128::new(elem as u64, (elem >> 64) as u64)); + return self.const_cell(elem); } // Stack read `sa[k]`: the frame cell `base + k` directly (no deref), // forwarded through any deferred copy/zero alias. @@ -887,7 +1013,9 @@ impl FnLower<'_> { dst } Expr::Sub(..) | Expr::Div(..) | Expr::Mod(..) => { - panic!("`-`, `//`, `%` are compile-time only (field subtraction is `+`); use them in an index, a bound, or a `Const` argument, got `{e:?}`") + panic!( + "`-`, `//`, `%` are compile-time only (field subtraction is `+`); use them in an index, a bound, or a `Const` argument, got `{e:?}`" + ) } Expr::Slice(..) => panic!("a slice is not a scalar; it is only a blake3 operand"), Expr::ListLit(..) => panic!("a list literal must be bound to a name: `x = [a, b]`"), @@ -940,7 +1068,10 @@ impl FnLower<'_> { // A constant-array element `NAME[i]` or `len(NAME)` used as an index / // bound / `unroll` count. An element too large for an index declines // (it is a field value; this evaluator also probes speculatively). - Expr::Index(..) => self.const_array_elem(idx).and_then(|e| u32::try_from(e).ok()), + Expr::Index(..) => self + .const_array_elem(idx) + .and_then(|e| (e.c1 == 0 && e.c2 == 0).then_some(e.c0)) + .and_then(|e| u32::try_from(e).ok()), Expr::Call(..) => self.const_len(idx).map(|n| n as u32), // Integer power `b ** e` (both compile-time), e.g. `2 ** c` for a bit // test. Overflow declines (see the Add/Sub/Mul comment above). @@ -976,14 +1107,19 @@ impl FnLower<'_> { .unwrap_or_else(|| panic!("`**` exponent must be a compile-time integer, got `{e:?}`")); // Fully constant → evaluate in the field and emit a single `SET`. if let Some(bc) = self.try_field_const(b) { - let mut acc = F128::ONE; + let mut acc = F192::ONE; for _ in 0..k { acc *= bc; } return self.const_cell(acc); } if k == 0 { - return self.one(); + let o = self.fresh(); + self.emit(LOp::Set { + o, + k: KVal::Const(F192::ONE), + }); + return o; } // Runtime base: square-and-multiply over the compile-time exponent bits. let base = self.expr(b); @@ -1004,7 +1140,7 @@ impl FnLower<'_> { /// If `e` is `NAME[i]` for a top-level constant array `NAME` with a /// compile-time index `i`, its element (a raw `u128`). - fn const_array_elem(&self, e: &Expr) -> Option { + fn const_array_elem(&self, e: &Expr) -> Option { if let Expr::Index(arr, idx) = e && let Expr::Var(v) = arr.as_ref() && let Some(a) = self.const_arrays.get(v) @@ -1030,11 +1166,45 @@ impl FnLower<'_> { None } + /// The field value of `e` when it is a trivial compile-time constant (a + /// literal, a literal-bound name, or `GEN ** 0`), for the `x*1`/`x+0` + /// arithmetic identities and the `== 0` test of [`Self::lower_if`]. + fn try_lit(&self, e: &Expr) -> Option { + match e { + Expr::Lit(n) => u64::try_from(*n).ok(), + Expr::Var(v) => self.consts.get(v).map(|&n| n as u64), + Expr::GPow(0) => Some(1), + _ => None, + } + } + + /// The compile-time g-power exponent of a heap-index expression, when it + /// has one: `1` (= `g^0`), `GEN`, `GEN ** k`, power-of-two literals + /// (`g = x`, so the literal `2^j` IS `g^j`), names bound to such + /// literals, and products of those (exponents add). `None` for runtime + /// values, and for exponents ≥ 2^MIN_LOG_MEM, which must not become a + /// `DEREF` `beta` immediate (`beta` is capped by the smallest admissible + /// memory size; the fallback MUL path handles any element). + fn try_gpow_index(&self, idx: &Expr) -> Option { + let cap = |k: u32| (k < (1u32 << lean_vm::cpu::MIN_LOG_MEM)).then_some(k); + let pow2 = |n: u128| (n.is_power_of_two() && n < (1 << 64)).then(|| n.trailing_zeros()); + match idx { + Expr::Lit(n) => pow2(*n).and_then(cap), + Expr::Var(v) => pow2(*self.consts.get(v)? as u128).and_then(cap), + Expr::Gen => Some(1), + Expr::GPow(k) => cap(u32::try_from(*k).ok()?), + Expr::GenPow(e) => cap(self.try_const_index(e)?), + Expr::Mul(a, b) => cap(self.try_gpow_index(a)?.checked_add(self.try_gpow_index(b)?)?), + _ => None, + } + } + /// Resolve a `blake3` operand — a size-2 `StackBuf` name, a 2-cell /// `StackBuf` slice `buf[lo:hi]`, or a 2-cell `HeapBuf` slice (cells - /// `ptr·g^lo`, `ptr·g^{lo+1}`) — with compile-time bounds. Stack operands - /// are used in place; heap operands must be bridged through the stack, - /// since `BLAKE3` addresses only frame cells (see [`Self::blake3_input`]). + /// `ptr·g^{lo+k}`, `k < 2`) — with compile-time bounds. A 256-bit operand is + /// two 128-bit cells. Stack operands are used in place; heap operands must be + /// bridged through the stack, since `BLAKE3` addresses only frame cells (see + /// [`Self::blake3_input`]). fn blake3_operand(&mut self, e: &Expr) -> B3Operand { match e { Expr::Var(_) => { @@ -1043,7 +1213,7 @@ impl FnLower<'_> { .expect("a bare blake3 operand must be a StackBuf; slice a HeapBuf: `buf[lo:lo + 2]`"); assert!( size == 2, - "a whole-StackBuf blake3 operand must have size 2; slice a larger one: `buf[lo:lo + 2]`" + "a whole-StackBuf blake3 operand must have size 2 (two 128-bit cells); slice a larger one: `buf[lo:lo + 2]`" ); B3Operand::Stack(base) } @@ -1064,10 +1234,10 @@ impl FnLower<'_> { } } // Runtime start (heap only): `buf[i:i + 2]` with a runtime - // g-power index `i` names the cells `buf·i`, `buf·i·g`. The + // g-power index `i` names the cells `buf·i·g^k`, k < 2. The // `hi` bound cannot be evaluated, only shape-checked: it must // be syntactically `lo + 2`. One MUL folds `i` into the - // pointer; the two-cell bridge is then offsets 0, 1 off it. + // pointer; the two-cell bridge is then offsets 0..2 off it. _ => { assert!( self.stack_of(arr).is_none(), @@ -1087,14 +1257,15 @@ impl FnLower<'_> { } } - /// A `blake3` *input* operand as a frame offset: stack runs in place; a - /// heap slice is pulled into a fresh stack pair first — one `DEREF` per - /// cell (`m[ptr·g^{lo+k}] == m[fp+t+k]`, the `β` immediate doing the - /// pointer offset). The heap cells must already be written. + /// A `blake3` *input* operand as its two independently-addressed 128-bit + /// chunk bases (each chunk is ONE 128-bit cell): stack runs in place; a heap + /// slice is pulled into a fresh stack pair first — one `DEREF` per cell + /// (`m[ptr·g^{lo+k}] == m[fp+t+k]`, the `β` immediate doing the pointer + /// offset). The heap cells must already be written. fn blake3_input(&mut self, e: &Expr) -> [Off; 2] { match self.blake3_operand(e) { - // A stack operand: the two words live at `o, o+1`; forward each cell's - // real source where one is known (a copy or a zero), so a hash of + // A stack operand: the two chunk cells are `o, o+1`; forward each + // cell's real source where known (a copy or a zero), so a hash of // non-adjacent values needs no assembling copies. B3Operand::Stack(o) => [self.word_src(o), self.word_src(o + 1)], B3Operand::Heap { ptr, lo } => { @@ -1129,14 +1300,12 @@ impl FnLower<'_> { } /// The cell holding the value of stack cell `o`, following a recorded copy / - /// zero alias to its real source (so `BLAKE3` reads the source directly and - /// the assembling copy is never emitted). Returns `o` when it holds a genuine - /// value. + /// zero alias to its real source. Returns `o` when it holds a genuine value. fn word_src(&mut self, o: Off) -> Off { match self.alias.get(&o).copied() { Some(Alias::Cell(s)) => self.word_src(s), - Some(Alias::Const(0, 0)) => self.const_cell(F128::ZERO), - Some(Alias::Const(lo, hi)) => self.const_cell(F128::new(lo, hi)), + Some(Alias::Const(v)) if v.is_zero() => self.zero(), + Some(Alias::Const(v)) => self.const_cell(v), None => o, } } @@ -1150,7 +1319,7 @@ impl FnLower<'_> { if let Some(elem) = self.const_array_elem(e) { self.emit(LOp::Set { o: dst, - k: KVal::Const(F128::new(elem as u64, (elem >> 64) as u64)), + k: KVal::Const(elem), }); return; } @@ -1168,20 +1337,20 @@ impl FnLower<'_> { Expr::Lit(n) => { self.emit(LOp::Set { o: dst, - k: KVal::Const(F128::new(*n as u64, (*n >> 64) as u64)), + k: KVal::Const(lit_field(*n)), }); } Expr::Gen => self.emit(LOp::Set { o: dst, - k: KVal::Const(g_pow(1)), + k: KVal::Const(g_pow(1).into()), }), Expr::GPow(k) => self.emit(LOp::Set { o: dst, - k: KVal::Const(g_pow_u128(*k)), + k: KVal::Const(g_pow_u128(*k).into()), }), Expr::GenPow(e) => self.emit(LOp::Set { o: dst, - k: KVal::Const(g_pow_u128(self.gpow_exp(e))), + k: KVal::Const(g_pow_u128(self.gpow_exp(e)).into()), }), Expr::Pow(b, e) => { let v = self.pow_expr(b, e); @@ -1189,9 +1358,9 @@ impl FnLower<'_> { } Expr::Add(a, b) => { // Identity fold (see the `expr` Add arm): `x + 0` copies `x`. - if self.try_field_const(a) == Some(F128::ZERO) { + if self.try_field_const(a) == Some(F192::ZERO) { self.expr_into(b, dst); - } else if self.try_field_const(b) == Some(F128::ZERO) { + } else if self.try_field_const(b) == Some(F192::ZERO) { self.expr_into(a, dst); } else { let (la, lb) = (self.expr(a), self.expr(b)); @@ -1200,9 +1369,9 @@ impl FnLower<'_> { } Expr::Mul(a, b) => { // Identity fold: `x * 1` copies `x`. - if self.try_field_const(a) == Some(F128::ONE) { + if self.try_field_const(a) == Some(F192::ONE) { self.expr_into(b, dst); - } else if self.try_field_const(b) == Some(F128::ONE) { + } else if self.try_field_const(b) == Some(F192::ONE) { self.expr_into(a, dst); } else { let (la, lb) = (self.expr(a), self.expr(b)); @@ -1226,13 +1395,33 @@ impl FnLower<'_> { } } - /// Compute the absolute pointer `arr·idx` into a fresh cell (heap addressing - /// in the exponent: cell `g^k` of the buffer sits at `arr·g^k`). - fn array_ptr(&mut self, arr: &Expr, idx: &Expr) -> Off { + /// Resolve a heap access `arr[idx]` to a `DEREF`-ready pair: a cell + /// holding a pointer `p` and a compile-time exponent `beta`, the accessed + /// cell being `m[p·g^beta]` (heap addressing in the exponent: cell `g^k` + /// of the buffer sits at `arr·g^k`). A constant g-power `idx`, or a + /// constant g-power *factor* of it, folds into the `beta` immediate, so + /// only a runtime factor costs a pointer `MUL` (and a wholly constant + /// index costs nothing at all). + fn array_ptr(&mut self, arr: &Expr, idx: &Expr) -> (Off, u32) { + if let Some(k) = self.try_gpow_index(idx) { + return (self.expr(arr), k); + } + // `buf[r * GEN ** k]` (either factor order): beta takes the constant, + // the pointer MUL takes only the runtime factor `r`. + if let Expr::Mul(a, b) = idx { + for (c, r) in [(a, b), (b, a)] { + if let Some(k) = self.try_gpow_index(c) { + let (la, lr) = (self.expr(arr), self.expr(r)); + let ptr = self.fresh(); + self.emit(LOp::Mul { a: la, b: lr, c: ptr }); + return (ptr, k); + } + } + } let (la, li) = (self.expr(arr), self.expr(idx)); let ptr = self.fresh(); self.emit(LOp::Mul { a: la, b: li, c: ptr }); - ptr + (ptr, 0) } /// The symbolic g-address of `e`, when it is one: a constant g-power @@ -1244,7 +1433,10 @@ impl FnLower<'_> { Expr::Lit(1) => Some(GAddr { base: None, exp: 0 }), Expr::Gen => Some(GAddr { base: None, exp: 1 }), Expr::GPow(k) => Some(GAddr { base: None, exp: *k }), - Expr::GenPow(e) => Some(GAddr { base: None, exp: self.try_const_index(e)? as u128 }), + Expr::GenPow(e) => Some(GAddr { + base: None, + exp: self.try_const_index(e)? as u128, + }), Expr::Var(v) => self .gaddrs .get(v) @@ -1257,28 +1449,37 @@ impl FnLower<'_> { /// `e` as a compile-time *field* constant, when it is one: a literal, `GEN`, /// `GEN ** k`, a var bound to a field constant (or a constant g-power), or - /// `+`/`*` of those evaluated in the field (XOR / GHASH). `None` for a - /// runtime value or a compile-time *integer* op (`//`/`%` are index-only). - fn try_field_const(&self, e: &Expr) -> Option { + /// `+`/`*` of those evaluated in the field (XOR / `K`-mul). `None` for a + /// runtime value, a literal exceeding the 64-bit word, or a compile-time + /// *integer* op (`//`/`%` are index-only). + fn try_field_const(&self, e: &Expr) -> Option { match e { - Expr::Lit(n) => Some(F128::new(*n as u64, (*n >> 64) as u64)), - Expr::Gen => Some(g_pow(1)), - Expr::GPow(k) => Some(g_pow_u128(*k)), - Expr::GenPow(e) => Some(g_pow_u128(self.try_const_index(e)? as u128)), + // A source literal fills the low 128 bits; g-powers/addresses embed in K. + Expr::Lit(n) => Some(lit_field(*n)), + Expr::Gen => Some(g_pow(1).into()), + Expr::GPow(k) => Some(g_pow_u128(*k).into()), + Expr::GenPow(e) => Some(g_pow_u128(self.try_const_index(e)? as u128).into()), Expr::Var(v) => self.fconsts.get(v).copied().or_else(|| match self.gaddrs.get(v) { - Some(GAddr { base: None, exp }) => Some(g_pow_u128(*exp)), + Some(GAddr { base: None, exp }) => Some(g_pow_u128(*exp).into()), _ => None, }), Expr::Add(a, b) => Some(self.try_field_const(a)? + self.try_field_const(b)?), Expr::Mul(a, b) => Some(self.try_field_const(a)? * self.try_field_const(b)?), // A constant-array element `NAME[i]` as a field value, or `len(NAME)`. - Expr::Index(..) => self.const_array_elem(e).map(|v| F128::new(v as u64, (v >> 64) as u64)), - Expr::Call(..) => self.const_len(e).map(|n| F128::new(n as u64, 0)), + Expr::Index(..) => self.const_array_elem(e), + Expr::Call(f, args) if f == "f192" && args.len() == 3 => { + let limb = |i: usize| match &args[i] { + Expr::Lit(n) => u64::try_from(*n).ok(), + _ => None, + }; + Some(F192::new(limb(0)?, limb(1)?, limb(2)?)) + } + Expr::Call(..) => self.const_len(e).map(|n| F192::new(n as u64, 0, 0)), // `b ** e` as a field constant (constant base, compile-time exponent). Expr::Pow(b, e) => { let bc = self.try_field_const(b)?; let k = self.try_const_index(e)?; - let mut acc = F128::ONE; + let mut acc = F192::ONE; for _ in 0..k { acc *= bc; } @@ -1295,7 +1496,11 @@ impl FnLower<'_> { match ga { GAddr { base: Some(c), exp: 0 } => c, GAddr { base, exp } => { - let k = self.const_cell(g_pow_u128(exp)); + let k = self.fresh(); + self.emit(LOp::Set { + o: k, + k: KVal::Const(g_pow_u128(exp).into()), + }); let Some(c) = base else { return k }; let o = self.fresh(); self.emit(LOp::Mul { a: c, b: k, c: o }); @@ -1310,8 +1515,12 @@ impl FnLower<'_> { /// indices are not checked (their value is not known here). fn check_heap_bound(&self, arr: &Expr, extra: u128, span: u128) { let Some(ga) = self.gaddr_of(arr) else { return }; - let (Some(base), Some(exp)) = (ga.base, ga.exp.checked_add(extra)) else { return }; - let Some(&size) = self.heap_sizes.get(&base) else { return }; + let (Some(base), Some(exp)) = (ga.base, ga.exp.checked_add(extra)) else { + return; + }; + let Some(&size) = self.heap_sizes.get(&base) else { + return; + }; if exp + span > size { let name = self .vars @@ -1336,14 +1545,19 @@ impl FnLower<'_> { self.check_heap_bound(arr, extra, 1); if let Some(ga) = self.gaddr_of(arr) && let (Some(base), Some(exp)) = (ga.base, ga.exp.checked_add(extra)) - && exp <= FOLD_MAX { - return (base, exp as u32); - } + && exp <= FOLD_MAX + { + return (base, exp as u32); + } let a = self.expr(arr); if extra == 0 { return (a, 0); } - let k = self.const_cell(g_pow_u128(extra)); + let k = self.fresh(); + self.emit(LOp::Set { + o: k, + k: KVal::Const(g_pow_u128(extra).into()), + }); let ptr = self.fresh(); self.emit(LOp::Mul { a, b: k, c: ptr }); (ptr, 0) @@ -1361,7 +1575,7 @@ impl FnLower<'_> { { panic!( "heap index folds to the field constant {:#x}:{:#x}, not a g-power — heap cell k is addressed as `buf[GEN ** k]` (did an integer index leak in from a StackBuf conversion?)", - c.hi, c.lo + c.c1, c.c0 ); } match self.gaddr_of(idx) { @@ -1382,7 +1596,9 @@ impl FnLower<'_> { } None => {} } - (self.array_ptr(arr, idx), 0) + // Fall back to the constant-g-power-factor fold (a runtime index still + // materializes the pointer `MUL`, with any constant factor in `β`). + self.array_ptr(arr, idx) } /// Consume the [`RetBind`] a single-value inlined tail return recorded, @@ -1396,23 +1612,66 @@ impl FnLower<'_> { match self.inline_stack_ret.take().and_then(|b| b.into_iter().next()) { Some(RetBind::Gaddr(ga)) => self.materialize(ga), Some(RetBind::Stack(base, size)) => { - assert_eq!(size, 1, "a multi-cell StackBuf return needs a `let` binding, not an expression use"); + assert_eq!( + size, 1, + "a multi-cell StackBuf return needs a `let` binding, not an expression use" + ); base } _ => dst, } } - /// Lower a call; returns the caller offsets bound to the returned values. + /// Lower a call; returns one caller offset per source-level return value. + /// A real-call StackBuf return is flattened into consecutive ABI cells and + /// copied into a fresh consecutive run in the caller. `inline_stack_ret` + /// describes those logical bindings to the surrounding let/tuple lowering. fn call(&mut self, callee: &str, args: &[Expr], n_ret: usize) -> Vec { assert!( callee != "blake3", "blake3 is a statement: `blake3(a, b, out)` writes the digest into the 2-cell stack run `out`" ); - let dsts: Vec = (0..n_ret).map(|_| self.fresh()).collect(); self.inline_stack_ret = None; - self.call_into(callee, args, &dsts); - dsts + if self.defs.get(callee).is_some_and(|d| d.inline) { + let dsts: Vec = (0..n_ret).map(|_| self.fresh()).collect(); + self.call_into(callee, args, &dsts); + return dsts; + } + + let shapes = self + .defs + .get(callee) + .map(|d| d.return_shapes.clone()) + .unwrap_or_else(|| vec![ReturnShape::Scalar; n_ret]); + assert_eq!( + shapes.len(), + n_ret, + "`{callee}` returns {} values, call binds {n_ret}", + shapes.len() + ); + let mut logical = Vec::with_capacity(n_ret); + let mut physical = Vec::new(); + let mut binds = Vec::with_capacity(n_ret); + for shape in shapes { + match shape { + ReturnShape::Scalar => { + let dst = self.fresh(); + logical.push(dst); + physical.push(dst); + binds.push(RetBind::Scalar); + } + ReturnShape::StackBuf(size) => { + assert!(size > 0, "a returned StackBuf must not be empty"); + let base = self.alloc_stack(size); + logical.push(base); + physical.extend(base..base + size); + binds.push(RetBind::Stack(base, size)); + } + } + } + self.lower_call(callee, args, physical.len(), None, Some(&physical), false); + self.inline_stack_ret = Some(binds); + logical } /// Evaluate `callee(args)` into `dsts` — inlining the callee when it is @@ -1420,6 +1679,19 @@ impl FnLower<'_> { fn call_into(&mut self, callee: &str, args: &[Expr], dsts: &[Off]) { assert!(callee != "blake3", "blake3 is a statement, not a value-returning call"); if !self.try_inline(callee, args, dsts) { + if let Some(def) = self.defs.get(callee) { + assert_eq!( + def.return_shapes.len(), + dsts.len(), + "`{callee}` returns {} values, call binds {}", + def.return_shapes.len(), + dsts.len() + ); + assert!( + def.return_shapes.iter().all(|s| *s == ReturnShape::Scalar), + "a normal function's multi-cell StackBuf return needs a `let` binding" + ); + } self.lower_call(callee, args, dsts.len(), None, Some(dsts), false); } } @@ -1467,10 +1739,20 @@ impl FnLower<'_> { let (params, rt_args, body, n_ret) = self .specialized_body(callee, args) .unwrap_or_else(|| panic!("`@inline {callee}`: bad arity or unresolved Const argument")); - assert_eq!(n_ret, dsts.len(), "`@inline {callee}` returns {n_ret} values, call binds {}", dsts.len()); + assert_eq!( + n_ret, + dsts.len(), + "`@inline {callee}` returns {n_ret} values, call binds {}", + dsts.len() + ); + assert!( + body_inlinable(&body, self.defs), + "`@inline {callee}` must be a single tail `return` with only builtin or @inline calls, and no loop/match" + ); assert!( - body_inlinable(&body), - "`@inline {callee}` must be a single tail `return` with no call/loop/match (see body_inlinable)" + !self.inline_calls.iter().any(|f| f == callee), + "recursive @inline expansion is not supported: {} -> {callee}", + self.inline_calls.join(" -> ") ); // Bind the params from the caller-scope arguments (symbolically where we // can, so a shifted-pointer arg keeps folding into `β`; a `StackBuf` arg @@ -1515,9 +1797,12 @@ impl FnLower<'_> { } } let saved_ret = self.inline_ret.replace(dsts.to_vec()); + self.inline_calls.push(callee.to_string()); for s in &body { self.stmt(s); } + let popped = self.inline_calls.pop(); + debug_assert_eq!(popped.as_deref(), Some(callee)); self.inline_ret = saved_ret; (self.vars, self.stacks, self.consts, self.gaddrs, self.fconsts) = saved; true @@ -1599,6 +1884,7 @@ impl FnLower<'_> { params: rt_params, const_params, n_ret: def.n_ret, + return_shapes: def.return_shapes.clone(), body, inline: false, }); @@ -1779,7 +2065,7 @@ impl FnLower<'_> { self.vars.remove(name); self.stacks.remove(name); self.gaddrs.remove(name); - self.fconsts.insert(name.clone(), F128::new(k as u64, 0)); + self.fconsts.insert(name.clone(), lit_field(k as u128)); } else if let Expr::Call(cf, cargs) = e && self.defs.contains_key(cf) { @@ -1848,7 +2134,7 @@ impl FnLower<'_> { self.emit(LOp::Xor { a: la, b: lb, c: t }); self.emit(LOp::Set { o: t, - k: KVal::Const(F128::ZERO), + k: KVal::Const(F192::ZERO), }); } Stmt::AssertNe(a, b) => self.lower_assert_ne(a, b), @@ -1866,7 +2152,7 @@ impl FnLower<'_> { let o = self.fresh(); self.emit(LOp::Set { o, - k: KVal::Const(F128::ZERO), + k: KVal::Const(F192::ZERO), }); } Stmt::If { @@ -1923,11 +2209,21 @@ impl FnLower<'_> { let key = name.strip_prefix("__kw_").unwrap(); assert!(kwargs.insert(key, &value[0]).is_none(), "duplicate {f} keyword `{key}`"); } - let allowed = ["cv", "counter", "chunk", "block_len", "flags", "step", "end", "root", "parent"]; + let allowed = [ + "cv", + "counter", + "chunk", + "block_len", + "flags", + "step", + "end", + "root", + "parent", + ]; assert!(kwargs.keys().all(|k| allowed.contains(k)), "unknown {f} keyword"); - let customized = kwargs.keys().any(|k| { - matches!(*k, "counter" | "chunk" | "flags" | "step" | "end" | "root" | "parent") - }); + let customized = kwargs + .keys() + .any(|k| matches!(*k, "counter" | "chunk" | "flags" | "step" | "end" | "root" | "parent")); assert!( !kwargs.contains_key("cv") || customized, "blake3 with cv= requires step=, flags=, or another structured metadata keyword" @@ -1945,12 +2241,12 @@ impl FnLower<'_> { self.default_blake3_cv() }; let const_kw = |this: &Self, name: &str, default: u128| -> u128 { - kwargs - .get(name) - .map(|e| this.const_index(e) as u128) - .unwrap_or(default) + kwargs.get(name).map(|e| this.const_index(e) as u128).unwrap_or(default) }; - assert!(!(kwargs.contains_key("counter") && kwargs.contains_key("chunk")), "use either counter= or chunk=, not both"); + assert!( + !(kwargs.contains_key("counter") && kwargs.contains_key("chunk")), + "use either counter= or chunk=, not both" + ); let counter = if kwargs.contains_key("chunk") { const_kw(self, "chunk", 0) } else { @@ -1970,14 +2266,21 @@ impl FnLower<'_> { } else { lean_vm::blake3_flock::FLAGS as u128 }; - if const_kw(self, "end", 0) != 0 { flags |= 1 << 1; } - if const_kw(self, "parent", 0) != 0 { flags |= 1 << 2; } - if const_kw(self, "root", 0) != 0 { flags |= 1 << 3; } + if const_kw(self, "end", 0) != 0 { + flags |= 1 << 1; + } + if const_kw(self, "parent", 0) != 0 { + flags |= 1 << 2; + } + if const_kw(self, "root", 0) != 0 { + flags |= 1 << 3; + } assert!(flags <= u32::MAX as u128, "BLAKE3 flags do not fit in u32"); let metadata = lean_vm::blake3_flock::metadata(counter as u64, block_len as u32, flags as u32); - // Each operand's two words are at `base, base+1`; the flexible - // opcode addresses them independently (`blake3_input` forwards - // the real word sources where it can). + // Each operand is two 128-bit chunk cells; the flexible opcode + // addresses the four input cells independently (`blake3_input` + // forwards the real chunk sources where it can). The digest + // occupies the two consecutive output cells `c, g·c`. self.emit(LOp::Blake3 { ins: [a[0], a[1], b[0], b[1]], cv, @@ -1996,6 +2299,28 @@ impl FnLower<'_> { } return; } + if f == "pack64x2_into" { + assert_eq!(args.len(), 3, "pack64x2_into(a, b, out) takes three scalar cells"); + let a = self.expr(&args[0]); + let b = self.expr(&args[1]); + let c = self.expr(&args[2]); + self.emit(LOp::Pack64x2 { a, b, c }); + return; + } + if f == "hint_f192_limbs" { + assert_eq!(args.len(), 2, "hint_f192_limbs(dest, value)"); + let (base, len) = self + .stack_of(&args[0]) + .expect("hint_f192_limbs destination must be a StackBuf"); + assert!( + (1..=3).contains(&len), + "hint_f192_limbs destination must have 1..=3 cells" + ); + let value = self.expr(&args[1]); + let value = self.word_src(value); + self.pending.push(Hint::FieldLimbs { value, base, len }); + return; + } self.call(f, args, 0); } Stmt::Store(arr, idx, val) => { @@ -2087,8 +2412,32 @@ impl FnLower<'_> { return; // a `return` in main is a no-op; main halts via the trailing sentinel jump (lower_func). } let ret_base = 2 + self.n_args; - for (i, e) in exprs.iter().enumerate() { - self.expr_into(e, ret_base + i as u32); + assert_eq!( + exprs.len(), + self.return_shapes.len(), + "function returns {} values here, but its ABI declares {}", + exprs.len(), + self.return_shapes.len() + ); + // Each logical value lands straight in its flattened return area. A + // StackBuf is copied cell-by-cell because its callee-frame offsets are + // not meaningful after control returns to the caller. + let mut ret = ret_base; + for (e, shape) in exprs.iter().zip(self.return_shapes.clone()) { + match shape { + ReturnShape::Scalar => self.expr_into(e, ret), + ReturnShape::StackBuf(size) => { + let (base, actual) = self + .stack_of(e) + .unwrap_or_else(|| panic!("expected a StackBuf({size}) return, got `{e:?}`")); + assert_eq!(actual, size, "returned StackBuf has size {actual}, expected {size}"); + for k in 0..size { + let src = self.word_src(base + k); + self.copy(src, ret + k); + } + } + } + ret += shape.cells(); } let one = self.one(); self.emit(LOp::Jump { oc: one, od: 0, of: 1 }); @@ -2112,6 +2461,13 @@ impl FnLower<'_> { let id = *self.loop_ctr; *self.loop_ctr += 1; let loop_name = format!("__loop{id}"); + if std::env::var("DBG_LOOPS").is_ok() { + let bound = match hi { + ForBound::Const(h) => format!("g^{lo}..g^{h}"), + ForBound::Runtime(e) => format!("g^{lo}..{e:?}"), + }; + eprintln!("DBG_LOOPS {loop_name} in {} for {var} in {bound}", self.fn_name); + } // A runtime stop bound is evaluated once here and threaded through the // helper as an extra leading parameter (the exit test compares the // advanced counter against it each iteration). @@ -2192,6 +2548,7 @@ impl FnLower<'_> { params, const_params, n_ret: 0, + return_shapes: vec![], body: loop_body, inline: false, }); @@ -2230,15 +2587,15 @@ fn exprs_eq(a: &[Expr], b: &[Expr]) -> bool { } /// A body safe to inline: a single **tail** `return`, and no construct whose -/// lowering needs its own frame or a dispatch — a call to a user function, a -/// runtime loop, or a match (which would recurse the inliner or reload a frame -/// pointer that is no longer the callee's). `blake3` is a builtin statement and -/// is fine; `unroll`/`if` are compile-time / same-frame and recurse into. -fn body_inlinable(body: &[Stmt]) -> bool { - matches!(body.split_last(), Some((Stmt::Return(_), rest)) if rest.iter().all(stmt_inline_safe)) +/// lowering needs its own frame or a dispatch — a non-inline user call, a +/// runtime loop, or a match (which would reload a frame pointer that is no +/// longer the callee's). Builtins and nested `@inline` calls are fine; +/// `unroll`/`if` are compile-time / same-frame and recurse into. +fn body_inlinable(body: &[Stmt], defs: &HashMap) -> bool { + matches!(body.split_last(), Some((Stmt::Return(_), rest)) if rest.iter().all(|s| stmt_inline_safe(s, defs))) } -fn stmt_inline_safe(s: &Stmt) -> bool { +fn stmt_inline_safe(s: &Stmt, defs: &HashMap) -> bool { match s { Stmt::Let(..) | Stmt::Store(..) @@ -2247,9 +2604,13 @@ fn stmt_inline_safe(s: &Stmt) -> bool { | Stmt::AssertEq(..) | Stmt::AssertNe(..) | Stmt::AssertLt(..) => true, - Stmt::Call(f, _) => f == "blake3", - Stmt::If { then, els, .. } => then.iter().all(stmt_inline_safe) && els.iter().all(stmt_inline_safe), - Stmt::Unroll { body, .. } => body.iter().all(stmt_inline_safe), + Stmt::Call(f, _) => { + f == "blake3" || f == "pack64x2_into" || f == "hint_f192_limbs" || defs.get(f).is_some_and(|d| d.inline) + } + Stmt::If { then, els, .. } => { + then.iter().all(|s| stmt_inline_safe(s, defs)) && els.iter().all(|s| stmt_inline_safe(s, defs)) + } + Stmt::Unroll { body, .. } => body.iter().all(|s| stmt_inline_safe(s, defs)), // Return (non-tail), For, Match, LetMatchRange, LetTuple, CallIfNe, user Call. _ => false, } @@ -2371,32 +2732,40 @@ pub(crate) fn lower_func( queue: &mut Vec, loop_ctr: &mut usize, defs: &HashMap, - const_arrays: &HashMap>, + const_arrays: &HashMap>, ) -> Lowered { let mut vars = HashMap::new(); for (i, p) in f.params.iter().enumerate() { vars.insert(p.clone(), 2 + i as u32); } - // Reserve [0,1] retpc/retfp, params, then return slots, then locals. - let next = 2 + f.params.len() as u32 + f.n_ret as u32; + // Reserve [0,1] retpc/retfp, params, then the flattened return area, then + // locals. A StackBuf(n) return occupies n consecutive physical slots. + let n_ret_cells: u32 = f.return_shapes.iter().map(|s| s.cells()).sum(); + let next = 2 + f.params.len() as u32 + n_ret_cells; let mut lowerer = FnLower { vars, stacks: HashMap::new(), consts: HashMap::new(), next, n_args: f.params.len() as u32, + return_shapes: f.return_shapes.clone(), is_main: f.name == "main", + fn_name: f.name.clone(), tail_call: false, code: Vec::new(), - const_pool: HashMap::new(), + one_off: None, heap_sizes: HashMap::new(), self_fp_off: None, + bounds: HashMap::new(), + const_cells: HashMap::new(), gaddrs: HashMap::new(), fconsts: HashMap::new(), inline_ret: None, inline_stack_ret: None, alias: HashMap::new(), + zero_off: None, pending: Vec::new(), + inline_calls: Vec::new(), blake3_iv: None, queue, loop_ctr, @@ -2425,5 +2794,6 @@ pub(crate) fn lower_func( name: f.name.clone(), code: lowerer.code, frame_size: lowerer.next, + abi_end: 2 + f.params.len() as u32 + n_ret_cells, } } diff --git a/crates/lean_compiler/src/parser.rs b/crates/lean_compiler/src/parser.rs index bb37e922..a6d2784f 100644 --- a/crates/lean_compiler/src/parser.rs +++ b/crates/lean_compiler/src/parser.rs @@ -18,7 +18,7 @@ pub fn parse(src: &str) -> Result { } /// Like [`parse`], but first applies compile-time **placeholder** replacements -/// (identifier-level text substitution — see [`apply_replacements`]). This is +/// (identifier-level text substitution — see `apply_replacements`). This is /// how a host injects sizes/flags into a program without editing it: write a /// placeholder identifier in source and map it to a value at compile time. The /// idiom is a placeholder feeding a named constant: @@ -32,8 +32,8 @@ pub fn parse(src: &str) -> Result { /// **Global constants.** Between the (optional) `snark_lib` import and the /// `def`s, the top level accepts constant declarations `NAME = `, /// where `` is a compile-time **integer** — decimal literals combined -/// with `+ - * / **` and parentheses (ordinary integer arithmetic, *not* the -/// runtime field's XOR/GHASH), and references to *earlier* constants. So a +/// with `+ - * / **` and parentheses (ordinary integer arithmetic, not runtime +/// field arithmetic), and references to *earlier* constants. So a /// derived size like `N_TWEAKS = 2 + (W - 1) * V + LOG_LIFETIME` comes out /// right. Each constant is evaluated and substituted, as a single decimal /// literal, everywhere its name appears in the functions below — so a constant @@ -70,7 +70,7 @@ pub fn parse_with_replacements(src: &str, replacements: &BTreeMap = BTreeMap::new(); - let mut const_arrays: Vec<(String, Vec)> = Vec::new(); + let mut const_arrays: Vec<(String, Vec)> = Vec::new(); let mut start = 0; while start < lines.len() { let (indent, line) = &lines[start]; @@ -81,11 +81,16 @@ pub fn parse_with_replacements(src: &str, replacements: &BTreeMap> 64) as u64, 0) + }; + elems.push(elem); } const_arrays.push((name, elems)); } else { // A scalar constant: evaluate it as a compile-time integer. - let value = eval_const_int(rhs).map_err(|e| format!("global constant `{name}`: {e}"))?; - consts.insert(name, value.to_string()); + if let Some(value) = parse_f192_const(rhs) { + let v = value.map_err(|e| format!("global constant `{name}`: {e}"))?; + consts.insert(name, format!("f192({},{},{})", v.c0, v.c1, v.c2)); + } else { + let value = eval_const_int(rhs).map_err(|e| format!("global constant `{name}`: {e}"))?; + consts.insert(name, value.to_string()); + } } start += 1; } @@ -118,14 +134,125 @@ pub fn parse_with_replacements(src: &str, replacements: &BTreeMap, + known: &HashMap>, + ) -> ReturnShape { + match e { + Expr::Var(v) => locals.get(v).copied().unwrap_or(ReturnShape::Scalar), + Expr::StackBuf(n) => ReturnShape::StackBuf((*n).try_into().expect("StackBuf size does not fit in u32")), + Expr::ListLit(es) => ReturnShape::StackBuf(es.len().try_into().expect("StackBuf size does not fit in u32")), + Expr::Call(f, _) => known + .get(f) + .filter(|r| r.len() == 1) + .and_then(|r| r.first()) + .copied() + .unwrap_or(ReturnShape::Scalar), + _ => ReturnShape::Scalar, + } + } + + fn scan( + body: &[Stmt], + params: &[String], + known: &HashMap>, + n_ret: usize, + ) -> Vec { + let mut locals: HashMap = + params.iter().map(|p| (p.clone(), ReturnShape::Scalar)).collect(); + let mut returns = vec![ReturnShape::Scalar; n_ret]; + for stmt in body { + match stmt { + Stmt::Let(name, e) => { + locals.insert(name.clone(), expr_shape(e, &locals, known)); + } + Stmt::LetTuple(names, f, _) => { + let shapes = known.get(f); + for (i, name) in names.iter().enumerate() { + let shape = shapes.and_then(|s| s.get(i)).copied().unwrap_or(ReturnShape::Scalar); + locals.insert(name.clone(), shape); + } + } + // `unroll` is straight-line expansion, so a binding in its last + // copy remains visible afterward. One symbolic scan is enough + // for representation shapes (the iteration value is scalar). + Stmt::Unroll { var, body, .. } => { + locals.insert(var.clone(), ReturnShape::Scalar); + for inner in body { + if let Stmt::Let(name, e) = inner { + locals.insert(name.clone(), expr_shape(e, &locals, known)); + } + } + } + Stmt::Return(es) => { + returns = es.iter().map(|e| expr_shape(e, &locals, known)).collect(); + } + _ => {} + } + } + returns + } + + let mut known: HashMap> = funcs + .iter() + .map(|f| (f.name.clone(), vec![ReturnShape::Scalar; f.n_ret])) + .collect(); + // A shape can only move from Scalar to one of the finite constructor + // shapes (or acquire one through a call), so `funcs.len() + 1` rounds are + // sufficient for the longest acyclic wrapper chain. + for _ in 0..=funcs.len() { + let next: HashMap> = funcs + .iter() + .map(|f| (f.name.clone(), scan(&f.body, &f.params, &known, f.n_ret))) + .collect(); + if next == known { + known = next; + break; + } + known = next; + } + for f in funcs { + f.return_shapes = known + .remove(&f.name) + .unwrap_or_else(|| vec![ReturnShape::Scalar; f.n_ret]); + } +} + +fn parse_f192_const(s: &str) -> Option> { + let inner = s.trim().strip_prefix("f192(")?.strip_suffix(')')?; + let parts = split_top(inner, ','); + Some((|| { + if parts.len() != 3 { + return Err("f192 needs exactly three limbs".into()); + } + let mut limbs = [0u64; 3]; + for (i, p) in parts.iter().enumerate() { + limbs[i] = + u64::try_from(eval_const_int(p.trim())?).map_err(|_| "an f192 limb does not fit in u64".to_string())?; + } + Ok(F192::new(limbs[0], limbs[1], limbs[2])) + })()) +} + /// Apply identifier-level **placeholder** replacements to source text before /// parsing: each maximal run of identifier characters (`[A-Za-z0-9_]`) that /// equals a key of `replacements` is replaced by its value; other text — @@ -167,8 +294,8 @@ fn is_ident(s: &str) -> bool { /// Evaluate a compile-time **integer** constant expression: decimal literals /// combined with `+`, `-`, `*`, `/` (truncating), `**` (power), and /// parentheses. This is ordinary integer arithmetic — a global constant is a -/// count / size / exponent — deliberately *distinct* from the runtime field's -/// `+` = XOR and `*` = GHASH, so derived sizes like `2 + (W - 1) * V + +/// count / size / exponent — deliberately *distinct* from runtime field +/// arithmetic, so derived sizes like `2 + (W - 1) * V + /// LOG_LIFETIME` come out right. All references to earlier constants have /// already been substituted to their decimal values, so the input is pure /// arithmetic. Overflow, division by zero, and a negative intermediate are @@ -268,7 +395,8 @@ fn eval_const_int(s: &str) -> Result { *p += 1; let exp = power(t, p)?; // right-associative let exp = u32::try_from(exp).map_err(|_| "`**` exponent too large".to_string())?; - base.checked_pow(exp).ok_or_else(|| "constant overflow in `**`".to_string()) + base.checked_pow(exp) + .ok_or_else(|| "constant overflow in `**`".to_string()) } else { Ok(base) } @@ -279,7 +407,8 @@ fn eval_const_int(s: &str) -> Result { *p += 1; let rhs = power(t, p)?; acc = if op == Tok::Mul { - acc.checked_mul(rhs).ok_or_else(|| "constant overflow in `*`".to_string())? + acc.checked_mul(rhs) + .ok_or_else(|| "constant overflow in `*`".to_string())? } else { acc.checked_div(rhs) .ok_or_else(|| "division by zero in constant expression".to_string())? @@ -293,9 +422,11 @@ fn eval_const_int(s: &str) -> Result { *p += 1; let rhs = term(t, p)?; acc = if op == Tok::Add { - acc.checked_add(rhs).ok_or_else(|| "constant overflow in `+`".to_string())? + acc.checked_add(rhs) + .ok_or_else(|| "constant overflow in `+`".to_string())? } else { - acc.checked_sub(rhs).ok_or_else(|| "constant is negative (underflow in `-`)".to_string())? + acc.checked_sub(rhs) + .ok_or_else(|| "constant is negative (underflow in `-`)".to_string())? }; } Ok(acc) @@ -303,7 +434,10 @@ fn eval_const_int(s: &str) -> Result { let mut pos = 0; let value = expr(&toks, &mut pos)?; if pos != toks.len() { - return Err(format!("unexpected trailing tokens in constant expression `{}`", s.trim())); + return Err(format!( + "unexpected trailing tokens in constant expression `{}`", + s.trim() + )); } Ok(value) } @@ -312,12 +446,13 @@ fn eval_const_int(s: &str) -> Result { /// `GEN ** k`, and `+`/`*` combinations of those — to its field element. /// Used for the `# public_input: , ` annotation of `.py` test /// programs (see `tests/py_source.rs`). -pub fn parse_const(s: &str) -> Result { - fn eval(e: &Expr) -> Result { +pub fn parse_const(s: &str) -> Result { + fn eval(e: &Expr) -> Result { match e { - Expr::Lit(n) => Ok(F128::new(*n as u64, (*n >> 64) as u64)), - Expr::Gen => Ok(g_pow(1)), - Expr::GPow(k) => Ok(g_pow_u128(*k)), + // An integer literal is the raw 128-bit bit pattern of a machine word. + Expr::Lit(n) => Ok(F192::new(*n as u64, (*n >> 64) as u64, 0)), + Expr::Gen => Ok(g_pow(1).into()), + Expr::GPow(k) => Ok(g_pow_u128(*k).into()), Expr::Add(a, b) => Ok(eval(a)? + eval(b)?), Expr::Mul(a, b) => Ok(eval(a)? * eval(b)?), other => Err(format!("not a constant expression: `{other:?}`")), @@ -352,7 +487,11 @@ impl Parser { return Err(format!("unknown decorator `@{}` (only `@inline`)", dec.trim())); } self.i += 1; - (indent, line) = self.lines.get(self.i).cloned().ok_or("`@inline` must precede a `def`")?; + (indent, line) = self + .lines + .get(self.i) + .cloned() + .ok_or("`@inline` must precede a `def`")?; true } else { false @@ -395,6 +534,7 @@ impl Parser { params, const_params, n_ret, + return_shapes: vec![ReturnShape::Scalar; n_ret], body, inline, }) @@ -1104,7 +1244,11 @@ fn parse_expr(s: &str) -> Result { for (op, seg) in ops.iter().zip(&segs[1..]) { let rhs = Box::new(parse_expr(seg)?); let lhs = Box::new(acc); - acc = if *op == b'+' { Expr::Add(lhs, rhs) } else { Expr::Sub(lhs, rhs) }; + acc = if *op == b'+' { + Expr::Add(lhs, rhs) + } else { + Expr::Sub(lhs, rhs) + }; } return Ok(acc); } @@ -1124,8 +1268,10 @@ fn parse_expr(s: &str) -> Result { } return Ok(acc); } - // `**` (compile-time power), tightest binding: `base ** k` with `k` a - // (possibly large) integer literal. + // `**` (compile-time power), tightest binding: `base ** k` with `k` an + // integer literal (possibly large), or a parenthesised compile-time + // integer expression like `GEN ** (2 * s + 1)`, evaluated at lowering, + // so it can reference `unroll` counters and constants. if let Some((base, exp)) = split_once_top(s, "**") { let base = parse_expr(&base)?; let exp_e = parse_expr(&exp)?; diff --git a/crates/lean_compiler/tests/assert_ne.rs b/crates/lean_compiler/tests/assert_ne.rs index 8de922df..9b6d801c 100644 --- a/crates/lean_compiler/tests/assert_ne.rs +++ b/crates/lean_compiler/tests/assert_ne.rs @@ -7,7 +7,7 @@ use lean_compiler::{compile, parse}; use lean_vm::cpu::{prove, verify}; -use primitives::field::{F128, g_pow}; +use primitives::field::{F64, F192, g_pow}; /// Honest inequality over runtime values: prove + verify pass, and corrupting /// the public output is still caught (the assert does not disturb the trace). @@ -26,12 +26,15 @@ def main(): return "; let program = compile(&parse(src).expect("parse")); - let want = [g_pow(12), g_pow(5)]; - let (proof, _) = prove(&program, want); + let want = [F192::from(g_pow(12)), F192::from(g_pow(5))]; + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); verify(&program, &want, &proof).expect("inequality program verifies"); - let bad = [g_pow(11), g_pow(5)]; - assert!(verify(&program, &bad, &proof).is_err(), "wrong public input must be rejected"); + let bad = [F192::from(g_pow(11)), F192::from(g_pow(5))]; + assert!( + verify(&program, &bad, &proof).is_err(), + "wrong public input must be rejected" + ); } /// The adversarial case: two hinted cells the prover sets *equal*, asserted @@ -49,12 +52,13 @@ def main(): p[GEN] = v[1] return "; - let run = |a: F128, b: F128| -> bool { + let run = |a: F64, b: F64| -> bool { let mut program = compile(&parse(src).expect("parse")); - program.set_witness("vals", vec![vec![a, b]]); + program.set_witness("vals", vec![vec![F192::from(a), F192::from(b)]]); std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let (proof, _) = prove(&program, [a, b]); - verify(&program, &[a, b], &proof).is_ok() + let pi = [F192::from(a), F192::from(b)]; + let (proof, _) = prove(&program, pi, lean_vm::pcs::LOG_INV_RATE); + verify(&program, &pi, &proof).is_ok() })) .unwrap_or(false) }; @@ -78,8 +82,8 @@ def main(): return "; let program = compile(&parse(src).expect("parse")); - let want = [F128::new(5, 0), F128::new(7, 0)]; - let (proof, _) = prove(&program, want); + let want = [F192::from(F64(5)), F192::from(F64(7))]; + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); verify(&program, &want, &proof).expect("loop inequality verifies"); } diff --git a/crates/lean_compiler/tests/const_placeholder.rs b/crates/lean_compiler/tests/const_placeholder.rs index 0ea31f22..050501f9 100644 --- a/crates/lean_compiler/tests/const_placeholder.rs +++ b/crates/lean_compiler/tests/const_placeholder.rs @@ -37,7 +37,11 @@ def main(): "; let ac = parse(with_const).expect("const program parses"); let ai = parse(inlined).expect("inlined program parses"); - assert_eq!(format!("{ac:?}"), format!("{ai:?}"), "constant must inline to its value"); + assert_eq!( + format!("{ac:?}"), + format!("{ai:?}"), + "constant must inline to its value" + ); let _ = compile(&ac); // and it lowers to a real program } @@ -93,8 +97,8 @@ def main(): let _ = compile(&parse(src).unwrap()); } -/// Constant expressions use **integer** arithmetic (`+ - * / **`), not the -/// runtime field's XOR/GHASH — so derived sizes/counts come out right. Filled +/// Constant expressions use **integer** arithmetic (`+ - * / **`), not runtime +/// field arithmetic, so derived sizes/counts come out right. Filled /// via placeholders, the whole set of derivations resolves to plain literals. #[test] fn const_integer_arithmetic_derivations() { @@ -198,7 +202,10 @@ def main(): return "; let err = parse(unfilled).expect_err("an unfilled placeholder must fail"); - assert!(err.contains("V_PLACEHOLDER"), "error should name the placeholder: {err}"); + assert!( + err.contains("V_PLACEHOLDER"), + "error should name the placeholder: {err}" + ); let dup = "\ N = 1 diff --git a/crates/lean_compiler/tests/cse.rs b/crates/lean_compiler/tests/cse.rs new file mode 100644 index 00000000..eb60384c --- /dev/null +++ b/crates/lean_compiler/tests/cse.rs @@ -0,0 +1,157 @@ +//! The value-numbering pass (`cse.rs`) folds away pure instructions the lowerer +//! emitted twice. These programs pin the cases where a "duplicate" is NOT dead. + +use lean_compiler::{compile, parse}; +use lean_vm::cpu::{prove, verify}; +use primitives::field::{F64, F192, g_pow}; + +/// A returned value that repeats a constant computed earlier in the same +/// function. The return slot lives in the callee frame and is read by the +/// CALLER, so eliminating that write leaves the caller reading an unwritten +/// (prover-chosen) cell: `walk` in the XMSS guest returned a flag exactly this +/// way, and folding it produced a proof whose caller-side assert failed. +#[test] +fn duplicate_constant_in_a_return_slot_survives() { + let src = "\ +def tag(x): + # `marker` is the same constant the flag below returns, and it is computed + # first, so the flag's `SET` is a textual duplicate of it. + marker = 7 + return x * marker, 7 + +def main(): + v, flag = tag(GEN ** 3) + p = 1 + p[1] = v + p[GEN] = flag + return +"; + let program = compile(&parse(src).expect("parse")); + let want = [F192::from(g_pow(3)) * F192::from(F64(7)), F192::from(F64(7))]; + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); + verify(&program, &want, &proof).expect("returned duplicate constant is preserved"); +} + +/// An argument slot written with a value that already exists in the caller: the +/// callee reads its arguments out of its own frame, so the store must stay. +#[test] +fn duplicate_argument_value_survives() { + let src = "\ +def add_both(a, b): + return a + b + +def main(): + k = GEN ** 5 + # Both arguments are the same expression, and the sum is computed here too, + # so every operand the call needs has a duplicate in this frame. + local = k + k + s = add_both(k, k) + p = 1 + p[1] = s + local + return +"; + let program = compile(&parse(src).expect("parse")); + // (k + k) + (k + k) == 0 in characteristic two. + let want = [F192::ZERO, F192::ZERO]; + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); + verify(&program, &want, &proof).expect("duplicated call arguments are preserved"); +} + +/// A duplicate on one side of a branch must not be folded into the other side's +/// computation: the value map is cleared at block boundaries, so each arm +/// recomputes what it needs. If it were folded, the untaken arm's cell would be +/// unwritten at the join. +#[test] +fn duplicates_are_not_folded_across_a_branch() { + let src = "\ +def main(): + x = GEN ** 3 + r = HeapBuf(2) + # The same constant in both arms: folding the second into the first would + # make the taken path store from a cell the untaken path was to write. + if x == GEN ** 3: + r[1] = GEN ** 4 + else: + r[1] = GEN ** 4 + p = 1 + p[1] = r[1] + p[GEN] = x + return +"; + let program = compile(&parse(src).expect("parse")); + let want = [F192::from(g_pow(4)), F192::from(g_pow(3))]; + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); + verify(&program, &want, &proof).expect("both branches keep their own constant"); +} + +/// The assert idiom is `XOR fp[t] = a ^ b` then `SET fp[t] = 0`, which panics as +/// a write-once conflict when `a != b`. Both instructions write `fp[t]`, so +/// neither may be folded away — otherwise a failing assert would silently pass. +/// Here the compared difference is also computed as an ordinary value, giving the +/// assert's `XOR` a duplicate to be folded into. +#[test] +fn assert_survives_a_duplicated_comparison() { + let src = "\ +def main(): + a = GEN ** 9 + b = GEN ** 9 + # The same XOR the assert needs, as a live value. + diff = a + b + assert a == b + p = 1 + p[1] = diff + p[GEN] = a + return +"; + let program = compile(&parse(src).expect("parse")); + let want = [F192::ZERO, F192::from(g_pow(9))]; + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); + verify(&program, &want, &proof).expect("passing assert still verifies"); +} + +/// The same shape, but with the assert failing: it must still panic (the `SET` +/// that collides with the `XOR` was not eliminated). +#[test] +#[should_panic(expected = "write-once conflict")] +fn failing_assert_still_conflicts() { + let src = "\ +def main(): + a = GEN ** 9 + b = GEN ** 10 + diff = a + b + assert a == b + p = 1 + p[1] = diff + p[GEN] = a + return +"; + let program = compile(&parse(src).expect("parse")); + let want = [F192::ZERO, F192::from(g_pow(9))]; + let _ = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); +} + +/// A hint at the end of a runtime branch is attached to a no-op anchor by the +/// lowerer. Even when that anchor repeats an earlier pure instruction, CSE must +/// retain it: moving the hint to the next textual instruction would move it to +/// the join and execute it when the branch is not taken. +#[test] +fn trailing_branch_hint_stays_in_its_branch() { + let src = "\ +def main(): + flag = StackBuf(1) + hint_witness(flag, \"flag\") + data = StackBuf(1) + if flag[0] == 1: + print(\"anchor\", flag[0]) + hint_witness(data, \"data\") + p = 1 + p[1] = flag[0] + p[GEN] = 0 + return +"; + let mut program = compile(&parse(src).expect("parse")); + program.set_witness("flag", vec![vec![F192::ZERO]]); + let want = [F192::ZERO, F192::ZERO]; + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); + verify(&program, &want, &proof).expect("untaken branch must not consume its witness"); +} diff --git a/crates/lean_compiler/tests/disassemble.rs b/crates/lean_compiler/tests/disassemble.rs index 26079549..529450ba 100644 --- a/crates/lean_compiler/tests/disassemble.rs +++ b/crates/lean_compiler/tests/disassemble.rs @@ -1,4 +1,5 @@ use lean_compiler::{compile, disassemble, parse}; +use primitives::pretty_integer; #[test] fn disassemble_simple_program() { @@ -19,9 +20,9 @@ def main(): println!("\n=== zkDSL source ===\n{src}"); println!( "=== compiled ISA ({} instructions, pc0 = {}, fp0 = {}) ===", - program.prog.len(), - program.pc0, - program.fp0, + pretty_integer(program.prog.len()), + pretty_integer(program.pc0), + pretty_integer(program.fp0), ); print!("{}", disassemble(&program.prog)); diff --git a/crates/lean_compiler/tests/field_div.rs b/crates/lean_compiler/tests/field_div.rs index 729dbd25..dc508786 100644 --- a/crates/lean_compiler/tests/field_div.rs +++ b/crates/lean_compiler/tests/field_div.rs @@ -7,7 +7,7 @@ use lean_compiler::{compile, parse}; use lean_vm::cpu::{prove, verify}; -use primitives::field::{F128, g_pow}; +use primitives::field::{F64, F192, g_pow}; /// `a / b` and `1 / b` over runtime values: the quotient satisfies `q·b == a`, /// checked by publishing it and reproducing the dividend. @@ -26,12 +26,15 @@ def main(): "; let program = compile(&parse(src).expect("parse")); // q·b must reproduce a = g^20; r·b must be 1. - let want = [g_pow(20), F128::ONE]; - let (proof, _) = prove(&program, want); + let want = [F192::from(g_pow(20)), F192::from(F64::ONE)]; + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); verify(&program, &want, &proof).expect("division program verifies"); - let bad = [g_pow(21), F128::ONE]; - assert!(verify(&program, &bad, &proof).is_err(), "wrong quotient product rejected"); + let bad = [F192::from(g_pow(21)), F192::from(F64::ONE)]; + assert!( + verify(&program, &bad, &proof).is_err(), + "wrong quotient product rejected" + ); } /// `//` stays compile-time floor division (an index), `/` is the runtime field @@ -51,8 +54,8 @@ def main(): "; let program = compile(&parse(src).expect("parse")); // q = g^6 / g^2 = g^4 (runtime `/`); z = g^(6//2) = g^3 (compile-time `//`). - let want = [g_pow(4), g_pow(3)]; - let (proof, _) = prove(&program, want); + let want = [F192::from(g_pow(4)), F192::from(g_pow(3))]; + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); verify(&program, &want, &proof).expect("mixed //-and-/ program verifies"); } @@ -70,15 +73,19 @@ def main(): p[GEN] = 1 return "; - let run = |den: F128| -> bool { + let run = |den: F64| -> bool { let mut program = compile(&parse(src).expect("parse")); - program.set_witness("den", vec![vec![den]]); + program.set_witness("den", vec![vec![F192::from(den)]]); std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let (proof, _) = prove(&program, [F128::ONE, F128::ONE]); - verify(&program, &[F128::ONE, F128::ONE], &proof).is_ok() + let (proof, _) = prove( + &program, + [F192::from(F64::ONE), F192::from(F64::ONE)], + lean_vm::pcs::LOG_INV_RATE, + ); + verify(&program, &[F192::from(F64::ONE), F192::from(F64::ONE)], &proof).is_ok() })) .unwrap_or(false) }; assert!(run(g_pow(4)), "nonzero divisor must verify"); - assert!(!run(F128::ZERO), "zero divisor must be rejected"); + assert!(!run(F64::ZERO), "zero divisor must be rejected"); } diff --git a/crates/lean_compiler/tests/field_towers.rs b/crates/lean_compiler/tests/field_towers.rs new file mode 100644 index 00000000..09035f20 --- /dev/null +++ b/crates/lean_compiler/tests/field_towers.rs @@ -0,0 +1,59 @@ +//! Integration coverage for the >128-bit field F192 = GF((2^64)^3), through +//! the same `primitives` re-export path the VM uses. The deep suite +//! (NEON-vs-reference on 10k random inputs, independent Python-generated +//! vectors, Frobenius/gcd irreducibility proofs of both moduli) lives in +//! `primitives::field::gf2_64x3`; this file keeps the main-workspace +//! `cargo test` honest about the essentials. + +use primitives::field::{F192, F192Unreduced}; +use rand::Rng; + +fn rand_f192(rng: &mut impl Rng) -> F192 { + F192::new(rng.random(), rng.random(), rng.random()) +} + +#[test] +fn f192_field_behaviour() { + let mut rng = rand::rng(); + for _ in 0..500 { + let (a, b, c) = (rand_f192(&mut rng), rand_f192(&mut rng), rand_f192(&mut rng)); + // ring axioms + agreement with the portable reference + assert_eq!(a * b, primitives::field::gf2_64x3::software::mul(a, b)); + assert_eq!(a * b, b * a); + assert_eq!((a * b) * c, a * (b * c)); + assert_eq!(a * (b + c), a * b + a * c); + assert_eq!(a.square(), a * a); + if !a.is_zero() { + assert_eq!(a * a.inv(), F192::ONE); + } + } + // y^3 = y + 1 (the defining relation) + assert_eq!(F192::Y * F192::Y * F192::Y, F192::Y + F192::ONE); +} + +#[test] +fn deferred_reduction_matches_reduced_sums() { + let mut rng = rand::rng(); + + let mut acc192 = F192Unreduced::ZERO; + let mut want192 = F192::ZERO; + for _ in 0..256 { + let (a, b) = (rand_f192(&mut rng), rand_f192(&mut rng)); + acc192 ^= a.mul_unreduced(b); + want192 += a * b; + } + assert_eq!(acc192.reduce(), want192); +} + +/// `a.square() = a * a` in `K[y]/(y³+y+1)`. +#[test] +fn tower_square_matches_mul() { + let mut rng = rand::rng(); + for _ in 0..500 { + let a = rand_f192(&mut rng); + assert_eq!(a.square(), a * a); + if !a.is_zero() { + assert_eq!(a * a.inv(), F192::ONE); + } + } +} diff --git a/crates/lean_compiler/tests/hint_log2_ceil.rs b/crates/lean_compiler/tests/hint_log2_ceil.rs index ed3d4b2d..a94b9ae3 100644 --- a/crates/lean_compiler/tests/hint_log2_ceil.rs +++ b/crates/lean_compiler/tests/hint_log2_ceil.rs @@ -6,7 +6,7 @@ use lean_compiler::{compile, parse}; use lean_vm::cpu::{prove, verify}; -use primitives::field::{F128, g_pow}; +use primitives::field::{F64, F192, g_pow}; fn log2_ceil_of(v: u128) -> usize { if v <= 1 { @@ -30,13 +30,16 @@ def main(): "; for v in [1u128, 2, 3, 4, 5, 7, 8, 200] { let mut program = compile(&parse(src).expect("parse")); - let bits: Vec = (0..8).map(|j| F128::new(((v >> j) & 1) as u64, 0)).collect(); + let bits: Vec = (0..8).map(|j| F192::from(F64(((v >> j) & 1) as u64))).collect(); program.set_witness("bits", vec![bits]); - let want = [g_pow(log2_ceil_of(v)), F128::ONE]; - let (proof, _) = prove(&program, want); + let want = [F192::from(g_pow(log2_ceil_of(v))), F192::from(F64::ONE)]; + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); verify(&program, &want, &proof).unwrap_or_else(|_| panic!("v={v}: log2_ceil advice must verify")); - let bad = [g_pow(log2_ceil_of(v) + 1), F128::ONE]; - assert!(verify(&program, &bad, &proof).is_err(), "v={v}: wrong g_mu must be rejected"); + let bad = [F192::from(g_pow(log2_ceil_of(v) + 1)), F192::from(F64::ONE)]; + assert!( + verify(&program, &bad, &proof).is_err(), + "v={v}: wrong g_mu must be rejected" + ); } } @@ -56,10 +59,10 @@ def main(): "; for (v, mu) in [(2u128, 5usize), (4, 5), (64, 6), (200, 8)] { let mut program = compile(&parse(src).expect("parse")); - let bits: Vec = (0..8).map(|j| F128::new(((v >> j) & 1) as u64, 0)).collect(); + let bits: Vec = (0..8).map(|j| F192::from(F64(((v >> j) & 1) as u64))).collect(); program.set_witness("bits", vec![bits]); - let want = [g_pow(mu), F128::ONE]; - let (proof, _) = prove(&program, want); + let want = [F192::from(g_pow(mu)), F192::from(F64::ONE)]; + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); verify(&program, &want, &proof).unwrap_or_else(|_| panic!("v={v}: floored log2_ceil must verify")); } } diff --git a/crates/lean_compiler/tests/inline_expr.rs b/crates/lean_compiler/tests/inline_expr.rs index 1bffe50c..236b6635 100644 --- a/crates/lean_compiler/tests/inline_expr.rs +++ b/crates/lean_compiler/tests/inline_expr.rs @@ -9,7 +9,7 @@ use lean_compiler::{compile, parse}; use lean_vm::blake3_flock::warm_setup; use lean_vm::cpu::{prove, verify}; -use primitives::field::F128; +use primitives::field::{F64, F192}; #[test] fn inline_call_in_expression_positions() { @@ -42,20 +42,23 @@ def main(): let program = compile(&parse(src).expect("parse")); warm_setup(1); - let (f3, f5, f7) = (F128::new(3, 0), F128::new(5, 0), F128::new(7, 0)); - let one = F128::ONE; + let (f3, f5, f7) = (F64(3), F64(5), F64(7)); + let one = F64::ONE; // statement position: idx 2 -> (1+3)·5 let x = (one + f3) * f5; // embedded in a product: idx 1 -> 7·(3·(1+5)) let y = f7 * (f3 * (one + f5)); // heap-store RHS: idx 3 -> 3·5 let o = f3 * f5; - let want = [x, y + o]; + let want = [F192::from(x), F192::from(y + o)]; - let (proof, _) = prove(&program, want); + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); verify(&program, &want, &proof).expect("expression-position inline calls compute correctly"); let mut bad = want; - bad[0] += F128::ONE; - assert!(verify(&program, &bad, &proof).is_err(), "wrong published value must be rejected"); + bad[0] += F192::ONE; + assert!( + verify(&program, &bad, &proof).is_err(), + "wrong published value must be rejected" + ); } diff --git a/crates/lean_compiler/tests/pack64x2.rs b/crates/lean_compiler/tests/pack64x2.rs new file mode 100644 index 00000000..4db57275 --- /dev/null +++ b/crates/lean_compiler/tests/pack64x2.rs @@ -0,0 +1,42 @@ +use lean_compiler::{compile, parse}; +use lean_vm::blake3_flock::warm_setup; +use lean_vm::cpu::{prove, verify}; +use primitives::field::{F64, F192}; + +#[test] +fn pack64x2_proves_and_verifies() { + let src = "\ +def main(): + a = 5 + b = 7 + packed = pack64x2(a, b) + p = 1 + p[1] = packed + p[GEN] = packed + return +"; + let program = compile(&parse(src).expect("parse")); + warm_setup(1); + let want = [F192::new(5, 7, 0), F192::new(5, 7, 0)]; + let (proof, stats) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); + assert_eq!(stats.counts[6], 1, "one PACK64X2 instruction"); + verify(&program, &want, &proof).expect("PACK64X2 program verifies"); +} + +#[test] +#[should_panic(expected = "PACK64X2 first input must be K-valued")] +fn pack64x2_rejects_extension_field_source() { + let src = "\ +def main(): + a = StackBuf(1) + hint_witness(a[0:1], \"a\") + packed = pack64x2(a[0], 7) + p = 1 + p[1] = packed + p[GEN] = packed + return +"; + let mut program = compile(&parse(src).expect("parse")); + program.set_witness("a", vec![vec![F192::new(5, 1, 0)]]); + let _ = program.execute([F192::from(F64::ONE), F192::from(F64::ONE)]); +} diff --git a/crates/lean_compiler/tests/print_debug.rs b/crates/lean_compiler/tests/print_debug.rs index d7ff2f5c..e9fa5a91 100644 --- a/crates/lean_compiler/tests/print_debug.rs +++ b/crates/lean_compiler/tests/print_debug.rs @@ -4,7 +4,7 @@ use lean_compiler::{compile, parse}; use lean_vm::blake3_flock::warm_setup; use lean_vm::cpu::{prove, verify}; -use primitives::field::F128; +use primitives::field::{F64, F192}; #[test] fn print_is_constraint_free() { @@ -24,7 +24,7 @@ def main(): "; let program = compile(&parse(src).expect("parse")); warm_setup(1); - let want = [F128::new(5, 0) * primitives::field::g_pow(1), F128::new(3, 0)]; - let (proof, _) = prove(&program, want); + let want = [F192::from(F64(5) * primitives::field::g_pow(1)), F192::from(F64(3))]; + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); verify(&program, &want, &proof).expect("prints must not disturb proving"); } diff --git a/crates/lean_compiler/tests/programs/blake3_chain.py b/crates/lean_compiler/tests/programs/blake3_chain.py index 0592f4d9..8a3fa9b8 100644 --- a/crates/lean_compiler/tests/programs/blake3_chain.py +++ b/crates/lean_compiler/tests/programs/blake3_chain.py @@ -1,8 +1,8 @@ # A short BLAKE3 chain over 256-bit stack values: blake3(h, h, h2), twice. -# Each StackBuf(2) holds the two 128-bit words of a 256-bit value in two -# consecutive frame cells, read in place by the BLAKE3 instruction, which -# writes the digest into the pre-allocated output pair. The public input is -# the chain's digest, BLAKE3²(5, 7), as two 128-bit words. +# A 256-bit BLAKE3 value uses two canonical 128-bit cells (StackBuf(2)); each +# scalar cell holds one word in its low lane, so `h = [5, 7]` hashes the words +# [5, 0, 7, 0]. The digest lands in the pre-allocated pair. The public input is +# the two 128-bit digest cells of the chain's result, BLAKE3(BLAKE3(·)). # public_input: 101229015297003380629709256178361811305, 199495362546883507010283175921733252645 from snark_lib import * diff --git a/crates/lean_compiler/tests/programs/blake3_heap_chain.py b/crates/lean_compiler/tests/programs/blake3_heap_chain.py index 318c52cf..4491b43a 100644 --- a/crates/lean_compiler/tests/programs/blake3_heap_chain.py +++ b/crates/lean_compiler/tests/programs/blake3_heap_chain.py @@ -1,7 +1,9 @@ # Runtime slices: `buf[i:i + 2]` with a runtime g-power index `i` names the -# heap cells `buf·i`, `buf·i·g` (one MUL folds `i` into the pointer). A BLAKE3 -# chain over heap pairs, addressed by the loop counter: pair k sits at cells -# g^{2k}, g^{2k+1}, and pair k+1 = H(pair k, pair k). Published: H³(5, 7). +# heap cells `buf·i·g^k`, k < 2 (one MUL folds `i` into the pointer). A BLAKE3 +# chain over heap pairs (256-bit BLAKE3 value = two canonical cells), +# addressed by the loop counter: value k sits at cells g^{2k}..g^{2k+1}, and +# value k+1 = H(value k, value k). Published: the two 128-bit digest cells of +# H^3(5, 7). # public_input: 259899574965733219954697446670390340005, 71594800443637044304569228067009621691 from snark_lib import * @@ -11,7 +13,7 @@ def main(): buf[1] = 5 buf[GEN] = 7 for i in mul_range(1, GEN ** 3): - b = i * i # pair k at cells g^{2k}, g^{2k+1} + b = i * i # value k at cells g^{2k}..g^{2k+1} blake3(buf[b:b + 2], buf[b:b + 2], buf[b * GEN ** 2:b * GEN ** 2 + 2]) p = GEN ** 0 p[1] = buf[GEN ** 6] diff --git a/crates/lean_compiler/tests/programs/blake3_slices.py b/crates/lean_compiler/tests/programs/blake3_slices.py index 88044170..b2907808 100644 --- a/crates/lean_compiler/tests/programs/blake3_slices.py +++ b/crates/lean_compiler/tests/programs/blake3_slices.py @@ -1,19 +1,22 @@ -# BLAKE3 over slices: `buf[lo:hi]` (2 cells) is a 256-bit operand, with -# compile-time bounds — literals, literal-bound names, and their integer -# arithmetic (`x:x + 2`). Slices work on a large StackBuf (in place) and on a -# HeapBuf (bridged through the stack, one DEREF per cell), as inputs and as -# the output. Published: H(H(a[0:2], hb[0:2]), a[0:2]) read back from the heap. +# BLAKE3 over slices: `buf[lo:hi]` (2 cells) is a 256-bit operand under 128-bit +# machine words, with compile-time bounds — literals, literal-bound names, and +# their integer arithmetic (`x:x + 2`). Slices work on a large StackBuf (in +# place) and on a HeapBuf (bridged through the stack, one DEREF per cell), as +# inputs and as the output. Published: the two 128-bit digest cells of +# H(H(a[0:2], hb[0:2]), a[0:2]) read back from the heap. # public_input: 73254051709246423672821570119667875293, 221212579854185352854904196652205296234 from snark_lib import * def main(): - a = StackBuf(8) + a = StackBuf(4) a[0] = 5 a[1] = 7 + a[2] = 0 + a[3] = 0 hb = HeapBuf(4) - hb[1] = 11 # heap cell g^0 - hb[GEN] = 13 # heap cell g^1 + hb[1] = 11 # heap cell g^0 + hb[GEN] = 13 # heap cell g^1 x = 0 h = StackBuf(2) blake3(a[x:x + 2], hb[0:2], h) # stack slice + heap input slice diff --git a/crates/lean_compiler/tests/programs/const_params.py b/crates/lean_compiler/tests/programs/const_params.py index 24bf0105..1730f375 100644 --- a/crates/lean_compiler/tests/programs/const_params.py +++ b/crates/lean_compiler/tests/programs/const_params.py @@ -2,9 +2,9 @@ # site passes a compile-time constant and gets a monomorphized copy with `k` # substituted as the integer literal, usable in compile-time positions (the # slice bounds below). The direct call and match_range arm 0 share the k=0 -# specialization. Published: H(pair0, pair0) + H(pair1, pair1), word-wise -# (`+` is XOR) — the direct k=0 digest XORed with the arm the runtime x = GEN -# selects (k=1). +# specialization. A 256-bit BLAKE3 value occupies two canonical cells. +# Published: the two 128-bit digest cells of H(quad0, quad0) XOR H(quad1, quad1) +# — the direct k=0 digest XORed with the arm the runtime x = GEN selects (k=1). # public_input: 151852673551549100809121251071251225977, 143253370905495339312277763262351734242 from snark_lib import * diff --git a/crates/lean_compiler/tests/programs/unroll.py b/crates/lean_compiler/tests/programs/unroll.py index 933e0f50..4cf84170 100644 --- a/crates/lean_compiler/tests/programs/unroll.py +++ b/crates/lean_compiler/tests/programs/unroll.py @@ -2,7 +2,8 @@ # as the integer literal of each iteration — zero loop overhead (no call, no # frame, no counter). Bounds are compile-time integers, including Const # parameters: `chain(buf, 3)` specializes and unrolls three BLAKE3 steps over -# heap slices indexed by `i`. Published: H³(5, 7) — same chain as +# heap slices indexed by `i` (a 256-bit BLAKE3 value is two canonical cells). +# Published: the two 128-bit digest cells of H^3(5, 7) — same chain as # blake3_heap_chain.py, unrolled instead of looped. # public_input: 259899574965733219954697446670390340005, 71594800443637044304569228067009621691 from snark_lib import * diff --git a/crates/lean_compiler/tests/programs/wots_walk.py b/crates/lean_compiler/tests/programs/wots_walk.py index 69173101..d9a39577 100644 --- a/crates/lean_compiler/tests/programs/wots_walk.py +++ b/crates/lean_compiler/tests/programs/wots_walk.py @@ -1,11 +1,12 @@ # A miniature WOTS-style chain walk bundling the DSL's moving parts: a # runtime digit is range-checked (dispatch soundness), then match_range # dispatches it to a Const-specialized walker whose BLAKE3 chain is unrolled -# over heap slices; the walker also builds g^{2n} at runtime (unrolled MULs) -# to read its final pair back through g-power indexing. The recomputation at -# the end lands on an already-written StackBuf pair, so write-once turns the -# hash into a digest assertion; the dead `if` branch holds an impossible -# assert that must never execute. Published: H²(5, 7). +# over heap slices (a 256-bit BLAKE3 value occupies two canonical cells); +# the walker also builds g^{2n} at runtime (unrolled MULs) to read its final +# pair back through g-power indexing. The recomputation at the end lands on an +# already-written StackBuf pair, so write-once turns the hash into a digest +# assertion; the dead `if` branch holds an impossible assert that must never +# execute. Published: the two 128-bit digest cells of H^2(5, 7). # public_input: 101229015297003380629709256178361811305, 199495362546883507010283175921733252645 from snark_lib import * @@ -22,7 +23,7 @@ def main(): v = StackBuf(2) v[0] = t0 v[1] = t1 - blake3(buf[2:4], buf[2:4], v) # recompute H(pair1, pair1): asserts == (t0, t1) + blake3(buf[2:4], buf[2:4], v) # recompute H(value1, value1): asserts v[0:2] == (t0, t1) p = GEN ** 0 p[1] = t0 p[GEN] = t1 diff --git a/crates/lean_compiler/tests/py_source.rs b/crates/lean_compiler/tests/py_source.rs index e997e5f7..7f63ed80 100644 --- a/crates/lean_compiler/tests/py_source.rs +++ b/crates/lean_compiler/tests/py_source.rs @@ -16,26 +16,30 @@ use std::fs; use lean_compiler::{compile, parse, parse_const}; use lean_vm::cpu::{prove, verify}; -use primitives::field::F128; +use primitives::field::F192; /// The `# public_input: , ` annotation, or `[0, 0]` if absent. -fn public_input(src: &str) -> [F128; 2] { +fn public_input(src: &str) -> [F192; 2] { for line in src.lines() { if let Some(rest) = line.trim().strip_prefix("# public_input:") { let parts: Vec<&str> = rest.split(',').collect(); - assert_eq!(parts.len(), 2, "`# public_input:` needs two field elements, got `{rest}`"); + assert_eq!( + parts.len(), + 2, + "`# public_input:` needs two field elements, got `{rest}`" + ); let elt = |s: &str| parse_const(s).unwrap_or_else(|e| panic!("bad public_input: {e}")); return [elt(parts[0]), elt(parts[1])]; } } - [F128::ZERO; 2] + [F192::ZERO; 2] } /// The `# witness : , …` annotations — one line per *entry* /// (repeated lines with the same name are the stream's successive entries, /// popped by successive `hint_witness` calls). -fn witness(src: &str) -> std::collections::HashMap>> { - let mut streams: std::collections::HashMap>> = Default::default(); +fn witness(src: &str) -> std::collections::HashMap>> { + let mut streams: std::collections::HashMap>> = Default::default(); for rest in src.lines().filter_map(|l| l.trim().strip_prefix("# witness ")) { let (name, vals) = rest.split_once(':').expect("`# witness` needs `name: values`"); let entry = vals @@ -68,7 +72,7 @@ fn all_py_programs() { for (stream, entries) in witness(&src) { program.set_witness(stream, entries); } - let (proof, _) = prove(&program, want); + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); verify(&program, &want, &proof).unwrap_or_else(|e| panic!("{name}: verify: {e:?}")); println!("{name}: ok"); } diff --git a/crates/lean_compiler/tests/python_verifier.rs b/crates/lean_compiler/tests/python_verifier.rs index 0697d6f6..7ee82543 100644 --- a/crates/lean_compiler/tests/python_verifier.rs +++ b/crates/lean_compiler/tests/python_verifier.rs @@ -1,6 +1,6 @@ use lean_compiler::{compile, parse_with_replacements}; -use lean_vm::cpu::{prove, DerefMode, Op, Program}; -use primitives::field::{g_pow, F128}; +use lean_vm::cpu::{DerefMode, Op, Program, prove, verify}; +use primitives::field::{F64, F192, g_pow}; use std::collections::BTreeMap; use std::path::Path; use std::process::Command; @@ -35,29 +35,34 @@ def main(): const LOOP_STEPS: usize = 16_384; -fn public_input() -> [F128; 2] { - use lean_vm::blake3_flock::{compression, digest, metadata, FLAGS, IV}; +fn public_input() -> [F192; 2] { + use lean_vm::blake3_flock::{FLAGS, IV, compression, digest, metadata}; - let seed = [F128::new(5, 0), F128::new(7, 0)]; + let seed = [F64(5), F64::ZERO, F64(7), F64::ZERO]; let metadata = metadata(0, 64, FLAGS); let digest = digest(&compression(seed, seed, IV, metadata)); + let digest = [ + F192::new(digest[0].0, digest[1].0, 0), + F192::new(digest[2].0, digest[3].0, 0), + ]; let mut value = digest[0]; - let mut index = F128::ONE; + let mut index = F192::ONE; + let generator = F192::from(g_pow(1)); for _ in 0..LOOP_STEPS { let candidate = value + index; - let product = candidate * g_pow(1); - value = (if product == F128::ZERO { + let product = candidate * generator; + value = (if product == F192::ZERO { candidate } else { product + candidate }) + index; - index *= g_pow(1); + index *= generator; } - [value, digest[1] * g_pow(1) + digest[1]] + [value, digest[1] * generator + digest[1]] } -fn field_json(value: F128) -> String { - format!("[{}, {}]", value.lo, value.hi) +fn field_json(value: F192) -> String { + format!("[{}, {}, {}]", value.c0, value.c1, value.c2) } fn operation_json(operation: Op) -> String { @@ -81,6 +86,9 @@ fn operation_json(operation: Op) -> String { Op::Jump { oc, od, of } => { format!(r#" {{"op":"jump","oc":{oc},"od":{od},"of":{of}}}"#) } + Op::Pack64x2 { a, b, c } => { + format!(r#" {{"op":"pack64x2","a":{a},"b":{b},"c":{c}}}"#) + } Op::Blake3 { ins, cv, out, metadata } => format!( r#" {{"op":"blake3","ins":[{},{},{},{}],"cv":{cv},"out":{out},"metadata":{}}}"#, ins[0], @@ -92,7 +100,7 @@ fn operation_json(operation: Op) -> String { } } -fn statement_json(program: &Program, public_input: [F128; 2]) -> String { +fn statement_json(program: &Program, public_input: [F192; 2]) -> String { let operations = program .prog .iter() @@ -114,7 +122,7 @@ fn test_python_verifier() { let ast = parse_with_replacements(SOURCE, &replacements).expect("parse zkDSL program"); let program = compile(&ast); let public_input = public_input(); - let (proof, stats) = prove(&program, public_input); + let (proof, stats) = prove(&program, public_input, 1); let directory = std::env::temp_dir().join(format!("leanvm-python-verifier-test-{}", std::process::id())); std::fs::create_dir_all(&directory).expect("create test directory"); @@ -127,9 +135,9 @@ fn test_python_verifier() { let verifier = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../python-verifier/verifier.py"); let verification_started = Instant::now(); let output = Command::new("python3") - .arg(verifier) - .arg(statement_path) - .arg(proof_path) + .arg(&verifier) + .arg(&statement_path) + .arg(&proof_path) .output() .expect("run native Python verifier"); let verification_time = verification_started.elapsed(); @@ -140,6 +148,42 @@ fn test_python_verifier() { ); assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "verification succeeded",); + let mut malformed_announcement = proof.clone(); + malformed_announcement.stream[0].c1 = 1; + assert!(verify(&program, &public_input, &malformed_announcement).is_err()); + std::fs::write( + &proof_path, + bincode::serialize(&malformed_announcement).expect("serialize malformed announcement"), + ) + .expect("write malformed announcement"); + let output = Command::new("python3") + .arg(&verifier) + .arg(&statement_path) + .arg(&proof_path) + .output() + .expect("run Python verifier on malformed announcement"); + assert!(!output.status.success(), "Python accepted a noncanonical announcement"); + + let mut malformed_root = proof.clone(); + let root_offset = lean_vm::tables::N_TABLES + 2; + malformed_root.stream[root_offset].c2 = 1; + assert!(verify(&program, &public_input, &malformed_root).is_err()); + std::fs::write( + &proof_path, + bincode::serialize(&malformed_root).expect("serialize malformed root"), + ) + .expect("write malformed root"); + let output = Command::new("python3") + .arg(verifier) + .arg(statement_path) + .arg(proof_path) + .output() + .expect("run Python verifier on malformed root"); + assert!( + !output.status.success(), + "Python accepted a noncanonical commitment root" + ); + println!( "zkDSL compiled to {} instructions; proved {} cycles in {} bytes; Python verified in {:.2?}", program.prog.len(), diff --git a/crates/lean_compiler/tests/range_check.rs b/crates/lean_compiler/tests/range_check.rs index 8b6a7988..e9a85d74 100644 --- a/crates/lean_compiler/tests/range_check.rs +++ b/crates/lean_compiler/tests/range_check.rs @@ -8,7 +8,7 @@ use lean_compiler::{compile, parse}; use lean_vm::cpu::{prove, verify}; -use primitives::field::{F128, g_pow}; +use primitives::field::{F64, F192, g_pow}; /// Both bound forms (`log GEN ** k` and a plain integer exponent) with the /// boundary elements (`g^{k-1}`, `1 = g^0`), end-to-end: prove + verify, and a @@ -31,14 +31,17 @@ def main(): return "; let program = compile(&parse(src).expect("parse")); - let want = [g_pow(12), g_pow(5)]; - let (proof, stats) = prove(&program, want); + let want = [F192::from(g_pow(12)), F192::from(g_pow(5))]; + let (proof, stats) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); // 2 DEREFs per range check (4 checks) + 2 publishing stores. assert_eq!(stats.counts[3], 10, "DEREF count"); verify(&program, &want, &proof).expect("range-checked program verifies"); - let bad = [g_pow(12), g_pow(6)]; - assert!(verify(&program, &bad, &proof).is_err(), "wrong public input must be rejected"); + let bad = [F192::from(g_pow(12)), F192::from(g_pow(6))]; + assert!( + verify(&program, &bad, &proof).is_err(), + "wrong public input must be rejected" + ); } /// A check whose two touched cells (`m[300]` and the complement's `m[99]`) are @@ -56,8 +59,8 @@ def main(): return "; let program = compile(&parse(src).expect("parse")); - let want = [g_pow(300), g_pow(300)]; - let (proof, _) = prove(&program, want); + let want = [F192::from(g_pow(300)), F192::from(g_pow(300))]; + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); verify(&program, &want, &proof).expect("deferred-fill program verifies"); } @@ -77,8 +80,8 @@ def main(): return "; let program = compile(&parse(src).expect("parse")); - let want = [F128::new(5, 0), F128::new(7, 0)]; - let (proof, stats) = prove(&program, want); + let want = [F192::from(F64(5)), F192::from(F64(7))]; + let (proof, stats) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); // 6 iterations × 2 range-check DEREFs, plus call/publish plumbing. assert!(stats.counts[3] >= 12, "at least the 12 range-check DEREFs"); verify(&program, &want, &proof).expect("loop range checks verify"); @@ -92,7 +95,7 @@ def main(): fn range_check_at_bound_rejected() { let src = "def main():\n x = GEN ** 8\n assert log x < 8\n return\n"; let program = compile(&parse(src).expect("parse")); - program.execute([F128::ZERO, F128::ZERO]); + program.execute([F192::ZERO, F192::ZERO]); } /// A value that is no small g-power at all (5 = x^2 + 1) fails at the first @@ -102,7 +105,7 @@ fn range_check_at_bound_rejected() { fn range_check_non_g_power_rejected() { let src = "def main():\n x = 5\n assert log x < 8\n return\n"; let program = compile(&parse(src).expect("parse")); - program.execute([F128::ZERO, F128::ZERO]); + program.execute([F192::ZERO, F192::ZERO]); } /// Bound 0 names the empty set — rejected at compile time. diff --git a/crates/lean_compiler/tests/stack_buf.rs b/crates/lean_compiler/tests/stack_buf.rs index eb2ec81c..40b9166b 100644 --- a/crates/lean_compiler/tests/stack_buf.rs +++ b/crates/lean_compiler/tests/stack_buf.rs @@ -1,38 +1,42 @@ //! `StackBuf` — a run of consecutive frame (stack) cells in the zkDSL. Indexed //! reads/writes go straight to `base+k` (no heap deref), and a size-2 `StackBuf` -//! is a `blake3` operand: its two cells hold the 256-bit value's two words, so +//! is a `blake3` operand: its two canonical 128-bit cells hold the 256-bit value, so //! `blake3(a, b, out)` reads them in place with no copies (a self-hash //! `blake3(h, h, out)` aliases one pair into both input operands) and writes //! the digest into the pre-allocated pair `out`. +//! +//! Since these DSL scalars are K-embedded F192 cells, a `StackBuf(2)` written +//! cell-by-cell holds the flock words `[v0, 0, v1, 0]` +//! — the reference `compress` below is fed that lane layout. -use lean_vm::blake3_flock::{compression, digest, metadata, warm_setup}; use lean_compiler::{compile, parse}; +use lean_vm::blake3_flock::{compression, digest, metadata, warm_setup}; use lean_vm::cpu::{prove, verify}; -use primitives::field::F128; +use primitives::field::{F64, F192}; -/// `BLAKE3(a, b)` reference (matches `cpu::blake3_compress`): the four words laid -/// little-endian into 64 bytes, hashed, digest split into two `F128` words. -fn compress(a: [F128; 2], b: [F128; 2]) -> [F128; 2] { +/// `BLAKE3(a, b)` reference (matches `cpu::blake3_compress`): the eight words +/// laid little-endian into 64 bytes, hashed, digest split into four `F64` words. +fn compress(a: [F64; 4], b: [F64; 4]) -> [F64; 4] { let mut input = [0u8; 64]; - for (slot, w) in input.chunks_exact_mut(16).zip([a[0], a[1], b[0], b[1]]) { - slot[..8].copy_from_slice(&w.lo.to_le_bytes()); - slot[8..].copy_from_slice(&w.hi.to_le_bytes()); + for (slot, w) in input.chunks_exact_mut(8).zip(a.into_iter().chain(b)) { + slot.copy_from_slice(&w.0.to_le_bytes()); } let d = blake3::hash(&input); let d = d.as_bytes(); - let word = |b: &[u8]| { - F128::new( - u64::from_le_bytes(b[..8].try_into().unwrap()), - u64::from_le_bytes(b[8..16].try_into().unwrap()), - ) - }; - [word(&d[..16]), word(&d[16..])] + std::array::from_fn(|k| F64(u64::from_le_bytes(d[8 * k..8 * k + 8].try_into().unwrap()))) +} + +/// The two 128-bit digest cells of `compress(a, b)` as `F192`s (lo = word 0/2, +/// hi = word 1/3) — what a `blake3(...)` output `StackBuf(2)` holds cell-by-cell. +fn digest_cells(a: [F64; 4], b: [F64; 4]) -> [F192; 2] { + let d = compress(a, b); + [F192::new(d[0].0, d[1].0, 0), F192::new(d[2].0, d[3].0, 0)] } /// A size-2 `StackBuf` fed to `blake3` as a self-hash `blake3(h, h)`, then the -/// digest published to `m[0], m[1]`. Proves and verifies, and a wrong published -/// digest is rejected — so the whole path (StackBuf load → aliased blake3 → -/// stack read → publish) is exercised end-to-end. +/// digest's two 128-bit cells published to `m[0], m[1]`. Proves and verifies, and +/// a wrong published digest is rejected — so the whole path (StackBuf load → +/// aliased blake3 → stack read → publish) is exercised end-to-end. #[test] fn stack_buf_blake3_self_hash() { let src = "\ @@ -50,16 +54,16 @@ def main(): let program = compile(&parse(src).expect("parse")); warm_setup(1); - let five = F128::new(5, 0); - let seven = F128::new(7, 0); - let want = compress([five, seven], [five, seven]); + // Each cell holds one scalar in its low lane, so the hashed words are [5,0,7,0]. + let h = [F64(5), F64(0), F64(7), F64(0)]; + let want = digest_cells(h, h); - let (proof, stats) = prove(&program, want); + let (proof, stats) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); assert_eq!(stats.counts[5], 1, "one BLAKE3 instruction"); verify(&program, &want, &proof).expect("StackBuf self-hash verifies"); let mut bad = want; - bad[0] += F128::ONE; + bad[0] += F192::ONE; assert!(verify(&program, &bad, &proof).is_err(), "wrong digest must be rejected"); } @@ -84,14 +88,13 @@ def main(): warm_setup(2); let mut input = Vec::new(); for value in 1u64..=5 { - input.extend_from_slice(&F128::new(value, 0).to_le_bytes()); + input.extend_from_slice(&value.to_le_bytes()); + input.extend_from_slice(&0u64.to_le_bytes()); } - let digest = blake3::hash(&input); - let want = [ - F128::from_le_bytes(digest.as_bytes()[..16].try_into().unwrap()), - F128::from_le_bytes(digest.as_bytes()[16..].try_into().unwrap()), - ]; - let (proof, stats) = prove(&program, want); + let d = blake3::hash(&input); + let word = |o: usize| u64::from_le_bytes(d.as_bytes()[o..o + 8].try_into().unwrap()); + let want = [F192::new(word(0), word(8), 0), F192::new(word(16), word(24), 0)]; + let (proof, stats) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); assert_eq!(stats.counts[5], 2); verify(&program, &want, &proof).expect("standard two-block BLAKE3 verifies"); } @@ -116,15 +119,12 @@ def main(): p[GEN] = out[1] return "; - let want = compress( - [F128::new(1, 0), F128::new(2, 0)], - [F128::new(3, 0), F128::new(4, 0)], - ); + let want = digest_cells([F64(1), F64(0), F64(2), F64(0)], [F64(3), F64(0), F64(4), F64(0)]); warm_setup(2); for flag in [0, 1] { let mut program = compile(&parse(src).expect("parse")); - program.set_witness("flag", vec![vec![F128::new(flag, 0)]]); - let (proof, _) = prove(&program, want); + program.set_witness("flag", vec![vec![F192::new(flag, 0, 0)]]); + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); verify(&program, &want, &proof).expect("post-join default IV is initialized on both paths"); } } @@ -148,15 +148,12 @@ def main(): p[GEN] = out[1] return "; - let want = compress( - [F128::new(1, 0), F128::new(2, 0)], - [F128::new(3, 0), F128::new(4, 0)], - ); + let want = digest_cells([F64(1), F64(0), F64(2), F64(0)], [F64(3), F64(0), F64(4), F64(0)]); warm_setup(1); for flag in [0, 1] { let mut program = compile(&parse(src).expect("parse")); - program.set_witness("flag", vec![vec![F128::new(flag, 0)]]); - let (proof, _) = prove(&program, want); + program.set_witness("flag", vec![vec![F192::new(flag, 0, 0)]]); + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); verify(&program, &want, &proof).expect("each branch initializes its default IV"); } } @@ -180,14 +177,15 @@ def main(): "; let program = compile(&parse(src).expect("parse")); let block = compression( - [F128::new(1, 0), F128::new(2, 0)], - [F128::new(3, 0), F128::new(4, 0)], - [F128::new(5, 0), F128::new(6, 0)], + [F64(1), F64(0), F64(2), F64(0)], + [F64(3), F64(0), F64(4), F64(0)], + [F64(5), F64(0), F64(6), F64(0)], metadata(0, 64, 10), ); - let want = digest(&block); + let d = digest(&block); + let want = [F192::new(d[0].0, d[1].0, 0), F192::new(d[2].0, d[3].0, 0)]; warm_setup(1); - let (proof, _) = prove(&program, want); + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); verify(&program, &want, &proof).expect("materialized custom CV verifies"); } @@ -225,12 +223,84 @@ def main(): "; let program = compile(&parse(src).expect("parse")); // `+` is XOR: 3 ^ 4 = 7. Published: (sa[2], sa[1]) = (7, 4). - let want = [F128::new(7, 0), F128::new(4, 0)]; - let (proof, stats) = prove(&program, want); + let want = [F192::from(F64(7)), F192::from(F64(4))]; + let (proof, stats) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); assert_eq!(stats.counts[5], 0, "no BLAKE3 here"); verify(&program, &want, &proof).expect("StackBuf indexing verifies"); } +/// A normal (non-`@inline`) function may return a StackBuf. Its cells cross the +/// call boundary through consecutive return slots and bind as a StackBuf in the +/// caller, including through another normal wrapper function. +#[test] +fn normal_function_returns_stackbuf() { + let src = "\ +def main(): + out = forward(5) + p = 1 + p[1] = out[0] + out[1] + p[GEN] = out[2] + return + +def forward(v): + out = make(v) + return out + +def make(v): + out = StackBuf(3) + out[0] = v + out[1] = v + 3 + out[2] = 11 + return out +"; + let program = compile(&parse(src).expect("parse")); + // Field addition is XOR: 5 ^ (5 ^ 3) == 3. + program.execute([F192::from(F64(3)), F192::from(F64(11))]); +} + +/// Tuple returns retain their source-level arity even though a StackBuf member +/// occupies several physical return cells. +#[test] +fn normal_function_returns_stackbuf_and_scalar() { + let src = "\ +def main(): + out, x = make(9) + p = 1 + p[1] = out[0] + out[1] + p[GEN] = x + return + +def make(v): + out = [v, 6] + return out, v + 1 +"; + let program = compile(&parse(src).expect("parse")); + program.execute([F192::from(F64(15)), F192::from(F64(8))]); +} + +/// HeapBuf already crosses a normal call as its one-cell pointer. Allocation +/// happened in the callee, so the caller needs no size metadata to dereference +/// and use the returned buffer. +#[test] +fn normal_function_returns_heapbuf_pointer() { + let src = "\ +def main(): + out = make() + p = 1 + p[1] = out[1] + p[GEN] = out[GEN] + return + +def make(): + out = HeapBuf(2) + out[1] = 17 + out[GEN] = 23 + return out +"; + let program = compile(&parse(src).expect("parse")); + program.execute([F192::from(F64(17)), F192::from(F64(23))]); +} + /// A StackBuf index literal that does not fit `u32` is rejected at compile time, /// not silently truncated modulo 2^32 (which would resolve `sa[2^32]` to `sa[0]`). #[test] @@ -247,8 +317,8 @@ fn stack_buf_index_overflow_rejected() { fn stack_buf_rebind_to_scalar() { let src = "def main():\n x = StackBuf(2)\n x = 5\n p = 1\n p[1] = x\n p[GEN] = x\n return\n"; let program = compile(&parse(src).expect("parse")); - let want = [F128::new(5, 0), F128::new(5, 0)]; - let (proof, _) = prove(&program, want); + let want = [F192::from(F64(5)), F192::from(F64(5))]; + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); verify(&program, &want, &proof).expect("rebound-scalar program verifies"); } @@ -293,18 +363,23 @@ def step(state, v): "; let program = compile(&parse(src).expect("parse")); - let tag = [F128::new(9, 0), F128::new(3, 0)]; // x == v == 9 (the scalar return), tag word 3 - let s1 = compress([F128::new(5, 0), F128::new(7, 0)], tag); - let s2 = compress(s1, tag); // the returned StackBuf fed back in - let want = [s2[0], s2[1]]; + // Each cell = one scalar in its low lane, so a StackBuf(2) hashes words + // [c0, 0, c1, 0]. x == v == 9 (the scalar return), so both steps use tag 9. + let tag = [F64(9), F64(0), F64(3), F64(0)]; + let s1 = compress([F64(5), F64(0), F64(7), F64(0)], tag); + let s2 = compress(s1, tag); // the returned StackBuf (holding s1's words) fed back in + let want = [F192::new(s2[0].0, s2[1].0, 0), F192::new(s2[2].0, s2[3].0, 0)]; - let (proof, stats) = prove(&program, want); + let (proof, stats) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); assert_eq!(stats.counts[5], 2, "two BLAKE3 instructions (one per inlined step)"); verify(&program, &want, &proof).expect("inline StackBuf+scalar tuple return verifies"); let mut bad = want; - bad[1] += F128::ONE; - assert!(verify(&program, &bad, &proof).is_err(), "wrong published state must be rejected"); + bad[1] += F192::ONE; + assert!( + verify(&program, &bad, &proof).is_err(), + "wrong published state must be rejected" + ); } /// Deferred stores made by a runtime branch must initialize buffers allocated @@ -343,7 +418,7 @@ def select_pair(flag, a, b): return first, second "; let program = compile(&parse(src).expect("parse")); - program.execute([F128::ONE, F128::ZERO]); + program.execute([F192::ONE, F192::ZERO]); } /// An `@inline` may also alias-return a folded **g-address** among its values: @@ -385,16 +460,16 @@ def step(state, cursor): let program = compile(&parse(src).expect("parse")); // a = hb[0] = 10, b = hb[1] = 20, v = hb[2] = 30 read through the cursor // returned twice-advanced. a + b is XOR: 10 ^ 20 = 30. - let want = [F128::new(30, 0), F128::new(30, 0)]; - let (proof, _) = prove(&program, want); + let want = [F192::from(F64(30)), F192::from(F64(30))]; + let (proof, _) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); verify(&program, &want, &proof).expect("inline advanced-cursor return verifies"); } -/// `x = [a, b]` — the list-literal StackBuf initializer: allocates the run and -/// writes the elements in place, sugar for alloc-then-store. The test mixes a -/// runtime value, a constant, and an expression; feeds the result to blake3; -/// and swaps a buffer through itself (`s = [s[1], s[0]]` reads the OLD -/// binding, per the let-rebind rule). +/// `x = [a, b, c, d]` — the list-literal StackBuf initializer: allocates the run +/// and writes the elements in place, sugar for alloc-then-store. The test mixes a +/// runtime value, a constant, and an expression; feeds the result to blake3; and +/// swaps a buffer through itself (`s = [s[1], s[0], …]` reads the OLD binding, +/// per the let-rebind rule). #[test] fn stack_buf_list_literal() { warm_setup(1); @@ -411,9 +486,9 @@ def main(): return "; let program = compile(&parse(src).expect("parse")); - // s = [7, 5] after the swap; t = [7 ^ 5, 3] = [2, 3]. - let want = compress([F128::new(7, 0), F128::new(5, 0)], [F128::new(2, 0), F128::new(3, 0)]); - let (proof, stats) = prove(&program, want); + // s = [7, 5] after the swap → words [7,0,5,0]; t = [7 ^ 5, 3] = [2, 3] → [2,0,3,0]. + let want = digest_cells([F64(7), F64(0), F64(5), F64(0)], [F64(2), F64(0), F64(3), F64(0)]); + let (proof, stats) = prove(&program, want, lean_vm::pcs::LOG_INV_RATE); assert_eq!(stats.counts[5], 1, "one BLAKE3 instruction"); verify(&program, &want, &proof).expect("list-literal StackBuf verifies"); } @@ -453,8 +528,9 @@ fn heap_hint_slice_oob_rejected() { let _ = compile(&parse(src).expect("parse")); } -/// A blake3 heap slice straddling the buffer end is rejected (the span is 2 -/// cells, so the last valid start is size − 2). +/// A blake3 heap slice straddling the buffer end is rejected. The 256-bit +/// operand `hb[7:9]` is two 128-bit cells, so the bound check trips at +/// `7 + 2 = 9 > 8`. #[test] #[should_panic(expected = "heap slice 7:9 out of bounds for `hb` (HeapBuf size 8)")] fn heap_blake3_slice_oob_rejected() { @@ -462,13 +538,13 @@ fn heap_blake3_slice_oob_rejected() { let _ = compile(&parse(src).expect("parse")); } -/// The last in-bounds index and a full-size slice still compile and run. +/// The last in-bounds index still compiles and runs. #[test] fn heap_index_boundary_ok() { warm_setup(1); let src = "def main():\n hb = HeapBuf(8)\n hb[GEN ** 7] = 5\n row = hb * GEN ** 4\n y = row[GEN ** 3]\n assert y == 5\n return\n"; let program = compile(&parse(src).expect("parse")); - let pi = [F128::new(3, 0), F128::new(4, 0)]; - let (proof, _) = prove(&program, pi); + let pi = [F192::from(F64(3)), F192::from(F64(4))]; + let (proof, _) = prove(&program, pi, lean_vm::pcs::LOG_INV_RATE); verify(&program, &pi, &proof).expect("boundary access verifies"); } diff --git a/crates/lean_compiler/tests/transcript_helpers.rs b/crates/lean_compiler/tests/transcript_helpers.rs new file mode 100644 index 00000000..2fa6ee76 --- /dev/null +++ b/crates/lean_compiler/tests/transcript_helpers.rs @@ -0,0 +1,49 @@ +use lean_compiler::{compile, parse}; +use primitives::field::F192; + +#[test] +fn transcript_helpers_are_ordinary_nested_inline_zkdsl() { + let src = r#" +from snark_lib import * + +Y = f192(0, 1, 0) + +@inline +def challenge_from_state(state): + lo = StackBuf(2) + hi = StackBuf(2) + hint_f192_limbs(lo, state[0]) + hint_f192_limbs(hi, state[1]) + pack64x2_into(lo[0], lo[1], state[0]) + pack64x2_into(hi[0], hi[1], state[1]) + return lo[0] + Y * (lo[1] + Y * hi[0]) + +@inline +def sponge_compress(state, scalar, tail, out): + limbs = StackBuf(3) + hint_f192_limbs(limbs, scalar) + block = StackBuf(2) + pack64x2_into(limbs[0], limbs[1], block[0]) + pack64x2_into(limbs[2], tail, block[1]) + assert scalar == limbs[0] + Y * (limbs[1] + Y * limbs[2]) + blake3(state, block, out) + return + +@inline +def observe(state, scalar): + out = StackBuf(2) + sponge_compress(state, scalar, 13, out) + return out + +def main(): + state = StackBuf(2) + state[0] = f192(1, 2, 0) + state[1] = f192(3, 4, 0) + out = observe(state, f192(5, 6, 7)) + challenge = challenge_from_state(out) + assert challenge == challenge + return +"#; + let program = compile(&parse(src).expect("parse transcript helpers")); + program.execute([F192::ZERO; 2]); +} diff --git a/crates/lean_compiler/zkDSL.md b/crates/lean_compiler/zkDSL.md index 91d386cc..e6449a7a 100644 --- a/crates/lean_compiler/zkDSL.md +++ b/crates/lean_compiler/zkDSL.md @@ -1,14 +1,14 @@ # zkDSL Language Reference (leanVM-b) -The zkDSL is a Python-syntax language that compiles to the leanVM-b ISA — six -instructions (`XOR`, `MUL`, `SET`, `DEREF`, `JUMP`, `BLAKE3`) over the binary -field GF(2^128), with write-once memory and all indices carried "in the +The zkDSL is a Python-syntax language that compiles to the leanVM-b ISA — seven +instructions (`XOR`, `MUL`, `SET`, `DEREF`, `JUMP`, `BLAKE3`, `PACK64X2`) over the binary +field GF(2^192), with write-once memory and all indices carried "in the exponent" as powers of a fixed generator. For the underlying VM and proving -system, see [`misc/doc.tex`](../../misc/doc.tex) (released as `doc.pdf`). +system, see [`misc/doc.tex`](../../misc/doc.tex). Source files use the `.py` extension and are **valid Python**: they import the [`snark_lib`](../../snark_lib.py) stub, which defines `GEN`, `log`, `mul_range`, -`HeapBuf`, `StackBuf`, and `blake3` so that editors, linters, and even +`HeapBuf`, `StackBuf`, `pack64x2`, and `blake3` so that editors, linters, and even `python3` itself accept the file. The compiler skips the import. Entry points: `lean_compiler::parse` / `parse_file_with_replacements` → @@ -24,15 +24,34 @@ no-ops, so this only checks that the file is well-formed. ## The field — and indices in the exponent -Every runtime value is one element of GF(2^128) in GHASH form -(`F_2[x]/(x^128 + x^7 + x^2 + x + 1)`). There are no runtime integers. - -- `+` is field addition = bitwise **XOR** (so `x + x == 0`), -- `*` is the field (GHASH) product, -- an integer literal `n` denotes the field element with bit pattern `n` - (bit `k` is the coefficient of `x^k`) — `5` is `1 + x^2`, not "five", -- `GEN` is the fixed generator `g = x` (multiplicative order `2^128 − 1`), -- `GEN ** e` is the compile-time constant `g^e` (`**` takes base `GEN` and a +The fields are + +`K = GF(2)[x]/(x^64 + x^4 + x^3 + x + 1)` and +`E = K[y]/(y^3 + y + 1) = GF(2^192)`. + +Machine **words** — the contents of a memory cell, an immediate, a hashed +value, the `JUMP` condition — are elements of `E`. **Addresses**, +the program counter, the frame pointer, read counters, operands, opcodes, and +domain separators live in the 64-bit subfield `K = GF(2^64)`. There are no +runtime integers. + +- `+` is field addition = bitwise **XOR** (192-bit on words, so `x + x == 0`), +- `*` is multiplication in `E`; + for g-powers and + addresses it stays within `K`, +- `/` is runtime field division, `a / b = a · b⁻¹`. It costs one `MUL`: the + compiler leaves the quotient cell unset and emits the checked relation + `quotient · b == a`, which witness generation back-solves. Division by zero is + undefined. This is distinct from `//`, compile-time integer floor division in + sizes and indices, +- an integer literal `n` supplies up to 128 raw bits and is embedded as + `F192(c0, c1, 0)`. This is a source-syntax limit, not the machine-word + width: words have three 64-bit limbs. Thus `5` is `1 + x^2`, not the integer five, and + `2 ** 64` is the tower element `y`. Full-width constants use + `f192(c0, c1, c2)`, with each limb an unsigned 64-bit compile-time integer, +- `GEN` is the fixed generator `g = x` of the 64-bit subfield `K^×` + (multiplicative order `2^64 − 1`), +- `GEN ** e` is the compile-time constant `g^e ∈ K` (`**` takes base `GEN` and a compile-time integer exponent — a literal, a constant, an `unroll` variable, `len(...)`, or index arithmetic of those). So `buf[GEN ** i]` names heap cell `i` directly inside an `unroll` loop, with no running-pointer cursor. @@ -41,10 +60,10 @@ Every runtime value is one element of GF(2^128) in GHASH form or field arithmetic in a value position (`x ** k`, e.g. a loop counter `g^i` raised to a stride to reach cell `i·stride`). The base may be runtime. -A logical **index** `i` is carried as `g^i`: incrementing is one -multiplication by `GEN`, and memory/bytecode addresses are g-powers. This is -the design idiom of the whole VM — loops, heap addressing, and range checks -below all live in the exponent. +A logical **index** `i` is carried as `g^i` in the 64-bit subfield (order +`2^64 − 1`): incrementing is one multiplication by `GEN`, and memory/bytecode +addresses are g-powers. This is the design idiom of the whole VM — loops, heap +addressing, and range checks below all live in the exponent, in `K`. ## Program shape @@ -67,9 +86,17 @@ def helper(a, b): # other functions anything else is a compile error (no multi-file programs yet). Comments (`#`) and blank lines are free. Indentation is block structure, as in Python. +Ordinary functions may return scalars, `HeapBuf` pointers, and `StackBuf` +values, including mixtures in a tuple return. A returned `StackBuf(n)` has a +compile-time-known size: its `n` cells are copied through `n` consecutive return +slots and the caller binds the result as a new `StackBuf(n)`. A `HeapBuf` return +is just its one-cell pointer; the allocation hint already ran where the buffer +was created, so no size metadata needs to cross the call. + ## Public input -Memory cells `m[0]` and `m[1]` hold the two public-input field elements. A +Memory cells `m[0]` and `m[1]` hold the two public-input words, each an F192 +machine word. A program *publishes* results by asserting them against those cells through the write-once heap store (the pointer `g^0` addresses absolute memory): @@ -217,11 +244,13 @@ def combine(a, b, k: Const): An `@inline` function is **expanded at each call site** instead of emitting a real call — no frame, no argument/return `DEREF`s, no call/return `JUMP`s. The -body must be a single **tail** `return`; it may contain `blake3`, `if`, and -`unroll`, but not a call to another (user) function, a `for`/`match`, or any -nested/early `return`. It is never lowered standalone; a call to a -non-`@inline` function is unchanged. (Distinct from `unroll(a, b)`, which -replicates a loop body: that one really does unroll.) +body must be a single **tail** `return`; it may contain builtins, calls to other +`@inline` functions, `if`, and `unroll`, but not a call to a non-inline user +function, a `for`/`match`, or any nested/early `return`. Nested inline calls +expand recursively; direct or indirect recursive inline calls are rejected. +An inline function is never lowered standalone; a call to a non-`@inline` +function is unchanged. (Distinct from `unroll(a, b)`, which replicates a loop +body: that one really does unroll.) An `@inline` function may also **return a `StackBuf`**: the caller's binding aliases the returned cell run (zero copies), and `StackBuf` arguments alias @@ -284,7 +313,7 @@ instruction until used as a value: defaults to the argument's source text; output goes to stderr as `[print] label = ...`, showing the decimal reading for small integers, `g^k` when the value is a small g-power (both when they overlap: `8 (g^3)`), or -`hi:lo` hex otherwise. Each print costs one anchor instruction, so the +`c2:c1:c0` hex otherwise, from the most significant limb to the least significant. Each print costs one anchor instruction, so the witness differs from a print-free build — strip prints before benchmarking. ## Memory @@ -352,17 +381,15 @@ instead). ### Slices — `buf[lo:hi]` -`buf[lo:hi]` names a run of cells (`hi` exclusive). Slices exist only as -`blake3` operands and must span exactly 2 cells (one 256-bit value). Two -forms: +`buf[lo:hi]` names a run of cells (`hi` exclusive). BLAKE3 operands must span exactly two cells; `hint_witness` accepts any supported literal length. Two forms: - **compile-time bounds** (integers, as for stack indexes): frame cells `base+lo .. base+hi` of a `StackBuf`, or heap cells `ptr·g^lo .. ptr·g^hi` of a `HeapBuf` — `hb[2:4]` is the pair `g^2, g^3`; -- **runtime start, heap only**: `buf[i:i + 2]` with a runtime g-power index - `i` (e.g. a loop counter) names the cells `buf·i`, `buf·i·g` — one `MUL` +- **runtime start, heap only**: `buf[i:i + k]` with a runtime g-power index + `i` (e.g. a loop counter) and literal length `k` names the cells `buf·i`, `buf·i·g`, and so on — one `MUL` folds `i` into the pointer. The `hi` bound cannot be evaluated, only - shape-checked: it must be syntactically `lo + 2` + shape-checked: it must be syntactically `lo + k` (`buf[b * GEN ** 2 : b * GEN ** 2 + 2]` is fine). A `StackBuf` slice cannot have a runtime start — frame offsets are baked into the bytecode operands. @@ -528,6 +555,17 @@ Statements without effect are rejected. A proof-enforced equality: 2 cycles (`XOR` into a fresh cell + `SET` it to zero, using write-once double-write as the assert). +### `assert a != b` + +A proof-enforced inequality. The compiler computes `a + b` with one `XOR` and +conditionally jumps over a poison path when it is nonzero. If the values are +equal, execution jumps to `GEN ** -1` conceptually—the field element `g⁻¹`, +outside the committed bytecode cube—so the bytecode bus cannot balance a +continuing trace. The honest path is 3 executed instructions (`XOR`, target +`SET`, `JUMP`), plus the same amortized self-frame/constant setup used by other +branches; no inverse hint is needed. A compile-time assertion such as +`assert 5 != 5` is rejected while compiling. + ### Range checks: `assert log x < log Y` and `assert log x < k` The *range check in the exponent*: proves `x ∈ {g^0, g^1, …, g^{k-1}}`, i.e. @@ -552,12 +590,68 @@ one amortized `SET` per distinct bound per frame: back-solves the complement `y = g^{k-1-e}` (the one unknown operand of a known product), and the double-write asserts `x·y = g^{k-1}`; 3. `DEREF` through `y` — bounds the complement; a "negative" `k-1-e` would - wrap to `≈ 2^128`, far beyond any memory size, so together `e ≤ k-1`. + wrap to `≈ 2^64`, far beyond any memory size, so together `e ≤ k-1`. The two `DEREF` target cells are unconstrained touches, back-filled at the end of execution. A failing check surfaces at witness generation as the complement's `DEREF` panic ("not a small g-power … a failed range check"). +## Packing two 64-bit cells — `pack64x2` + +```python +packed = pack64x2(lo, hi) +``` + +`pack64x2(a, b)` is an expression that takes one VM cycle. It proves that both +source memory words are in the base field GF(2^64), then returns their canonical +128-bit packing `(a.c0, b.c0, 0)` as one GF(2^192) word. The proof comes from +the memory bus itself: the two source accesses use literal-zero upper limbs, +and the destination access uses the tuple `(a.c0, b.c0, 0)`. Consequently a +source with either upper limb nonzero cannot satisfy the memory permutation. + +This is useful before treating values supplied as GF(2^192) hints as serialized +64-bit limbs. The returned packed word may be ignored when only the range +assertion is needed. + +The recursion transcript uses `challenge_from_state(state)` to reinterpret the +first three 64-bit lanes of a canonical two-cell BLAKE3 digest as one extension +field challenge. For `state = [s0, s1]`, it lowers exactly as follows (the +limb hints cost no cycles, but are not trusted): + +```python +lo = StackBuf(2) +hi = StackBuf(2) +hint_f192_limbs(lo, s0) # advice: [d0, d1] +hint_f192_limbs(hi, s1) # advice: [d2, d3] +pack64x2_into(lo[0], lo[1], s0) +pack64x2_into(hi[0], hi[1], s1) +challenge = lo[0] + lo[1] * f192(0, 1, 0) + hi[0] * f192(0, 0, 1) +``` + +The two in-place `PACK64X2` instructions prove through write-once memory that +`s0 = (d0,d1,0)` and `s1 = (d2,d3,0)`. Consequently all four digest lanes are +really in GF(2^64); the challenge is `d0 + d1·Y + d2·Y²`, while `d3` is checked +but deliberately discarded. `challenge_from_state` is not a compiler +intrinsic: this is the complete `@inline` helper used by the recursion guest. + +Likewise, the recursion guest's `sponge_compress(state, scalar, tail, out)` is +ordinary straight-line zkDSL: + +```python +limbs = StackBuf(3) +hint_f192_limbs(limbs, scalar) # advice: scalar's three K coordinates +block = StackBuf(2) +pack64x2_into(limbs[0], limbs[1], block[0]) +pack64x2_into(limbs[2], tail, block[1]) +assert scalar == limbs[0] + Y * (limbs[1] + Y * limbs[2]) +blake3(state, block, out) +``` + +The first two rows range-check all four serialized lanes and form the exact +64-byte BLAKE3 block `[scalar.c0, scalar.c1, scalar.c2, tail]`; the equality +prevents the advice from changing `scalar`. The final row is the VM's sole, +canonical `BLAKE3` instruction. + ## BLAKE3 ```python @@ -608,17 +702,19 @@ the hash root. Parent-node compressions use `parent=1`, the standard IV, and a Operands are size-2 `StackBuf`s or 2-cell slices: - **stack operands** are read/written in place — zero copies; a self-hash - `blake3(h, h, out)` aliases one pair into both inputs; -- the instruction addresses its **four input words independently**, so when a - 256-bit operand is *assembled* from values that live in different cells — - the idiom `p = StackBuf(2); p[0] = tweak; p[1] = pp; blake3(p, …)` — the - copies vanish: a stack store of a plain copy or a zero is forwarded to its - source (see "Variables"), and `BLAKE3` reads each word where it already is; + `blake3(h, h, out)` aliases one 2-cell pair into both inputs; +- the instruction addresses its **four canonical 128-bit message chunks + independently** (each is a full F192 memory cell constrained at this use to + the BLAKE3 subspace `c2 = 0`), so when a 256-bit operand is + *assembled* from values that live in different places — the idiom + `p = StackBuf(2); p[0] = t0; p[1] = t1; blake3(p, …)` — the copies vanish: + a stack store of a plain copy or a zero is forwarded to its source (see + "Variables"), and `BLAKE3` reads each chunk where it already is; - the chaining value has only one opcode offset and therefore must be - consecutive. If a 2-cell `cv` was assembled from non-adjacent copied words, - the compiler materializes those two words into a fresh consecutive run; + consecutive. If a 2-cell `cv` was assembled from non-adjacent copied cells, + the compiler materializes those two cells into a fresh consecutive run; - **heap slices** are still bridged through the stack for the *input pull* (the - operand's word comes from the heap): +1 `DEREF` per heap word, and the output, + operand's words come from the heap): +1 `DEREF` per heap cell, and the output, if a heap slice, is stored after — write-once memory fills whichever side is unset. @@ -650,7 +746,7 @@ critical vulnerability. Runtime-start heap slices (`buf[i:i + k]`, `k` a literal) work too. The prover supplies streams with `program.set_witness("name", entries)` -(`Vec>`); test programs declare them as annotations, one line per +(`Vec>`); test programs declare them as annotations, one line per entry — repeated lines with the same name are its successive entries: ```python @@ -680,9 +776,11 @@ completely unconstrained: the program must re-verify them in-circuit. | `x = ` / `GEN ** k` | 1 `SET` | | `a + b` | 1 `XOR` | | `a * b` | 1 `MUL` | +| `a / b` | 1 `MUL` (write-once back-solve; division by zero is undefined) | | heap read / store `buf[i]` | 1 `DEREF`; +1 `MUL` for a *runtime* index (a compile-time g-power offset folds into the `DEREF` — free) | | stack read / store `sa[k]` | 0 (direct cell addressing) | | `assert a == b` | 2 | +| `assert a != b` | 3 on the accepting path (+ amortized branch setup) | | `assert log x < k` | 3 (+1 `SET` amortized per bound per frame) | | `if a == b: …` | 3 (+2 to skip a non-empty `else`; +2 amortized `self-fp` per branching function); **0 if the condition is compile-time** | | `match log(x): …` | ≈ 7, independent of the case count | @@ -725,4 +823,4 @@ Mutable variables; conditions other than field (in)equality; `match` defaults `mul_range` or range-check bounds (a substituted literal is a bit-pattern element, not the g-power a bound needs); runtime slice starts on a `StackBuf`; runtime range-check bounds (`assert log a < log b` with runtime `b`); -precompiles beyond `BLAKE3`. +precompiles beyond `BLAKE3` and `PACK64X2`. diff --git a/crates/lean_vm/src/blake3_flock.rs b/crates/lean_vm/src/blake3_flock.rs index 491d5bb9..51e4b265 100644 --- a/crates/lean_vm/src/blake3_flock.rs +++ b/crates/lean_vm/src/blake3_flock.rs @@ -1,12 +1,13 @@ //! Bridge to the flock BLAKE3 prover ([`flock::blake3`]), single-PCS. //! -//! `q_pkd` (flock's packed BLAKE3 witness) is committed as a column in leanVM-b's -//! ONE stacked witness (§3.1) — no separate flock commitment. The VM's `BLAKE3` -//! table binds to it by point-eval equality (its value columns and `q_pkd`'s -//! slots are point-evals of the same committed stack), and flock's R1CS validity -//! is discharged by a Ligerito over that same stacked commitment -//! ([`flock::blake3::Blake3Setup::prove_validity_stacked`], -//! which lifts the ring-switch weight into the stack domain). +//! `q_pkd` (flock's packed BLAKE3 witness, 64 bits per `F64` word) is committed +//! as a column in leanVM-b's ONE stacked `F64` witness (§3.1) — no separate flock +//! commitment. The VM's `BLAKE3` table binds to it by point-eval equality (its +//! value columns and `q_pkd`'s slots are point-evals of the same committed +//! stack), and flock's R1CS validity is discharged by the same stacked Ligerito: +//! the reduction's two tower-field claims pass through +//! [`ring_switch_open`] / [`ring_switch_verify`] and join the batch-mixed +//! opening ([`::pcs::stack_open`]). //! //! ## The mapping //! @@ -15,30 +16,29 @@ //! little-endian order. All inputs are witness values in `q_pkd`; memory binds //! `a`, `b`, and `cv`, while the bytecode interaction binds `metadata`. //! -//! ## The layout (after the alignment re-layout, `M_BASE = 640`) +//! ## The layout (aligned re-layout, `M_BASE = 640`, 64-bit words) //! //! Each compression's `2^K_LOG` bits pack into [`PACKED_PER_INSTANCE`]` -//! = 2^(K_LOG-7)` `F128` coordinates; each VM-visible 128-bit word is one whole -//! packed coordinate at a fixed within-instance slot: +//! = 2^(K_LOG-6)` `F64` words; each VM-visible 64-bit word is one whole packed +//! word at a fixed within-instance slot (bit position / 64): //! //! ```text -//! c0,c1 = slots 2,3 a0,a1 = slots 5,6 b0,b1 = slots 7,8 -//! cv = slots 0,1 counter‖blen‖flags = slot 9 +//! c0..c3 = slots 4..8 a0..a3 = slots 10..14 b0..b3 = slots 14..18 +//! cv0..cv3 = slots 0..4 counter = slot 18 blen‖flags = slot 19 //! ``` //! -//! The VM routes claims on all nine aligned words directly to these slots. +//! All compression inputs are free witness rows; the VM routes claims on all +//! eighteen aligned words directly to these slots. -use primitives::field::F128; use crate::transcript::{ProverState, VerifierState}; -use ::pcs::LOG_PACKING; -use ::pcs::Commitment; -#[cfg(test)] -use ::pcs::ProverData; +use ::pcs::pack::{LOG_PACKING, PACKING_WIDTH}; use flock::blake3::{ - Blake3Setup, Compression, K_LOG, ReducedClaims, ReductionReplay, blake3_compress, + Blake3Setup, Compression, K_LOG, PackedWitnessClaims, ReductionReplay, blake3_compress, generate_witness_with_ab_packed_and_lincheck, min_n_blocks_log, }; use flock::verifier::VerifyError; +use primitives::field::{F64, F192}; +use primitives::multilinear::lagrange_weights_naive; /// A `ẑ(point) = value` claim on the committed witness `q_pkd`, recovered by the /// Flock zerocheck + lincheck reduction ([`prove_reduction`] / [`verify_reduction`]) @@ -49,23 +49,23 @@ pub use flock::proof::ZClaim; /// CHUNK_END(2) | ROOT(8) = 11`. pub const FLAGS: u32 = flock::blake3::PINNED_FLAGS; -/// Packed `F128` coordinates per compression instance: `K / 128 = 2^(K_LOG-7)`. +/// Packed `F64` words per compression instance: `K / 64 = 2^(K_LOG-6)`. /// Instance `j` occupies packed indices `[j*PACKED_PER_INSTANCE, (j+1)*…)`. pub const PACKED_PER_INSTANCE: usize = 1 << (K_LOG - LOG_PACKING); -/// Reduction-side buffers emitted in the same fused pass as the committed -/// `q_pkd`. They stay prover-local and are retained only until flock consumes -/// them after the stack commitment. +/// Flock-native reduction buffers emitted in the same fused pass as the +/// committed, flattened `q_pkd`. They stay prover-local until reduction. /// -/// At `2^17` compression slots these three buffers total 768 MiB. Reuse does -/// not increase peak memory—the legacy reduction allocated the same buffers -/// while the committed stack was live—but it deliberately extends their -/// lifetime across commit, bus, and constraint proving to save one witness -/// generation pass. +/// At `2^17` compression slots these buffers total 1.375 GiB: three packed +/// `F192` vectors (`z`, `A·z`, `B·z`) plus the byte-striped lincheck layout. +/// Reuse does not increase peak memory—the legacy reduction allocated the same +/// buffers while the committed stack was live—but it extends their lifetime +/// across commit, bus, and constraint proving to save one witness pass. pub(crate) struct PreparedReductionWitness { n_blocks: usize, - a_packed: Vec, - b_packed: Vec, + z_packed: Vec, + a_packed: Vec, + b_packed: Vec, z_lincheck: Vec, } @@ -74,13 +74,9 @@ impl PreparedReductionWitness { self.n_blocks } - pub(crate) fn prove( - &self, - q_pkd: &[F128], - ps: &mut ProverState, - ) -> ReducedClaims { + pub(crate) fn prove(&self, ps: &mut ProverState) -> PackedWitnessClaims { setup_for(self.n_blocks).prove_reduction_precomputed( - q_pkd, + &self.z_packed, &self.a_packed, &self.b_packed, &self.z_lincheck, @@ -89,85 +85,103 @@ impl PreparedReductionWitness { } } -// Within-instance packed-coordinate (slot) indices of the VM-visible words, -// fixed by the aligned flock layout (asserted by `layout_constants` there). -pub const SLOT_C0: usize = 2; -pub const SLOT_C1: usize = 3; +// Within-instance packed-word (slot) indices of the VM-visible words, fixed by +// the aligned flock layout (bit bases asserted by `layout_constants` there): +// `CV_BASE = 0` → cv words 0..4, `OUT_LO_BASE = 256` → c words 4..8, `M_BASE +// = 640` → a words 10..14 and b words 14..18, metadata (counter, blen‖flags) +// words 18..20. pub const SLOT_CV0: usize = 0; -pub const SLOT_CV1: usize = 1; -pub const SLOT_A0: usize = 5; -pub const SLOT_A1: usize = 6; -pub const SLOT_B0: usize = 7; -pub const SLOT_B1: usize = 8; -pub const SLOT_METADATA: usize = 9; - -/// The nine VM-visible packed slots in canonical table order -/// `[a0,a1,b0,b1,c0,c1,cv0,cv1,metadata]`. -pub const VM_SLOTS: [usize; 9] = [ +pub const SLOT_C0: usize = 4; +pub const SLOT_A0: usize = 10; +pub const SLOT_B0: usize = 14; +pub const SLOT_METADATA: usize = 18; + +/// The eighteen within-instance value slots in canonical order +/// `[a0..a3, b0..b3, c0..c3, cv0..cv3, md_lo, md_hi]`, matching +/// `tables::BLAKE3_VALUE_COLS`. +pub const SLOTS: [usize; 18] = [ SLOT_A0, - SLOT_A1, + SLOT_A0 + 1, + SLOT_A0 + 2, + SLOT_A0 + 3, SLOT_B0, - SLOT_B1, + SLOT_B0 + 1, + SLOT_B0 + 2, + SLOT_B0 + 3, SLOT_C0, - SLOT_C1, + SLOT_C0 + 1, + SLOT_C0 + 2, + SLOT_C0 + 3, SLOT_CV0, - SLOT_CV1, + SLOT_CV0 + 1, + SLOT_CV0 + 2, + SLOT_CV0 + 3, SLOT_METADATA, + SLOT_METADATA + 1, ]; - -/// Split a 128-bit field element into the four little-endian `u32` words flock's -/// message uses (`lo` → words 0,1; `hi` → words 2,3) — the VM memory byte order. -fn words_of(x: F128) -> [u32; 4] { - [x.lo as u32, (x.lo >> 32) as u32, x.hi as u32, (x.hi >> 32) as u32] +/// Split a 64-bit field element into the two little-endian `u32` words flock's +/// message uses — the VM memory byte order. +fn words_of(x: F64) -> [u32; 2] { + [x.0 as u32, (x.0 >> 32) as u32] } -/// Inverse of [`words_of`]: pack four little-endian `u32` words into the `F128`. -pub fn pack_words(w: [u32; 4]) -> F128 { - F128::new( - (w[0] as u64) | ((w[1] as u64) << 32), - (w[2] as u64) | ((w[3] as u64) << 32), - ) +/// Inverse of `words_of`: pack two little-endian `u32` words into the `F64`. +pub fn pack_words(w: [u32; 2]) -> F64 { + F64((w[0] as u64) | ((w[1] as u64) << 32)) } -/// Pack BLAKE3's compression metadata as one little-endian `F128`. -pub const fn metadata(counter: u64, block_len: u32, flags: u32) -> F128 { - F128::new(counter, (block_len as u64) | ((flags as u64) << 32)) +/// Pack BLAKE3's compression metadata as one little-endian 128-bit value in +/// the two low K-lanes of a 192-bit word (top lane zero). +pub const fn metadata(counter: u64, block_len: u32, flags: u32) -> F192 { + F192::new(counter, (block_len as u64) | ((flags as u64) << 32), 0) } -/// Unpack `counter:u64 | block_len:u32 | flags:u32` from one `F128`. -pub const fn unpack_metadata(x: F128) -> (u64, u32, u32) { - (x.lo, x.hi as u32, (x.hi >> 32) as u32) +/// Unpack `counter:u64 | block_len:u32 | flags:u32` from a 192-bit word (the +/// top lane must be zero). +pub const fn unpack_metadata(x: F192) -> (u64, u32, u32) { + assert!(x.c2 == 0, "BLAKE3 metadata must have a zero top lane"); + (x.c0, x.c1 as u32, (x.c1 >> 32) as u32) } -/// BLAKE3's standard IV as the two VM memory words used for a chaining value. -pub const IV: [F128; 2] = [ - F128::new(0xbb67_ae85_6a09_e667, 0xa54f_f53a_3c6e_f372), - F128::new(0x9b05_688c_510e_527f, 0x5be0_cd19_1f83_d9ab), +/// BLAKE3's standard IV as four flock words (the two chaining-value cells' +/// low lanes, in canonical lane order). +pub const IV: [F64; 4] = [ + F64(0xbb67_ae85_6a09_e667), + F64(0xa54f_f53a_3c6e_f372), + F64(0x9b05_688c_510e_527f), + F64(0x5be0_cd19_1f83_d9ab), +]; + +/// The standard IV as the two 192-bit VM memory cells a chaining value +/// occupies (canonical 128-bit chunks, top limbs zero). +pub const IV_CELLS: [F192; 2] = [ + F192::new(0xbb67_ae85_6a09_e667, 0xa54f_f53a_3c6e_f372, 0), + F192::new(0x9b05_688c_510e_527f, 0x5be0_cd19_1f83_d9ab, 0), ]; /// The flock [`Compression`] for one VM instruction. -pub fn compression(a: [F128; 2], b: [F128; 2], cv: [F128; 2], meta: F128) -> Compression { +pub fn compression(a: [F64; 4], b: [F64; 4], cv: [F64; 4], meta: F192) -> Compression { let mut m = [0u32; 16]; - m[0..4].copy_from_slice(&words_of(a[0])); - m[4..8].copy_from_slice(&words_of(a[1])); - m[8..12].copy_from_slice(&words_of(b[0])); - m[12..16].copy_from_slice(&words_of(b[1])); + for (i, &w) in a.iter().enumerate() { + m[2 * i..2 * i + 2].copy_from_slice(&words_of(w)); + } + for (i, &w) in b.iter().enumerate() { + m[8 + 2 * i..8 + 2 * i + 2].copy_from_slice(&words_of(w)); + } let mut cv_words = [0u32; 8]; - cv_words[0..4].copy_from_slice(&words_of(cv[0])); - cv_words[4..8].copy_from_slice(&words_of(cv[1])); + for (i, &w) in cv.iter().enumerate() { + cv_words[2 * i..2 * i + 2].copy_from_slice(&words_of(w)); + } let (counter, block_len, flags) = unpack_metadata(meta); (cv_words, m, counter, block_len, flags) } -/// The low 256-bit output `c = (c0, c1)` of an arbitrary compression. This is +/// The low 256-bit output `c = (c0..c3)` of an arbitrary compression. This is /// `blake3::hash(a‖b)` only for the standard IV and one-block root metadata. -pub fn digest(block: &Compression) -> [F128; 2] { +pub fn digest(block: &Compression) -> [F64; 4] { let st = blake3_compress(&block.0, &block.1, block.2, block.3, block.4); - [ - pack_words([st[0], st[1], st[2], st[3]]), - pack_words([st[4], st[5], st[6], st[7]]), - ] + std::array::from_fn(|k| pack_words([st[2 * k], st[2 * k + 1]])) } /// flock's `n_blocks_log` for `n` compressions (lincheck floor `≥ 3`). The VM's @@ -178,7 +192,7 @@ pub fn n_blocks_log(n: usize) -> usize { } /// The variable count (`log2` length) of the committed `q_pkd` column for `n` -/// executed compressions: `K_LOG + n_blocks_log(max(n,1)) - 7`. Always ≥ 1 +/// executed compressions: `K_LOG + n_blocks_log(max(n,1)) - 6`. Always ≥ 1 /// instance — `n = 0` still commits one padding instance (uniform proof shape). pub fn qpkd_kappa(n: usize) -> usize { K_LOG + n_blocks_log(n.max(1)) - LOG_PACKING @@ -192,26 +206,40 @@ pub fn padding_compression() -> Compression { flock::blake3::padding_block() } +/// Flatten flock's packed witness (128 bits per `F192` word, bit `i` at +/// position `i`) into the committed `F64` packing (64 bits per word): word `j` +/// becomes words `2j` (lo lanes, bits 0..64) and `2j+1` (hi lanes, bits +/// 64..128), which is exactly `pack_witness`'s convention on the same bit string. +fn flatten_packed(packed: &[F192]) -> Vec { + let mut out = Vec::with_capacity(packed.len() * 2); + for w in packed { + debug_assert_eq!(w.c2, 0, "Flock's 128-bit packed witness escaped its subspace"); + out.push(F64(w.c0)); + out.push(F64(w.c1)); + } + out +} + /// Build the committed `q_pkd` column (flock's packed witness) for `blocks`, padded /// to `2^n_blocks_log(max(blocks.len(),1))` instances (the unused ones /// [`padding_compression`] blocks). Deterministic, so it matches what the reduction /// regenerates. An empty `blocks` yields one padding cube (all instances are padding). -pub fn build_qpkd(blocks: &[Compression]) -> Vec { +pub fn build_qpkd(blocks: &[Compression]) -> Vec { build_qpkd_prepared(blocks).0 } -/// Build `q_pkd` and retain the `A·z`, `B·z`, and lincheck layouts produced by -/// that same witness pass, so the reduction does not regenerate them later. -pub(crate) fn build_qpkd_prepared( - blocks: &[Compression], -) -> (Vec, PreparedReductionWitness) { +/// Build the committed `q_pkd` and retain the Flock-native layouts produced by +/// that same fused pass so reduction does not regenerate them later. +pub(crate) fn build_qpkd_prepared(blocks: &[Compression]) -> (Vec, PreparedReductionWitness) { let n_blocks = blocks.len().max(1); - let (q_pkd, a_packed, b_packed, z_lincheck) = + let (z_packed, a_packed, b_packed, z_lincheck) = generate_witness_with_ab_packed_and_lincheck(blocks, n_blocks_log(n_blocks)); + let q_pkd = flatten_packed(&z_packed); ( q_pkd, PreparedReductionWitness { n_blocks, + z_packed, a_packed, b_packed, z_lincheck, @@ -219,55 +247,44 @@ pub(crate) fn build_qpkd_prepared( ) } -/// The digest `(c0, c1)` of [`padding_compression`], i.e. `blake3(0^64)`. It is +/// The digest `(c0..c3)` of [`padding_compression`], i.e. `blake3(0^64)`. It is /// NONZERO, so the VM pads its BLAKE3 output value columns with this. -pub fn padding_digest() -> [F128; 2] { +pub fn padding_digest() -> [F64; 4] { digest(&padding_compression()) } -/// `log2` of the within-instance packed span (`PACKED_PER_INSTANCE = 2^7`): the +/// `log2` of the within-instance packed span (`PACKED_PER_INSTANCE = 2^8`): the /// number of low coords of a `q_pkd` point that carry the slot's bits, and the -/// stride between consecutive instances' same-slot coords in `q_pkd`. A value +/// stride between consecutive instances' same-slot words in `q_pkd`. A value /// claim on `q_pkd` is thus a boolean-selector (strided) claim with this stride. pub const SLOT_STRIDE_LOG: usize = K_LOG - LOG_PACKING; -/// Memoized BLAKE3 R1CS [`Blake3Setup`], keyed by the executed-instance count. +/// Memoized BLAKE3 R1CS [`Blake3Setup`], keyed by its power-of-two shape. /// Building it (the symbolic constraint walk over `2^K_LOG` slots) costs /// ~hundreds of ms — fixed per circuit shape, independent of `N` or the proof. /// So we build each shape once and reuse it across `prove`, `verify`, and /// repeated proofs; the per-setup caches then stay warm, making verification /// milliseconds rather than rebuilding the circuit each time. /// -/// The cache is bounded ([`SETUP_CACHE_CAP`]): `verify` calls this with the -/// PROVER-ANNOUNCED count, so an attacker cycling distinct counts could otherwise -/// grow it without limit. Past the cap we build an ephemeral (uncached) setup — -/// correct, just not memoized; legit workloads use only a handful of sizes. -const SETUP_CACHE_CAP: usize = 256; - -fn setup_cache() -> &'static std::sync::Mutex>> { - static CACHE: std::sync::OnceLock>>> = +type SetupCell = std::sync::Arc>>; + +fn setup_cache() -> &'static std::sync::Mutex> { + static CACHE: std::sync::OnceLock>> = std::sync::OnceLock::new(); CACHE.get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new())) } fn setup_for(n_blocks: usize) -> std::sync::Arc { - let cache = setup_cache(); - // Fast path: build OUTSIDE the lock so a concurrent builder (e.g. the - // background warm spawned by `cpu::prove`) doesn't serialize behind us — the - // ~hundreds-of-ms build must not hold the mutex. - if let Some(s) = cache.lock().expect("BLAKE3 setup cache poisoned").get(&n_blocks) { - return std::sync::Arc::clone(s); - } - let setup = std::sync::Arc::new(Blake3Setup::new(n_blocks)); - let mut map = cache.lock().expect("BLAKE3 setup cache poisoned"); - // Re-check: another thread may have inserted while we built (harmless — one wins). - if let Some(s) = map.get(&n_blocks) { - return std::sync::Arc::clone(s); - } - if map.len() < SETUP_CACHE_CAP { - map.insert(n_blocks, std::sync::Arc::clone(&setup)); - } - setup + let shape = n_blocks_log(n_blocks); + let cell = { + let mut cache = setup_cache().lock().expect("BLAKE3 setup cache poisoned"); + std::sync::Arc::clone( + cache + .entry(shape) + .or_insert_with(|| std::sync::Arc::new(std::sync::OnceLock::new())), + ) + }; + std::sync::Arc::clone(cell.get_or_init(|| std::sync::Arc::new(Blake3Setup::new(1usize << shape)))) } /// Pre-build (and cache) the flock BLAKE3 R1CS setup. This is the fixed, @@ -295,135 +312,157 @@ pub fn family_digest() -> [u8; 32] { } /// **Flock reduction only** (prover): run flock's BLAKE3 zerocheck + lincheck -/// over `blocks`, binding to `commitment`, and return the two claims -/// [`ReducedClaims`] on the committed witness `q_pkd` — `ab` (`A∘B`, lincheck) -/// and `c` (`C`, zerocheck) — along with the regenerated packed witness. The -/// sub-proof scalars ride the shared transcript stream (`ps.add_scalar` at the -/// protocol points). Does NOT open the PCS: the caller discharges the returned -/// claims via [`crate::pcs::open`] (as [`crate::cpu`]'s prove does). This is -/// the clean seam the PCS builds on. +/// over `blocks` and return the two [`PackedWitnessClaims`] on the committed +/// witness `q_pkd` — `ab` (`A∘B`, lincheck) and `c` (`C`, zerocheck) — along +/// with the regenerated packed witness (already flattened to the committed +/// `F64` packing). The sub-proof scalars ride the shared transcript stream +/// (`ps.add_scalar` at the protocol points); flock runs natively in the tower +/// field on the shared sponge. Does NOT open the PCS: the caller discharges the +/// returned claims via [`crate::pcs::open`] (as [`crate::cpu`]'s prove does). +/// The statement is already transcript-bound (the fs_seed, the announced +/// sizes, and the commitment root on the stream), so `commitment` is only a +/// symmetry witness that the caller committed first. pub fn prove_reduction( blocks: &[Compression], - commitment: &Commitment, + commitment: &::pcs::ligerito::Commitment, ps: &mut ProverState, -) -> (Vec, ReducedClaims) { - setup_for(blocks.len()).prove_reduction(blocks, commitment, ps) +) -> (Vec, PackedWitnessClaims) { + let _ = commitment; + let (z_packed, reduced) = setup_for(blocks.len()).prove_reduction(blocks, ps); + (flatten_packed(&z_packed), reduced) } -/// **Flock reduction only** (verifier): mirror of [`prove_reduction`]. Rebuild the -/// stack commitment from `root`/`mu`, replay the zerocheck + lincheck sub-proofs -/// straight off the shared stream (each scalar bound as it is read), and recover -/// the two `(ab, c)` claims on `q_pkd` for the PCS to discharge — plus the -/// reassembled records and reduction claims ([`ReductionReplay`]). +/// **Flock reduction only** (verifier): mirror of [`prove_reduction`]. Replay +/// the zerocheck + lincheck sub-proofs straight off the shared stream (each +/// scalar bound as it is read), and recover the two `(ab, c)` claims on `q_pkd` +/// for the PCS to discharge — plus the reassembled reduction claims +/// ([`ReductionReplay`]). `root`/`mu` are symmetry witnesses (the statement is +/// bound via the seed + announced sizes + the root on the stream). pub fn verify_reduction( n_blocks: usize, root: &[u8; 32], mu: usize, vs: &mut VerifierState, ) -> Result { - let commitment = crate::pcs::commitment_from_root(*root, mu); - setup_for(n_blocks).verify_reduction(&commitment, vs) + let _ = (root, mu); + setup_for(n_blocks).verify_reduction(vs) } -/// The multilinear tail `x_inner_rest ++ x_outer` of a quirky point — the -/// `x_outer_full` the PCS ring-switch front-end consumes. -fn x_outer_full(point: &flock::lincheck::QuirkyPoint) -> Vec { - let mut v = point.x_inner_rest.clone(); - v.extend_from_slice(&point.x_outer); - v +/// One flock claim as a tower [`crate::pcs::RingSwitchClaim`]: the quirky point +/// splits at the packing boundary. Its univariate-skip coordinate `z_skip` +/// covers exactly the `k_skip = LOG_PACKING = 6` packed variables, so the +/// packing prefix is the 64 φ8-Lagrange weights at `z_skip`, and the WHOLE +/// multilinear tail `x_inner_rest ++ x_outer` is the suffix point (`q_pkd` has +/// `2^(K_LOG + n_log − 6)` words, and no coordinate is split off into the +/// prefix). +fn ring_claim(z: &ZClaim, captured: Option<&[F192]>, qpkd_vars: usize) -> crate::pcs::RingSwitchClaim { + let prefix_weights: Vec = lagrange_weights_naive(LOG_PACKING, z.point.z_skip); + let mut suffix_point: Vec = z.point.x_inner_rest.clone(); + suffix_point.extend_from_slice(&z.point.x_outer); + // Length invariant: prefix (6) + suffix == K_LOG + n_blocks_log, i.e. the + // suffix spans exactly the committed q_pkd cube. + assert_eq!( + suffix_point.len(), + qpkd_vars, + "ring-switch suffix must span the q_pkd cube" + ); + // Precomputed s_hat_v (prover side): flock's reduction captures the 128 + // bit-slice MLEs w.r.t. its OWN 128-bit packing, whose prefix absorbs + // z_skip AND the first inner-rest coordinate `c`; the 64-bit packing here + // keeps `c` in the suffix. The 64-wide values recombine linearly: 64-word + // `y = 2y' + b` is the b-half of 128-word `y'`, and bit `i` of that half + // is bit `i + 64b` of the 128-word, so + // s64[i] = (1+c)·s128[i] + c·s128[i+64]. + // Lincheck already captures the 64 slices expected by the K ring switch. + // Zerocheck's fused kernel captures two 64-slice banks around the first + // suffix coordinate; fold that coordinate here without rescanning q_pkd. + let s_hat_v = captured.and_then(|s| match s.len() { + PACKING_WIDTH => Some(s.to_vec()), + n if n == 2 * PACKING_WIDTH && !z.point.x_inner_rest.is_empty() => { + let c = z.point.x_inner_rest[0]; + Some( + (0..PACKING_WIDTH) + .map(|i| (F192::ONE + c) * s[i] + c * s[i + PACKING_WIDTH]) + .collect(), + ) + } + _ => None, + }); + crate::pcs::RingSwitchClaim { + prefix_weights, + suffix_point, + value: z.value, + s_hat_v, + } } -/// Package the prover's reduction claims ([`ReducedClaims`]) as a +/// Package the prover's reduction claims ([`PackedWitnessClaims`]) as a /// [`crate::pcs::RingSwitchOpen`], so the PCS discharges flock's `(ab, c)` -/// validity in the SAME opening as leanVM's point claims. `offset` is `q_pkd`'s +/// validity in the same opening as leanVM's point claims. `offset` is `q_pkd`'s /// slot in the committed stack; the opener slices `q_pkd` from there. -pub fn ring_switch_open(n_blocks: usize, offset: usize, reduced: &ReducedClaims) -> crate::pcs::RingSwitchOpen { - let setup = setup_for(n_blocks); +pub fn ring_switch_open(n_blocks: usize, offset: usize, reduced: &PackedWitnessClaims) -> crate::pcs::RingSwitchOpen { + let qpkd_vars = qpkd_kappa(n_blocks); crate::pcs::RingSwitchOpen { offset, - qpkd_vars: qpkd_kappa(n_blocks), - x_outers: vec![ - x_outer_full(&reduced.ab.claim.point), - x_outer_full(&reduced.c.claim.point), + qpkd_vars, + claims: vec![ + ring_claim(&reduced.ab.claim, reduced.ab.s_hat_v.as_deref(), qpkd_vars), + ring_claim(&reduced.c.claim, reduced.c.s_hat_v.as_deref(), qpkd_vars), ], - s_hat_v: vec![reduced.ab.s_hat_v.clone(), reduced.c.s_hat_v.clone()], - padding: ::pcs::PaddingSpec { - k_log: setup.r1cs.k_log, - useful_bits_per_block: setup.r1cs.useful_bits, - }, } } /// Verifier counterpart of [`ring_switch_open`]: package the recovered `(ab, c)` -/// claims (from [`verify_reduction`]) as a [`crate::pcs::RingSwitchVerify`]. +/// claims (from [`verify_reduction`]) as a [`crate::pcs::RingSwitchVerify`], the +/// same statement data; the transmitted opening travels separately (read off the +/// `openings` hint channel by the caller). pub fn ring_switch_verify(n_blocks: usize, offset: usize, ab: ZClaim, c: ZClaim) -> crate::pcs::RingSwitchVerify { + let qpkd_vars = qpkd_kappa(n_blocks); crate::pcs::RingSwitchVerify { offset, - qpkd_vars: qpkd_kappa(n_blocks), - values: vec![ab.value, c.value], - z_skips: vec![ab.point.z_skip, c.point.z_skip], - x_outers: vec![x_outer_full(&ab.point), x_outer_full(&c.point)], + qpkd_vars, + claims: vec![ring_claim(&ab, None, qpkd_vars), ring_claim(&c, None, qpkd_vars)], } } // (No write/read_stack_proof: flock's scalar sub-proof rides the shared stream // via add_scalar/next_scalar at its protocol points, exactly like leanVM's own -// scalars; the one Merkle-bearing Ligerito rides the `openings` hint channel.) - -/// Prove `blocks` are valid compressions in two clean phases, discharging the -/// proof against the caller's already-committed `stack` (with `q_pkd` the aligned -/// sub-block at `stack_offset`), reusing its `prover_data`/`commitment`, on the -/// shared transcript `ps`: -/// 1. the Flock reduction ([`prove_reduction`]): zerocheck + lincheck → the -/// `(ab, c)` claims on `q_pkd`; -/// 2. the PCS: one stacked Ligerito discharging those claims together with the -/// caller's `stack_pd` point claims. -#[allow(clippy::too_many_arguments)] -#[cfg(test)] -pub(crate) fn prove_validity_stacked( - blocks: &[Compression], - stack: &[F128], - stack_offset: usize, - prover_data: &ProverData, - commitment: &Commitment, - stack_pd: &[(Vec, F128)], - ps: &mut ProverState, -) -> ::pcs::ligerito::LigeritoProof { - setup_for(blocks.len()) - .prove_validity_stacked(blocks, stack, stack_offset, prover_data, commitment, stack_pd, ps) -} - -/// Verifier side of [`prove_validity_stacked`], in the same two phases: -/// [`verify_reduction`] (replay zerocheck + lincheck → `(ab, c)` claims), then -/// verify the SINGLE stacked Ligerito against `commitment` on the shared -/// transcript. `stack_pd` are all of leanVM's point claims (bus / constraint / -/// public-input / binding) folded into the same opening. -#[cfg(test)] -pub(crate) fn verify_validity_stacked( - n_blocks: usize, - commitment: &Commitment, - stack_offset: usize, - stack_pd: &[(Vec, F128)], - open: &::pcs::ligerito::LigeritoProof, - vs: &mut VerifierState, -) -> Result<(), VerifyError> { - setup_for(n_blocks).verify_validity_stacked(commitment, stack_offset, stack_pd, open, vs) -} +// scalars; the one Merkle-bearing stacked opening rides the `openings` hint +// channel.) #[cfg(test)] mod tests { use super::*; - fn f(lo: u64, hi: u64) -> F128 { - F128::new(lo, hi) + #[test] + fn setup_cache_is_keyed_by_shape() { + let one = setup_for(1); + let eight = setup_for(8); + let nine = setup_for(9); + assert!(std::sync::Arc::ptr_eq(&one, &eight)); + assert!(!std::sync::Arc::ptr_eq(&eight, &nine)); + } + + fn f(x: u64) -> F64 { + F64(x) } fn sample_blocks(n: usize) -> Vec { (0..n as u64) .map(|i| { compression( - [f(0x11 * (i + 1), 0x22 * (i + 1)), f(0x33 * (i + 1), 0x44 * (i + 1))], - [f(0x55 * (i + 1), 0x66 * (i + 1)), f(0x77 * (i + 1), 0x88 * (i + 1))], + [ + f(0x11 * (i + 1)), + f(0x22 * (i + 1)), + f(0x33 * (i + 1)), + f(0x44 * (i + 1)), + ], + [ + f(0x55 * (i + 1)), + f(0x66 * (i + 1)), + f(0x77 * (i + 1)), + f(0x88 * (i + 1)), + ], IV, metadata(0, 64, FLAGS), ) @@ -431,15 +470,15 @@ mod tests { .collect() } - /// `q_pkd`'s aligned packed slots hold the VM's 128-bit words in our field + /// `q_pkd`'s aligned packed slots hold the VM's 64-bit words in our field /// representation, and the digest matches the `blake3` crate. #[test] fn qpkd_words_match_layout() { - let inputs: Vec<([F128; 2], [F128; 2])> = (0..5u64) + let inputs: Vec<([F64; 4], [F64; 4])> = (0..5u64) .map(|i| { ( - [f(0x1000 + i, 0x2000 + i), f(0x3000 + i, 0x4000 + i)], - [f(0x5000 + i, 0x6000 + i), f(0x7000 + i, 0x8000 + i)], + [f(0x1000 + i), f(0x2000 + i), f(0x3000 + i), f(0x4000 + i)], + [f(0x5000 + i), f(0x6000 + i), f(0x7000 + i), f(0x8000 + i)], ) }) .collect(); @@ -452,91 +491,31 @@ mod tests { let slot = |j: usize, s: usize| q_pkd[j * PACKED_PER_INSTANCE + s]; for (j, (&(a, b), blk)) in inputs.iter().zip(&blocks).enumerate() { - assert_eq!(slot(j, SLOT_A0), a[0]); - assert_eq!(slot(j, SLOT_A1), a[1]); - assert_eq!(slot(j, SLOT_B0), b[0]); - assert_eq!(slot(j, SLOT_B1), b[1]); + for k in 0..4 { + assert_eq!(slot(j, SLOT_A0 + k), a[k]); + assert_eq!(slot(j, SLOT_B0 + k), b[k]); + } let mut input = [0u8; 64]; - for (s, w) in input.chunks_exact_mut(16).zip([a[0], a[1], b[0], b[1]]) { - s[..8].copy_from_slice(&w.lo.to_le_bytes()); - s[8..].copy_from_slice(&w.hi.to_le_bytes()); + for (s, w) in input.chunks_exact_mut(8).zip(a.into_iter().chain(b)) { + s.copy_from_slice(&w.0.to_le_bytes()); } let h = *blake3::hash(&input).as_bytes(); - let word = |o: usize| { - F128::new( - u64::from_le_bytes(h[o..o + 8].try_into().unwrap()), - u64::from_le_bytes(h[o + 8..o + 16].try_into().unwrap()), - ) - }; - assert_eq!(digest(blk), [word(0), word(16)]); - assert_eq!(slot(j, SLOT_C0), word(0)); - assert_eq!(slot(j, SLOT_C1), word(16)); + let word = |o: usize| F64(u64::from_le_bytes(h[o..o + 8].try_into().unwrap())); + let d: [F64; 4] = std::array::from_fn(|k| word(8 * k)); + assert_eq!(digest(blk), d); + for k in 0..4 { + assert_eq!(slot(j, SLOT_C0 + k), d[k]); + } } - // Input slots for this default-root test: cv = IV in slots 0,1 and the packed - // counter‖counter_hi‖block_len‖flags word in slot 9. + // Input slots for this default-root test: cv = IV in slots 0..4, the + // zero counter word in slot 18, and the packed block_len‖flags word in + // slot 19. let iv = flock::blake3::BLAKE3_IV; - assert_eq!(slot(0, 0), pack_words([iv[0], iv[1], iv[2], iv[3]])); - assert_eq!(slot(0, 1), pack_words([iv[4], iv[5], iv[6], iv[7]])); - assert_eq!(slot(0, 9), pack_words([0, 0, 64, FLAGS])); - } - - /// flock's validity proof, discharged by a Ligerito over a single committed - /// stack containing `q_pkd` (plus a dummy column) — proves and verifies on - /// the shared transcript, and a corrupted `q_pkd` is rejected. - #[test] - fn validity_stacked_roundtrip() { - let blocks = sample_blocks(4); - let q_pkd = build_qpkd(&blocks); - let dummy = vec![f(7, 9); 8]; - let cols = vec![q_pkd.clone(), dummy]; - let stacked = crate::witness::stack(&cols); - let offset = stacked.placements[0].offset; - - // Also fold in one full-stack point claim (exercises the pd path of the - // single fused opening). - let pd_point: Vec = (0..stacked.m).map(|i| f(0x100 + i as u64, 0x7)).collect(); - let pd_value = primitives::multilinear::mle_eval(&stacked.q, &pd_point); - let stack_pd = vec![(pd_point, pd_value)]; - - let mut ps = ProverState::new(b"vstack", &[]); - let committed = crate::pcs::commit(&mut ps, &stacked.q); - let proof = prove_validity_stacked( - &blocks, - &stacked.q, - offset, - &committed.prover_data, - &committed.commitment, - &stack_pd, - &mut ps, - ); - let bundle = ps.into_proof(); - - let mut vs = VerifierState::new(b"vstack", &bundle, &[]); - let root = crate::pcs::read_commitment(&mut vs).unwrap(); - let commitment = crate::pcs::commitment_from_root(root, stacked.m); - verify_validity_stacked(blocks.len(), &commitment, offset, &stack_pd, &proof, &mut vs) - .expect("validity verifies"); - - // A mismatched transcript (different domain) diverges the shared sponge, - // so the validity proof must be rejected. - let mut vs_bad = VerifierState::new(b"different-domain", &bundle, &[]); - let root_b = crate::pcs::read_commitment(&mut vs_bad).unwrap(); - let commitment_b = crate::pcs::commitment_from_root(root_b, stacked.m); - assert!( - verify_validity_stacked(blocks.len(), &commitment_b, offset, &stack_pd, &proof, &mut vs_bad).is_err(), - "validity under a mismatched transcript must fail" - ); - - // A tampered pd value must be rejected too. - let mut bad_pd = stack_pd.clone(); - bad_pd[0].1 += F128::ONE; - let mut vs_pd = VerifierState::new(b"vstack", &bundle, &[]); - let root_p = crate::pcs::read_commitment(&mut vs_pd).unwrap(); - let commitment_p = crate::pcs::commitment_from_root(root_p, stacked.m); - assert!( - verify_validity_stacked(blocks.len(), &commitment_p, offset, &bad_pd, &proof, &mut vs_pd).is_err(), - "tampered pd value must fail" - ); + for k in 0..4 { + assert_eq!(slot(0, k), pack_words([iv[2 * k], iv[2 * k + 1]])); + } + assert_eq!(slot(0, 18), pack_words([0, 0])); + assert_eq!(slot(0, 19), pack_words([64, FLAGS])); } /// The Flock reduction (zerocheck + lincheck) is a clean, self-contained @@ -547,13 +526,13 @@ mod tests { fn reduction_roundtrip() { let blocks = sample_blocks(4); let q_pkd = build_qpkd(&blocks); - let dummy = vec![f(7, 9); 8]; + let dummy = vec![f(7); 8]; let stacked = crate::witness::stack(&[q_pkd.clone(), dummy]); let offset = stacked.placements[0].offset; // Prover: commit, then run ONLY the reduction (no PCS open). let mut ps = ProverState::new(b"reduce", &[]); - let committed = crate::pcs::commit(&mut ps, &stacked.q); + let committed = crate::pcs::commit(&mut ps, &stacked.q, crate::pcs::LOG_INV_RATE); let (z_packed, reduced) = prove_reduction(&blocks, &committed.commitment, &mut ps); let bundle = ps.into_proof(); @@ -564,8 +543,7 @@ mod tests { // Verifier: replay the reduction and recover the claims. let mut vs = VerifierState::new(b"reduce", &bundle, &[]); let root = crate::pcs::read_commitment(&mut vs).unwrap(); - let replay = verify_reduction(blocks.len(), &root, stacked.m, &mut vs) - .expect("reduction verifies"); + let replay = verify_reduction(blocks.len(), &root, stacked.m, &mut vs).expect("reduction verifies"); // Prover and verifier agree on the claims left for the PCS. assert_eq!(reduced.ab.claim, replay.ab, "ab claim mismatch"); @@ -582,4 +560,67 @@ mod tests { ); } } + + /// flock's validity claims, discharged by ONE stacked Ligerito over a + /// hand-stacked witness containing `q_pkd` (plus a dummy column) together + /// with an ordinary point claim: the full prove_reduction → ring-switch → + /// stack_open seam without the VM pipeline. Proves and verifies on the + /// shared transcript; a mismatched domain and a tampered point value are + /// rejected. + #[test] + fn validity_stacked_roundtrip() { + let blocks = sample_blocks(4); + let q_pkd = build_qpkd(&blocks); + let dummy: Vec = (0..8u64).map(|i| f(0x9000 + i)).collect(); + let stacked = crate::witness::stack(&[q_pkd.clone(), dummy.clone()]); + let offset = stacked.placements[0].offset; + + // One ordinary point claim on the dummy column (exercises the point-claim + // path of the single fused opening). + let dummy_pl = stacked.placements[1]; + let low_point: Vec = (0..dummy_pl.n_vars) + .map(|i| F192::new(0x100 + i as u64, 0x7, 0x55)) + .collect(); + let pd_value = primitives::multilinear::mle_eval(&dummy, &low_point); + let points = vec![crate::pcs::SlotClaim::Point { + offset: dummy_pl.offset, + low_point: low_point.clone(), + value: pd_value, + }]; + + let mut ps = ProverState::new(b"vstack", &[]); + let committed = crate::pcs::commit(&mut ps, &stacked.q, crate::pcs::LOG_INV_RATE); + let (_z, reduced) = prove_reduction(&blocks, &committed.commitment, &mut ps); + let ring = ring_switch_open(blocks.len(), offset, &reduced); + let open = crate::pcs::open(&mut ps, &committed, &stacked.q, &points, &ring); + ps.hint_opening(open); + let bundle = ps.into_proof(); + + let run = |label: &'static [u8], points: &[crate::pcs::SlotClaim]| -> Result<(), &'static str> { + let mut vs = VerifierState::new(label, &bundle, &[]); + let root = crate::pcs::read_commitment(&mut vs).map_err(|_| "root")?; + let replay = verify_reduction(blocks.len(), &root, stacked.m, &mut vs).map_err(|_| "reduction")?; + let open = vs.next_opening().map_err(|_| "opening hint")?; + let ring = ring_switch_verify(blocks.len(), offset, replay.ab, replay.c); + crate::pcs::verify(&mut vs, points, &ring, open, stacked.m, crate::pcs::LOG_INV_RATE, &root) + .map_err(|_| "opening")?; + vs.finish().map_err(|_| "leftover") + }; + + run(b"vstack", &points).expect("validity verifies"); + + // A mismatched transcript (different domain) diverges the shared sponge, + // so the stacked opening must be rejected. + assert!( + run(b"different-domain", &points).is_err(), + "validity under a mismatched transcript must fail" + ); + + // A tampered point value must be rejected too. + let mut bad_points = points.clone(); + if let crate::pcs::SlotClaim::Point { value, .. } = &mut bad_points[0] { + *value += F192::ONE; + } + assert!(run(b"vstack", &bad_points).is_err(), "tampered point value must fail"); + } } diff --git a/crates/lean_vm/src/constraints.rs b/crates/lean_vm/src/constraints.rs index 0f07c850..c3607682 100644 --- a/crates/lean_vm/src/constraints.rs +++ b/crates/lean_vm/src/constraints.rs @@ -1,4 +1,4 @@ -//! The tables' local constraints (§4.1), proven by ONE sumcheck for all tables. +//! The tables' local constraints (§4.1), proven by one sumcheck for all tables. //! //! Each table folds its identities with a DISJOINT range of one `η`'s powers, so //! the batch is a polynomial in `η` whose coefficients are the individual sums and @@ -35,18 +35,19 @@ //! are no rounds in which no table has joined. use crate::PAR_THRESHOLD; -use primitives::field::{F128, mul_by_x}; -use primitives::multilinear::{add3, build_eq, fold_high_inplace, lagrange_eval, quad_nodes, shrink_eq_high, tri_nodes}; use crate::transcript::{ProverState, VerifierState}; use crate::witness::Column; +use primitives::field::{F192, F192Unreduced, mul_by_g, mul_by_g_e}; +use primitives::multilinear::{ + add3, eq_table, fold_high_inplace, fold_high_k, lagrange_eval, quad_nodes, shrink_eq_high, tri_nodes, xor3, +}; use rayon::prelude::*; -/// One table's involved columns' evaluations at its zerocheck point (fixed column -/// order), reconstructed identically by prover and verifier. +/// One table's involved columns' evaluations at its zerocheck point. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Claims { - pub rho: Vec, - pub evals: Vec, + pub rho: Vec, + pub evals: Vec, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -56,12 +57,10 @@ pub enum Error { FinalMismatch, } -/// A table's row constraint: identity `i` weighted by `pows[i]` (its slice of the -/// batch's `η`-powers), read off the involved columns' values. -pub type Constraint<'a> = Box F128 + Sync + 'a>; +/// One table's row constraint: identity `i` is weighted by `pows[i]`. +pub type Constraint<'a> = Box F192 + Sync + 'a>; -/// One table's place in the batch. Both sides build this list identically -/// ([`crate::cpu`]), so their rounds cannot drift apart. +/// One table's place in the shared batch. pub struct Air<'a> { pub tau: usize, pub n_cols: usize, @@ -69,8 +68,7 @@ pub struct Air<'a> { pub eval: Constraint<'a>, } -/// Where each table's disjoint range of `η`-powers starts: the exclusive prefix -/// sum of the identity counts. The recursion guest bakes these same offsets. +/// Start of each table's disjoint range of `η`-powers. pub fn eta_offsets(n_constraints: impl Iterator) -> Vec { n_constraints .scan(0usize, |off, n| { @@ -83,9 +81,9 @@ pub fn eta_offsets(n_constraints: impl Iterator) -> Vec { /// The batch's `η`-powers: `η^0 … η^{total-1}`, sliced per table by [`eta_offsets`]. /// The caller needs these too, to weight the claims it attaches. -pub fn eta_powers(eta: F128, total: usize) -> Vec { +pub fn eta_powers(eta: F192, total: usize) -> Vec { let mut pows = Vec::with_capacity(total); - let mut p = F128::ONE; + let mut p = F192::ONE; for _ in 0..total { pows.push(p); p *= eta; @@ -93,18 +91,57 @@ pub fn eta_powers(eta: F128, total: usize) -> Vec { pows } -/// One table's degree-2 round message at the nodes `{0, 1, g}`; `half` is the -/// stride to the bound (highest) variable's `1` half. Char-2 makes the nodes free: -/// `lo`, `hi`, and `lo + mul_by_x(lo+hi)` — a shift-fold, no PMULL. -fn table_message( +/// First active round for a table: evaluate its `K` columns at `{0,1,g}`. +fn table_message_k( cols: &[Column], - eval: &(dyn Fn(&[F128], &[F128]) -> F128 + Sync), - pows: &[F128], + eval: &(dyn Fn(&[F192], &[F192]) -> F192 + Sync), + pows: &[F192], half: usize, - eqr: &[F128], -) -> [F128; 3] { + eqr: &[F192], +) -> [F192; 3] { let ncols = cols.len(); - let summand = |i: usize, scratch: &mut [F128]| -> [F128; 3] { + let summand = |i: usize, scratch: &mut [F192]| -> [F192Unreduced; 3] { + let e = eqr[i]; + let (v0, rest) = scratch.split_at_mut(ncols); + let (v1, v2) = rest.split_at_mut(ncols); + for (ci, c) in cols.iter().enumerate() { + let (lo, hi) = (c[i], c[i + half]); + v0[ci] = F192::from(lo); + v1[ci] = F192::from(hi); + v2[ci] = F192::from(lo + mul_by_g(lo + hi)); + } + [ + e.mul_unreduced(eval(pows, v0)), + e.mul_unreduced(eval(pows, v1)), + e.mul_unreduced(eval(pows, v2)), + ] + }; + let acc = if half >= PAR_THRESHOLD { + (0..half) + .into_par_iter() + .fold( + || ([F192Unreduced::ZERO; 3], vec![F192::ZERO; 3 * ncols]), + |(acc, mut scratch), i| (xor3(acc, summand(i, &mut scratch)), scratch), + ) + .map(|(acc, _)| acc) + .reduce(|| [F192Unreduced::ZERO; 3], xor3) + } else { + let mut scratch = vec![F192::ZERO; 3 * ncols]; + (0..half).fold([F192Unreduced::ZERO; 3], |acc, i| xor3(acc, summand(i, &mut scratch))) + }; + [acc[0].reduce(), acc[1].reduce(), acc[2].reduce()] +} + +/// Later active rounds after the table has been lifted into `E`. +fn table_message_e( + cols: &[Vec], + eval: &(dyn Fn(&[F192], &[F192]) -> F192 + Sync), + pows: &[F192], + half: usize, + eqr: &[F192], +) -> [F192; 3] { + let ncols = cols.len(); + let summand = |i: usize, scratch: &mut [F192]| -> [F192Unreduced; 3] { let e = eqr[i]; let (v0, rest) = scratch.split_at_mut(ncols); let (v1, v2) = rest.split_at_mut(ncols); @@ -112,23 +149,28 @@ fn table_message( let (lo, hi) = (c[i], c[i + half]); v0[ci] = lo; v1[ci] = hi; - v2[ci] = lo + mul_by_x(lo + hi); + v2[ci] = lo + mul_by_g_e(lo + hi); } - [e * eval(pows, v0), e * eval(pows, v1), e * eval(pows, v2)] + [ + e.mul_unreduced(eval(pows, v0)), + e.mul_unreduced(eval(pows, v1)), + e.mul_unreduced(eval(pows, v2)), + ] }; - if half >= PAR_THRESHOLD { + let acc = if half >= PAR_THRESHOLD { (0..half) .into_par_iter() .fold( - || ([F128::ZERO; 3], vec![F128::ZERO; 3 * ncols]), - |(acc, mut scratch), i| (add3(acc, summand(i, &mut scratch)), scratch), + || ([F192Unreduced::ZERO; 3], vec![F192::ZERO; 3 * ncols]), + |(acc, mut scratch), i| (xor3(acc, summand(i, &mut scratch)), scratch), ) .map(|(acc, _)| acc) - .reduce(|| [F128::ZERO; 3], add3) + .reduce(|| [F192Unreduced::ZERO; 3], xor3) } else { - let mut scratch = vec![F128::ZERO; 3 * ncols]; - (0..half).fold([F128::ZERO; 3], |acc, i| add3(acc, summand(i, &mut scratch))) - } + let mut scratch = vec![F192::ZERO; 3 * ncols]; + (0..half).fold([F192Unreduced::ZERO; 3], |acc, i| xor3(acc, summand(i, &mut scratch))) + }; + [acc[0].reduce(), acc[1].reduce(), acc[2].reduce()] } /// Prove that every table's batched constraint vanishes on all of its rows, as ONE @@ -138,40 +180,46 @@ fn table_message( pub fn prove( airs: &[Air<'_>], cols: &mut [Vec], - eta: F128, - zeta: &[F128], - sigma: &[F128], + eta: F192, + zeta: &[F192], + sigma: &[F192], ps: &mut ProverState, ) -> Vec { let n = airs.iter().map(|a| a.tau).max().unwrap_or(0); debug_assert!(zeta.len() >= n, "the eq point must cover the tallest table"); let offsets = eta_offsets(airs.iter().map(|a| a.n_constraints)); let pows = eta_powers(eta, airs.iter().map(|a| a.n_constraints).sum()); - // η^{offset_t}, already inside `pows`; the rounds then fold in the pre-join // challenges and the eq factor, so `weights` is the whole per-table state. - let mut weights = vec![F128::ONE; airs.len()]; + let mut weights = vec![F192::ONE; airs.len()]; // ONE eq table over the low (still free) variables serves every active table. - let mut eqr = build_eq(&zeta[..n.saturating_sub(1)]); + let mut eqr = eq_table(&zeta[..n.saturating_sub(1)]); let nd = tri_nodes(); - let mut rho = vec![F128::ZERO; n]; + let mut rho = vec![F192::ZERO; n]; + let mut folded: Vec>>> = (0..airs.len()).map(|_| None).collect(); // `k`, the challenges drawn so far, common to every air that is still waiting. - let mut k = F128::ONE; + let mut k = F192::ONE; for j in 0..n { let m = n - 1 - j; // the variable this round binds // The waiting airs contribute the line `Y·k·Σσ`, whose slope `u` is all there // is to it. It is NOT sent on its own: it folds into `h` below, and only `h` // travels. `msg` is the joined airs' degree-2 cofactor, `h`'s multiplicand. - let waiting = airs.iter().zip(sigma).filter(|(a, _)| a.tau <= m).fold(F128::ZERO, |acc, (_, &s)| acc + s); + let waiting = airs + .iter() + .zip(sigma) + .filter(|(a, _)| a.tau <= m) + .fold(F192::ZERO, |acc, (_, &s)| acc + s); let u = k * waiting; - let mut msg = [F128::ZERO; 3]; + let mut msg = [F192::ZERO; 3]; for (t, air) in airs.iter().enumerate() { if air.tau > m { let w = &pows[offsets[t]..offsets[t] + air.n_constraints]; - let p = table_message(&cols[t], &*air.eval, w, 1 << m, &eqr); - for i in 0..3 { - msg[i] += weights[t] * p[i]; - } + let p = if let Some(table) = &folded[t] { + table_message_e(table, &*air.eval, w, 1 << m, &eqr) + } else { + table_message_k(&cols[t], &*air.eval, w, 1 << m, &eqr) + }; + msg = add3(msg, p.map(|x| weights[t] * x)); } } shrink_eq_high(&mut eqr); @@ -182,29 +230,38 @@ pub fn prove( let q = quad_nodes(); debug_assert_eq!(q[..3], nd[..], "the cubic's first three nodes are the cofactor's"); let p4 = [msg[0], msg[1], msg[2], lagrange_eval(&nd, &msg, q[3])]; - let h: [F128; 4] = std::array::from_fn(|i| (F128::ONE + zeta[m] + q[i]) * p4[i] + q[i] * u); + let h: [F192; 4] = std::array::from_fn(|i| (F192::ONE + zeta[m] + q[i]) * p4[i] + q[i] * u); // A separate pass: the challenge only exists once the message is bound. ps.add_scalars(&h); let rk = ps.sample(); rho[m] = rk; k *= rk; - let eq_k = F128::ONE + zeta[m] + rk; + let eq_k = F192::ONE + zeta[m] + rk; for (t, air) in airs.iter().enumerate() { weights[t] *= if air.tau > m { eq_k } else { rk }; - if air.tau > m { + if air.tau <= m { + continue; + } + if let Some(table) = &mut folded[t] { if m >= PAR_THRESHOLD.trailing_zeros() as usize { - cols[t].par_iter_mut().for_each(|c| fold_high_inplace(c, rk)); + table.par_iter_mut().for_each(|c| fold_high_inplace(c, rk)); } else { - cols[t].iter_mut().for_each(|c| fold_high_inplace(c, rk)); + table.iter_mut().for_each(|c| fold_high_inplace(c, rk)); } + } else { + folded[t] = Some(cols[t].iter().map(|c| fold_high_k(c, rk)).collect()); } } } airs.iter() - .zip(cols.iter()) - .map(|(air, c)| { - let evals: Vec = c.iter().map(|col| col[0]).collect(); + .enumerate() + .map(|(t, air)| { + let evals: Vec = if let Some(table) = &folded[t] { + table.iter().map(|c| c[0]).collect() + } else { + cols[t].iter().map(|c| F192::from(c[0])).collect() + }; ps.add_scalars(&evals); Claims { rho: rho[..air.tau].to_vec(), @@ -218,9 +275,9 @@ pub fn prove( /// settle against the commitment. pub fn verify( airs: &[Air<'_>], - eta: F128, - zeta: &[F128], - target: F128, + eta: F192, + zeta: &[F192], + target: F192, vs: &mut VerifierState, ) -> Result, Error> { let n = airs.iter().map(|a| a.tau).max().unwrap_or(0); @@ -229,15 +286,14 @@ pub fn verify( } let offsets = eta_offsets(airs.iter().map(|a| a.n_constraints)); let pows = eta_powers(eta, airs.iter().map(|a| a.n_constraints).sum()); - let nd = quad_nodes(); - let mut weights = vec![F128::ONE; airs.len()]; + let mut weights = vec![F192::ONE; airs.len()]; // An ordinary sumcheck for `target`, which the caller supplies. Each round // arrives as the round polynomial itself at `nd`, so the two steps are the // textbook ones and nothing has to be reapplied: no eq factor, no separate // waiting term. `ζ` and the heights enter only `weights`, never the check. let mut claim = target; - let mut rho = vec![F128::ZERO; n]; + let mut rho = vec![F192::ZERO; n]; for j in 0..n { let m = n - 1 - j; let h = vs.next_scalars(4).map_err(|_| Error::Truncated)?; @@ -247,13 +303,13 @@ pub fn verify( let rk = vs.sample(); rho[m] = rk; claim = lagrange_eval(&nd, &h, rk); - let eq_k = F128::ONE + zeta[m] + rk; + let eq_k = F192::ONE + zeta[m] + rk; for (t, air) in airs.iter().enumerate() { weights[t] *= if air.tau > m { eq_k } else { rk }; } } - let mut acc = F128::ZERO; + let mut acc = F192::ZERO; let mut claims = Vec::with_capacity(airs.len()); for (t, air) in airs.iter().enumerate() { let evals = vs.next_scalars(air.n_cols).map_err(|_| Error::Truncated)?; @@ -274,25 +330,23 @@ pub fn verify( mod tests { use super::*; use crate::transcript::{Proof, ProverState, VerifierState}; + use primitives::field::F64; - /// Two identities, `c0·c1 + c2` and `c0 + c3`, so a table is satisfied exactly - /// when `c2 = c0·c1` and `c3 = c0` on every row (characteristic 2). - fn synth_eval(pows: &[F128], v: &[F128]) -> F128 { + fn synth_eval(pows: &[F192], v: &[F192]) -> F192 { pows[0] * (v[0] * v[1] + v[2]) + pows[1] * (v[0] + v[3]) } - /// Rows satisfying both identities, from an arbitrary `(a, b)` per row. fn good_table(tau: usize, salt: u64) -> Vec { let n = 1usize << tau; - let a: Vec = (0..n).map(|i| F128::new(i as u64 + salt, 7 * salt + 1)).collect(); - let b: Vec = (0..n).map(|i| F128::new(3 * i as u64 + 1, salt)).collect(); - let ab: Vec = a.iter().zip(&b).map(|(&x, &y)| x * y).collect(); + let a: Vec = (0..n).map(|i| F64(i as u64 + salt)).collect(); + let b: Vec = (0..n).map(|i| F64(3 * i as u64 + 1 + salt)).collect(); + let ab: Vec = a.iter().zip(&b).map(|(&x, &y)| x * y).collect(); vec![a.clone(), b, ab, a] } /// A third, attached "identity": the linear form `vals[1]`, whose claimed sum /// is an evaluation of column 1 rather than zero. - fn synth_eval_attached(pows: &[F128], v: &[F128]) -> F128 { + fn synth_eval_attached(pows: &[F192], v: &[F192]) -> F192 { synth_eval(pows, v) + pows[2] * v[1] } @@ -307,48 +361,44 @@ mod tests { .collect() } - const SEED: [F128; 2] = [F128::ONE, F128::ZERO]; + const SEED: [F192; 2] = [F192::ONE, F192::ZERO]; /// The eq point and `η` are the caller's; the tests fix them. - fn eta_zeta(taus: &[usize]) -> (F128, Vec) { + fn eta_zeta(taus: &[usize]) -> (F192, Vec) { let n = taus.iter().copied().max().unwrap_or(0); - let eta = F128::new(0x9e37_79b9_7f4a_7c15, 0x1234_5678_9abc_def0); - let zeta = (0..n).map(|i| F128::new(i as u64 + 3, 0x5555 * (i as u64 + 1))).collect(); + let eta = F192::new(0x9e37_79b9_7f4a_7c15, 0x1234_5678_9abc_def0, 7); + let zeta = (0..n) + .map(|i| F192::new(i as u64 + 3, 0x5555 * (i as u64 + 1), i as u64 + 11)) + .collect(); (eta, zeta) } fn run(taus: &[usize], mut cols: Vec>) -> (Proof, Result, Error>) { let airs = airs_for(taus, false); let (eta, zeta) = eta_zeta(taus); - let zeros = vec![F128::ZERO; taus.len()]; + let zeros = vec![F192::ZERO; taus.len()]; let mut ps = ProverState::new(b"zc-test", &SEED); let pclaims = prove(&airs, &mut cols, eta, &zeta, &zeros, &mut ps); let proof = ps.into_proof(); let mut vs = VerifierState::new(b"zc-test", &proof, &SEED); - let vclaims = verify(&airs, eta, &zeta, F128::ZERO, &mut vs); + let vclaims = verify(&airs, eta, &zeta, F192::ZERO, &mut vs); if let Ok(vc) = &vclaims { - assert_eq!(&pclaims, vc, "prover and verifier reconstruct the same claims"); + assert_eq!(&pclaims, vc); } (proof, vclaims) } - /// Tables of DIFFERENT heights batch into one sumcheck, every claim landing on - /// a prefix of the tallest table's point. #[test] fn ragged_batch_verifies() { let taus = [5usize, 3, 5, 0, 1]; - let cols: Vec> = taus.iter().enumerate().map(|(i, &t)| good_table(t, i as u64)).collect(); + let cols = taus.iter().enumerate().map(|(i, &t)| good_table(t, i as u64)).collect(); let claims = run(&taus, cols).1.expect("honest batch verifies"); let tallest = claims.iter().max_by_key(|c| c.rho.len()).unwrap().rho.clone(); for (c, &tau) in claims.iter().zip(&taus) { - assert_eq!(c.rho.len(), tau); - assert_eq!(c.rho, tallest[..tau], "claims land on nested points"); + assert_eq!(c.rho, tallest[..tau]); } } - /// A single violated row in ANY table, including one whose padding rounds - /// dominate, must be caught: the disjoint `eta` ranges stop the tables from - /// cancelling. #[test] fn one_bad_row_in_any_table_is_rejected() { let taus = [5usize, 3, 5, 0, 1]; @@ -356,12 +406,8 @@ mod tests { for col in [2usize, 3] { let mut cols: Vec> = taus.iter().enumerate().map(|(i, &t)| good_table(t, i as u64)).collect(); - let row = (1usize << taus[bad]) - 1; - cols[bad][col][row] += F128::ONE; - assert!( - run(&taus, cols).1.is_err(), - "table {bad} column {col} violation must be rejected" - ); + cols[bad][col][(1usize << taus[bad]) - 1] += F64::ONE; + assert!(run(&taus, cols).1.is_err()); } } } @@ -379,15 +425,15 @@ mod tests { let pows = eta_powers(eta, 3 * taus.len()); // σ_t = η^{offset_t + 2} · col_1(ζ[..τ_t]): the attached identity is `vals[1]`, // so its eq-weighted sum over the table's cube is that column's evaluation. - let sigmas: Vec = taus + let sigmas: Vec = taus .iter() .enumerate() .map(|(t, &tau)| pows[3 * t + 2] * primitives::multilinear::mle_eval(&cols[t][1], &zeta[..tau])) .collect(); - let settle = |sig: &[F128], mut cols: Vec>| -> Result, Error> { + let settle = |sig: &[F192], mut cols: Vec>| -> Result, Error> { let airs = airs_for(&taus, true); - let target = sig.iter().fold(F128::ZERO, |a, &b| a + b); + let target = sig.iter().fold(F192::ZERO, |a, &b| a + b); let mut ps = ProverState::new(b"zc-test", &SEED); let pclaims = prove(&airs, &mut cols, eta, &zeta, sig, &mut ps); let proof = ps.into_proof(); @@ -401,7 +447,7 @@ mod tests { settle(&sigmas, cols.clone()).expect("honest attached claims verify"); for bad in 0..taus.len() { let mut wrong = sigmas.clone(); - wrong[bad] += F128::ONE; + wrong[bad] += F192::ONE; assert!( settle(&wrong, cols.clone()).is_err(), "a wrong claimed sum for table {bad} must be rejected" @@ -414,17 +460,17 @@ mod tests { #[test] fn tampered_transcript_is_rejected() { let taus = [4usize, 2, 4]; - let cols: Vec> = taus.iter().enumerate().map(|(i, &t)| good_table(t, i as u64)).collect(); + let cols = taus.iter().enumerate().map(|(i, &t)| good_table(t, i as u64)).collect(); let (proof, ok) = run(&taus, cols); assert!(ok.is_ok()); let airs = airs_for(&taus, false); let (eta, zeta) = eta_zeta(&taus); for i in 0..proof.stream.len() { let mut bad = proof.clone(); - bad.stream[i] += F128::ONE; + bad.stream[i] += F192::ONE; let mut vs = VerifierState::new(b"zc-test", &bad, &SEED); assert!( - verify(&airs, eta, &zeta, F128::ZERO, &mut vs).is_err(), + verify(&airs, eta, &zeta, F192::ZERO, &mut vs).is_err(), "tampered word {i} must be rejected" ); } diff --git a/crates/lean_vm/src/cpu/execute.rs b/crates/lean_vm/src/cpu/execute.rs index a3bcacac..e76ce5a1 100644 --- a/crates/lean_vm/src/cpu/execute.rs +++ b/crates/lean_vm/src/cpu/execute.rs @@ -1,40 +1,49 @@ //! The write-once execution interpreter: run the compiled program to produce -//! the final memory image and the per-opcode [`Trace`] (§7). +//! the final memory image and the per-opcode [`Trace`]. use std::collections::HashMap; use super::*; -use primitives::{field::mul_by_x, pretty_f64, pretty_integer}; +use primitives::{ + field::{F64, F192, mul_by_g}, + pretty_f64, pretty_integer, +}; pub struct Execution { - pub mem: Vec, // data memory after the run, write-once (size cells, power of two) + pub mem: Vec, // data memory after the run, write-once (size cells, power of two) pub cycles: usize, // number of instructions the run executed (trace length) pub mem_used: usize, // cells actually touched, before the power-of-two pad of `mem` pub(crate) trace: Trace, // rows + final access-count columns, emitted in the same walk } +/// A memory word interpreted as a K-valued address: valid only when both +/// extension limbs are zero (every g-power is a K-element). +fn as_addr(v: F192) -> Option { + (v.c1 == 0 && v.c2 == 0).then_some(F64(v.c0)) +} + impl Program { /// Run the program in write-once *fill* mode to produce its [`Execution`]: /// the final memory image and the step count. The public input seeds the /// first two memory cells `m[0], m[1]` (§e2e-pi). Compilation yields the /// `Program`; executing it (here) and proving it are separate later phases. - pub fn execute(&self, public_input: [F128; 2]) -> Execution { + pub fn execute(&self, public_input: [F192; 2]) -> Execution { use super::hints::{RHint, grow_gpow}; let ending_pc = (self.prog.len() - 1) as u32; // last bytecode slot, g^{B-1} // g^j and its reverse index g^j ↦ j, grown lazily (deep recursion is // unbounded). Seed enough for the program counters / return targets. - let mut gpow: Vec = vec![F128::ONE]; + let mut gpow: Vec = vec![F64::ONE]; let mut gmap = super::hints::GPowMap::default(); - gmap.insert(F128::ONE, 0u32); + gmap.insert(F64::ONE, 0u32); grow_gpow(&mut gpow, &mut gmap, self.prog.len() + 2); // Dense write-once data memory (read path stays a vector for speed), the // per-cell access count (g^{count}, default g^0 = 1), and a written mask. - let mut mem: Vec = vec![F128::ZERO; self.main_frame.max(2) as usize]; + let mut mem: Vec = vec![F192::ZERO; self.main_frame.max(2) as usize]; let mut written: Vec = vec![false; mem.len()]; - let mut mem_count: Vec = vec![F128::ONE; mem.len()]; + let mut mem_count: Vec = vec![F64::ONE; mem.len()]; // Seed the public input into m[0], m[1] (addresses g^0, g^1, §e2e-pi). mem[0] = public_input[0]; mem[1] = public_input[1]; @@ -42,7 +51,7 @@ impl Program { written[1] = true; // Per-pc bytecode execution count (g^{count}). - let mut bytecode_count: Vec = vec![F128::ONE; self.prog.len()]; + let mut bytecode_count: Vec = vec![F64::ONE; self.prog.len()]; let mut next_free = self.main_frame; let (mut pc, mut fp) = (self.pc0, self.fp0); @@ -51,6 +60,9 @@ impl Program { /// Debug: the pc of the currently executing instruction, so the /// write-once panic can report where the conflict happened. static DBG_PC: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// Name of the currently executing computed-advice hint, if the + /// conflicting write came from a hint rather than an instruction. + static DBG_HINT: std::cell::Cell> = const { std::cell::Cell::new(None) }; } // `DBG_PROF=1`: per-pc step counts, printed as a per-function cycle // profile after the run (needs `fn_ranges`, i.e. a compiled program). @@ -60,7 +72,7 @@ impl Program { // sequentially). let mut wit_pos: HashMap = HashMap::new(); // Baby-step table for `hint_decompose_bits_exponent`, built on first use. - let mut dlog_cache: Option<(super::hints::GPowMap, F128)> = None; + let mut dlog_cache: Option<(super::hints::GPowMap, F64)> = None; // Per-opcode trace rows, accumulated during the walk and assembled into the // `Trace` once the run finishes (alongside the final count columns). @@ -70,6 +82,7 @@ impl Program { let mut deref: Vec = Vec::new(); let mut jump: Vec = Vec::new(); let mut blake3: Vec = Vec::new(); + let mut pack64x2: Vec = Vec::new(); // `DEREF Cell` touches whose two sides are both still unwritten (the // range-check gadget's unconstrained target cells): `(deref row index, @@ -78,39 +91,45 @@ impl Program { // end-of-execution deref-hint resolution). let mut deferred: Vec<(usize, usize, u32)> = Vec::new(); + // Attribution aid: LEANVM_PC_HISTO=1 dumps per-pc execution counts + // alongside the disassembly after the run, tying cycles to source. + // Grow the dense vectors so `idx` is in range (keeps mem/written/mem_count in // sync). All accessed cells satisfy cell < next_free after their frame's // allocation, so this only ever extends. - fn ensure(mem: &mut Vec, written: &mut Vec, mem_count: &mut Vec, idx: usize) { + fn ensure(mem: &mut Vec, written: &mut Vec, mem_count: &mut Vec, idx: usize) { if idx >= mem.len() { let n = idx + 1; - mem.resize(n, F128::ZERO); + mem.resize(n, F192::ZERO); written.resize(n, false); - mem_count.resize(n, F128::ONE); + mem_count.resize(n, F64::ONE); } } // Read a cell; an unwritten cell reads as ZERO. - fn get(mem: &[F128], written: &[bool], cell: u32) -> F128 { + fn get(mem: &[F192], written: &[bool], cell: u32) -> F192 { let c = cell as usize; if c < written.len() && written[c] { mem[c] } else { - F128::ZERO + F192::ZERO } } // Write-once store: writing a different value to an already-set cell panics. - fn put(mem: &mut Vec, written: &mut Vec, mem_count: &mut Vec, cell: u32, v: F128) { + fn put(mem: &mut Vec, written: &mut Vec, mem_count: &mut Vec, cell: u32, v: F192) { ensure(mem, written, mem_count, cell as usize); let c = cell as usize; if written[c] { assert!( mem[c] == v, - "write-once conflict at cell {cell} (pc {}): had {:x}:{:x}, new {:x}:{:x}", + "write-once conflict at cell {cell} (pc {}, hint {:?}): had {:x}:{:x}:{:x}, new {:x}:{:x}:{:x}", DBG_PC.with(|p| p.get()), - mem[c].hi, - mem[c].lo, - v.hi, - v.lo + DBG_HINT.with(|h| h.get()), + mem[c].c2, + mem[c].c1, + mem[c].c0, + v.c2, + v.c1, + v.c0 ); } else { mem[c] = v; @@ -121,19 +140,23 @@ impl Program { // with g^n = x, by baby-step giant-step (baby table g^j for j < 2^17, // built once per run; giant step ×g^(-2^17)). Prover-side only — the // guest re-verifies the hinted bits in-circuit. - fn bounded_dlog(cache: &mut Option<(super::hints::GPowMap, F128)>, x: F128, nbits: u32) -> u128 { + fn bounded_dlog(cache: &mut Option<(super::hints::GPowMap, F64)>, x: F64, nbits: u32) -> u128 { const LOG_BABY: u32 = 17; let (baby, giant) = cache.get_or_insert_with(|| { let mut table = super::hints::GPowMap::default(); - let mut p = F128::ONE; + let mut p = F64::ONE; for j in 0..(1u32 << LOG_BABY) { table.insert(p, j); - p = mul_by_x(p); + p = mul_by_g(p); } (table, p.inv()) // p = g^(2^17); its inverse is the giant step }); let mut y = x; - let max_giant = if nbits > LOG_BABY { 1u64 << (nbits - LOG_BABY) } else { 1 }; + let max_giant = if nbits > LOG_BABY { + 1u64 << (nbits - LOG_BABY) + } else { + 1 + }; for a in 0..max_giant { if let Some(&j) = baby.get(&y) { return (a as u128) << LOG_BABY | j as u128; @@ -144,13 +167,13 @@ impl Program { } // Read the running access count and advance it by ×g (the free increment). - // ×g is ×x, i.e. `mul_by_x` — a shift+fold, not a PMULL; this runs on every + // ×g is ×x, i.e. `mul_by_g` — a shift+fold, not a PMULL; this runs on every // memory access (several million per run), so the cheap form matters. - fn bump_access_count(mem: &mut Vec, written: &mut Vec, mem_count: &mut Vec, cell: u32) -> F128 { + fn bump_access_count(mem: &mut Vec, written: &mut Vec, mem_count: &mut Vec, cell: u32) -> F64 { ensure(mem, written, mem_count, cell as usize); let cell_idx = cell as usize; let count = mem_count[cell_idx]; - mem_count[cell_idx] = mul_by_x(count); + mem_count[cell_idx] = mul_by_g(count); count } @@ -189,6 +212,19 @@ impl Program { entry.clone() }; for h in hs { + DBG_HINT.with(|slot| { + slot.set(Some(match h { + RHint::Alloc { .. } => "Alloc", + RHint::AllocDyn { .. } => "AllocDyn", + RHint::WitnessStack { .. } => "WitnessStack", + RHint::WitnessHeap { .. } => "WitnessHeap", + RHint::Log2Ceil { .. } => "Log2Ceil", + RHint::BitDecompose { .. } => "BitDecompose", + RHint::BitDecomposeExp { .. } => "BitDecomposeExp", + RHint::FieldLimbs { .. } => "FieldLimbs", + RHint::Print { .. } => "Print", + })) + }); match h { // A fresh region: write its base `g^{next_free}` into the // pointer cell (once) and reserve `size` cells. `AllocDyn` @@ -200,7 +236,8 @@ impl Program { // the cell holds g^k, allocate k cells (reverse // g-power lookup, growing the index if needed). RHint::AllocDyn { ptr, size } => { - let sz = get(&mem, &written, fp + size); + let sz = as_addr(get(&mem, &written, fp + size)) + .expect("HeapBuf size is not a K-valued g-power"); let cells = gmap.get(&sz).copied().unwrap_or_else(|| { grow_gpow(&mut gpow, &mut gmap, 1 << 20); *gmap @@ -218,7 +255,7 @@ impl Program { next_free += size; grow_gpow(&mut gpow, &mut gmap, (base + size) as usize); ensure(&mut mem, &mut written, &mut mem_count, next_free as usize); - mem[cell as usize] = gpow[base as usize]; + mem[cell as usize] = F192::from(gpow[base as usize]); written[cell as usize] = true; } } @@ -228,20 +265,25 @@ impl Program { if written[c as usize] { let v = mem[c as usize]; // Small integers and small g-powers overlap (8 = x^3 - // = g^3): show every reading that applies. - match (gmap.get(&v), v.hi == 0 && v.lo < 1 << 32) { + // = g^3): show every reading that applies. Only a + // K-valued word (extension limbs 0) can be a g-power. + let k = as_addr(v).and_then(|lo| gmap.get(&lo).copied()); + let small = v.c2 == 0 && v.c1 == 0 && v.c0 < 1 << 32; + match (k, small) { (Some(k), true) => eprintln!( "[print] {label} = {} (g^{})", - pretty_integer(v.lo), + pretty_integer(v.c0), pretty_integer(k) ), (Some(k), false) => { eprintln!("[print] {label} = g^{}", pretty_integer(k)) } (None, true) => { - eprintln!("[print] {label} = {}", pretty_integer(v.lo)) + eprintln!("[print] {label} = {}", pretty_integer(v.c0)) + } + (None, false) => { + eprintln!("[print] {label} = {:#x}:{:#x}:{:#x}", v.c2, v.c1, v.c0) } - (None, false) => eprintln!("[print] {label} = {:#x}:{:#x}", v.hi, v.lo), } } else { eprintln!("[print] {label} = "); @@ -254,7 +296,8 @@ impl Program { } } RHint::WitnessHeap { name, ptr, lo, len } => { - let p = get(&mem, &written, fp + ptr); + let p = as_addr(get(&mem, &written, fp + ptr)) + .expect("hint_witness heap pointer is not a K-valued g-power"); let b = *gmap .get(&p) .unwrap_or_else(|| panic!("hint_witness heap pointer is not a g-power")); @@ -263,8 +306,14 @@ impl Program { put(&mut mem, &mut written, &mut mem_count, b + lo + k as u32, v); } } - RHint::Log2Ceil { bits_ptr, dst, nbits, floor } => { - let p = get(&mem, &written, fp + bits_ptr); + RHint::Log2Ceil { + bits_ptr, + dst, + nbits, + floor, + } => { + let p = as_addr(get(&mem, &written, fp + bits_ptr)) + .expect("log2_ceil bits pointer is not a K-valued g-power"); let b = *gmap .get(&p) .unwrap_or_else(|| panic!("log2_ceil bits pointer is not a g-power")); @@ -274,33 +323,64 @@ impl Program { word |= 1u128 << j; } } - let cl = if word <= 1 { 0 } else { u128::BITS - (word - 1).leading_zeros() }; + let cl = if word <= 1 { + 0 + } else { + u128::BITS - (word - 1).leading_zeros() + }; let mu = cl.max(*floor); - put(&mut mem, &mut written, &mut mem_count, fp + dst, primitives::field::g_pow(mu as usize)); + put( + &mut mem, + &mut written, + &mut mem_count, + fp + dst, + F192::from(primitives::field::g_pow(mu as usize)), + ); } RHint::BitDecompose { value, bits_ptr, nbits } => { + assert!(*nbits <= 192, "a machine word has 192 bits"); let v = get(&mem, &written, fp + value); - let word = (v.hi as u128) << 64 | v.lo as u128; + let limbs = [v.c0, v.c1, v.c2]; + let bp = as_addr(get(&mem, &written, fp + bits_ptr)) + .expect("decompose bits pointer is not a K-valued g-power"); let bb = *gmap - .get(&get(&mem, &written, fp + bits_ptr)) + .get(&bp) .unwrap_or_else(|| panic!("decompose bits pointer is not a g-power")); for j in 0..*nbits { - let bit = ((word >> j) & 1) as u64; - put(&mut mem, &mut written, &mut mem_count, bb + j, F128::new(bit, 0)); + let bit = (limbs[j as usize / 64] >> (j % 64)) & 1; + put(&mut mem, &mut written, &mut mem_count, bb + j, F192::new(bit, 0, 0)); } } RHint::BitDecomposeExp { value, bits_ptr, nbits } => { - let x = get(&mem, &written, fp + value); + let x = as_addr(get(&mem, &written, fp + value)) + .expect("hint_decompose_bits_exponent value is not a K-valued g-power"); let n = bounded_dlog(&mut dlog_cache, x, *nbits); - let bb = *gmap - .get(&get(&mem, &written, fp + bits_ptr)) - .unwrap_or_else(|| panic!("hint_decompose_bits_exponent bits pointer is not a g-power")); + let bp = as_addr(get(&mem, &written, fp + bits_ptr)) + .expect("hint_decompose_bits_exponent bits pointer is not a K-valued g-power"); + let bb = *gmap.get(&bp).unwrap_or_else(|| { + panic!("hint_decompose_bits_exponent bits pointer is not a g-power") + }); for j in 0..*nbits { let bit = ((n >> j) & 1) as u64; - put(&mut mem, &mut written, &mut mem_count, bb + j, F128::new(bit, 0)); + put(&mut mem, &mut written, &mut mem_count, bb + j, F192::new(bit, 0, 0)); + } + } + RHint::FieldLimbs { value, base, len } => { + assert!((1..=3).contains(len), "an F192 value has three K limbs"); + let v = get(&mem, &written, fp + value); + let limbs = [v.c0, v.c1, v.c2]; + for j in 0..*len { + put( + &mut mem, + &mut written, + &mut mem_count, + fp + base + j, + F192::new(limbs[j as usize], 0, 0), + ); } } } + DBG_HINT.with(|slot| slot.set(None)); } } // Cover the g-powers this step may index (g²·pc return target, g^fp). @@ -308,7 +388,7 @@ impl Program { let bytecode_read = { let v = bytecode_count[pc as usize]; - bytecode_count[pc as usize] = mul_by_x(v); + bytecode_count[pc as usize] = mul_by_g(v); v }; @@ -386,7 +466,13 @@ impl Program { } => { let a1 = fp + alpha; let p = get(&mem, &written, a1); - let base = match gmap.get(&p) { + let p_addr = as_addr(p).unwrap_or_else(|| { + panic!( + "DEREF pointer is not a K-valued g-power at pc {pc}: {:x}:{:x}", + p.c1, p.c0 + ) + }); + let base = match gmap.get(&p_addr) { Some(&b) => b, None => { // Not indexed yet: grow the g-power index to the minimum @@ -396,12 +482,12 @@ impl Program { // pointer: a wild deref, or a failed range check // (`assert log _ < _`) surfacing honestly. grow_gpow(&mut gpow, &mut gmap, 1 << MIN_LOG_MEM); - *gmap.get(&p).unwrap_or_else(|| { + *gmap.get(&p_addr).unwrap_or_else(|| { panic!( "DEREF pointer is not a small g-power at pc {pc}: a wild \ pointer, or a failed range check \ - (value 0x{:016x}{:016x})", - p.hi, p.lo + (value 0x{:016x})", + p_addr.0 ) }) } @@ -438,11 +524,11 @@ impl Program { } } DerefMode::Pc => { - let v = gpow[pc as usize + 2]; + let v = F192::from(gpow[pc as usize + 2]); put(&mut mem, &mut written, &mut mem_count, a2 as u32, v); } DerefMode::Fp => { - let v = gpow[fp as usize]; + let v = F192::from(gpow[fp as usize]); put(&mut mem, &mut written, &mut mem_count, a2 as u32, v); } } @@ -479,19 +565,21 @@ impl Program { // `b = [c ≠ 0]` is needed now; `w = c⁻¹` is only recorded into // the trace (never used for control flow), so it is deferred to // ONE batched Montgomery inversion after the run — computing it - // per-jump here runs a 254-mul Fermat inverse on every taken - // branch (~2^17 of them), which dominated `execute`. Placeholder - // 0 now; batch-filled below (bit-identical to `c.inv()`). - let b = if c.is_zero() { F128::ZERO } else { F128::ONE }; - let w = F128::ZERO; + // per-jump here runs a Fermat inverse on every taken branch + // (~2^17 of them), which dominated `execute`. Placeholder 0 now; + // batch-filled below (bit-identical to `c.inv()`). + let b = if c.is_zero() { F64::ZERO } else { F64::ONE }; + let w = F192::ZERO; let rc = bump_access_count(&mut mem, &mut written, &mut mem_count, ac); let rd = bump_access_count(&mut mem, &mut written, &mut mem_count, ad); let rf = bump_access_count(&mut mem, &mut written, &mut mem_count, af); let taken = !c.is_zero(); let (npc, nfp) = if taken { - (d, f) + let dpc = as_addr(d).expect("JUMP target is not a K-valued g-power"); + let ffp = as_addr(f).expect("JUMP fp is not a K-valued g-power"); + (dpc, ffp) } else { - (mul_by_x(gpow[pc as usize]), gpow[fp as usize]) + (mul_by_g(gpow[pc as usize]), gpow[fp as usize]) }; jump.push(Jrow { pc, @@ -515,40 +603,75 @@ impl Program { bytecode_read, }); if taken { - pc = *gmap.get(&d).expect("JUMP target not a g-power"); - fp = *gmap.get(&f).expect("JUMP fp not a g-power"); + pc = *gmap.get(&npc).expect("JUMP target not a g-power"); + fp = *gmap.get(&nfp).expect("JUMP fp not a g-power"); } else { pc += 1; } } + Op::Pack64x2 { a, b, c } => { + let (aa, ab, ac) = (fp + a, fp + b, fp + c); + let va = get(&mem, &written, aa); + let vb = get(&mem, &written, ab); + assert_eq!((va.c1, va.c2), (0, 0), "PACK64X2 first input must be K-valued"); + assert_eq!((vb.c1, vb.c2), (0, 0), "PACK64X2 second input must be K-valued"); + put(&mut mem, &mut written, &mut mem_count, ac, F192::new(va.c0, vb.c0, 0)); + let ra = bump_access_count(&mut mem, &mut written, &mut mem_count, aa); + let rb = bump_access_count(&mut mem, &mut written, &mut mem_count, ab); + let rc = bump_access_count(&mut mem, &mut written, &mut mem_count, ac); + pack64x2.push(Xrow { + pc, + fp, + aa, + ab, + ac, + ra, + rb, + rc, + bytecode_read, + }); + pc += 1; + } Op::Blake3 { ins, cv, out, metadata } => { - // Four independently-addressed message words; the chaining - // value and output each span two consecutive words. + // Four independently-addressed 128-bit message chunks, each a + // single cell; the chaining value and the output each span two + // consecutive cells. let (aa0, aa1, ab0, ab1) = (fp + ins[0], fp + ins[1], fp + ins[2], fp + ins[3]); let acv = fp + cv; let ac = fp + out; - let va0 = get(&mem, &written, aa0); - let va1 = get(&mem, &written, aa1); - let vb0 = get(&mem, &written, ab0); - let vb1 = get(&mem, &written, ab1); - let vcv0 = get(&mem, &written, acv); - let vcv1 = get(&mem, &written, acv + 1); - // Compress the 64 message bytes to the 32-byte result, then write - // it to c's two words. No table constraint covers the digest (the - // relation is proven by flock, §blake3_flock); the interpreter - // still computes the definite digest so the output cells are - // consistent for any later read. - let (vc0, vc1) = blake3_compress(va0, va1, vb0, vb1, vcv0, vcv1, metadata); - put(&mut mem, &mut written, &mut mem_count, ac, vc0); - put(&mut mem, &mut written, &mut mem_count, ac + 1, vc1); - let ra0 = bump_access_count(&mut mem, &mut written, &mut mem_count, aa0); - let ra1 = bump_access_count(&mut mem, &mut written, &mut mem_count, aa1); - let rb0 = bump_access_count(&mut mem, &mut written, &mut mem_count, ab0); - let rb1 = bump_access_count(&mut mem, &mut written, &mut mem_count, ab1); - let rcv0 = bump_access_count(&mut mem, &mut written, &mut mem_count, acv); - let rcv1 = bump_access_count(&mut mem, &mut written, &mut mem_count, acv + 1); - let rc0 = bump_access_count(&mut mem, &mut written, &mut mem_count, ac); - let rc1 = bump_access_count(&mut mem, &mut written, &mut mem_count, ac + 1); + let words = [aa0, aa1, ab0, ab1, acv, acv + 1].map(|a| get(&mem, &written, a)); + assert!( + words.iter().all(|w| w.c2 == 0), + "BLAKE3 input cell must be a canonical 128-bit embedding" + ); + let va = [F64(words[0].c0), F64(words[0].c1), F64(words[1].c0), F64(words[1].c1)]; + let vb = [F64(words[2].c0), F64(words[2].c1), F64(words[3].c0), F64(words[3].c1)]; + let vcv = [F64(words[4].c0), F64(words[4].c1), F64(words[5].c0), F64(words[5].c1)]; + // Compress the 64 message bytes to the 32-byte result, then + // write it to c's two cells. No table constraint covers the + // digest (the relation is proven by flock, §blake3_flock); the + // interpreter still computes the definite digest so the output + // cells are consistent for any later read. + let vc = blake3_compress(va, vb, vcv, metadata); + let outputs = [F192::new(vc[0].0, vc[1].0, 0), F192::new(vc[2].0, vc[3].0, 0)]; + put(&mut mem, &mut written, &mut mem_count, ac, outputs[0]); + put(&mut mem, &mut written, &mut mem_count, ac + 1, outputs[1]); + let ra = [ + bump_access_count(&mut mem, &mut written, &mut mem_count, aa0), + bump_access_count(&mut mem, &mut written, &mut mem_count, aa1), + ]; + let rb = [ + bump_access_count(&mut mem, &mut written, &mut mem_count, ab0), + bump_access_count(&mut mem, &mut written, &mut mem_count, ab1), + ]; + let rcv = [ + bump_access_count(&mut mem, &mut written, &mut mem_count, acv), + bump_access_count(&mut mem, &mut written, &mut mem_count, acv + 1), + ]; + let rc = [ + bump_access_count(&mut mem, &mut written, &mut mem_count, ac), + bump_access_count(&mut mem, &mut written, &mut mem_count, ac + 1), + ]; blake3.push(Brow { pc, fp, @@ -558,23 +681,15 @@ impl Program { ab1, acv, ac, - va0, - va1, - vb0, - vb1, - vcv0, - vcv1, + va, + vb, + vcv, + vc, metadata, - vc0, - vc1, - ra0, - ra1, - rb0, - rb1, - rcv0, - rcv1, - rc0, - rc1, + ra, + rb, + rcv, + rc, bytecode_read, }); pc += 1; @@ -595,13 +710,32 @@ impl Program { }) .collect(); rows.sort_by_key(|(_, c)| std::cmp::Reverse(*c)); - eprintln!( - "== DBG_PROF: cycles by function ({} total) ==", - pretty_integer(steps) - ); + // `DBG_PROF_DUMP=path`: also write the raw per-pc counts plus the + // function table, so an offline pass can attribute the straight-line + // cycles of one big function to its source regions (the call sites of + // the lowered `for` helpers are the landmarks). + if let Ok(path) = std::env::var("DBG_PROF_DUMP") { + let mut out = format!("# steps {steps}\n"); + for (name, entry, len) in &self.fn_ranges { + out += &format!("F {name} {entry} {len}\n"); + } + for (pc, c) in p.iter().enumerate() { + if *c > 0 { + out += &format!("{pc} {c}\n"); + } + } + let path = if std::path::Path::new(&path).exists() { + format!("{path}.{}", std::process::id()) + } else { + path + }; + std::fs::write(&path, out).expect("write DBG_PROF_DUMP"); + eprintln!("== DBG_PROF: per-pc counts written to {path}"); + } + eprintln!("== DBG_PROF: cycles by function ({} total) ==", pretty_integer(steps)); for (name, c) in rows.iter().filter(|(_, c)| *c > 0) { eprintln!( - " {:>12} {:>7}% {name}", + " {:>13} {:>7}% {name}", pretty_integer(c), pretty_f64(100.0 * *c as f64 / steps as f64) ); @@ -631,20 +765,20 @@ impl Program { for (_, a2, a3) in deferred { // Never written: the cells are genuinely unconstrained; fix them (and // the rows, already ZERO) to ZERO. - put(&mut mem, &mut written, &mut mem_count, a2 as u32, F128::ZERO); - put(&mut mem, &mut written, &mut mem_count, a3, F128::ZERO); + put(&mut mem, &mut written, &mut mem_count, a2 as u32, F192::ZERO); + put(&mut mem, &mut written, &mut mem_count, a3, F192::ZERO); } // Fill the deferred JUMP inverse hints `w = c⁻¹` (the is-nonzero witness) // in ONE batched Montgomery inversion: a single field inverse plus ~2·#jumps - // multiplies, instead of a 254-mul Fermat inverse per taken branch. `w` is - // only recorded into the trace, so this reproduces exactly the per-jump - // `c.inv()` (0 for the c = 0 rows). `prefix[i]` is the running product of the - // nonzero conditions before row `i`; `acc` ends as the product of all - // nonzero conditions (nonzero, so invertible). + // multiplies, instead of a full inverse per taken branch. `w` is only + // recorded into the trace, so this reproduces exactly the per-jump `c.inv()` + // (0 for the c = 0 rows). `prefix[i]` is the running product of the nonzero + // conditions before row `i`; `acc` ends as the product of all nonzero + // conditions (nonzero, so invertible). { - let mut acc = F128::ONE; - let mut prefix: Vec = Vec::with_capacity(jump.len()); + let mut acc = F192::ONE; + let mut prefix: Vec = Vec::with_capacity(jump.len()); for r in &jump { prefix.push(acc); if !r.c.is_zero() { @@ -665,8 +799,8 @@ impl Program { let mem_used = mem.len(); let cells = mem.len().next_power_of_two().max(1 << MIN_LOG_MEM); assert!(cells <= 1 << MAX_LOG_MEM, "data memory exceeds 2^{MAX_LOG_MEM} cells"); - mem.resize(cells, F128::ZERO); - mem_count.resize(cells, F128::ONE); + mem.resize(cells, F192::ZERO); + mem_count.resize(cells, F64::ONE); let trace = Trace { xor, mul, @@ -674,6 +808,7 @@ impl Program { deref, jump, blake3, + pack64x2, mem_count, bytecode_count, }; diff --git a/crates/lean_vm/src/cpu/hints.rs b/crates/lean_vm/src/cpu/hints.rs index 9b029b6f..0f7362e9 100644 --- a/crates/lean_vm/src/cpu/hints.rs +++ b/crates/lean_vm/src/cpu/hints.rs @@ -2,15 +2,15 @@ //! resolved hint ops a [`super::Program`] carries ([`RHint`]), and the g-power //! table + reverse index the hint interpreter grows on demand. -use primitives::field::F128; +use primitives::field::F64; use std::collections::HashMap; /// Frame-relative offset operand (matches the compiler's `ir::Off`). pub type Off = u32; /// A fast [`std::hash::Hasher`] for the g-power reverse index (`g^k ↦ k`). The -/// keys are field elements that are effectively uniform, so one multiplicative -/// mix of the two 64-bit limbs distributes well — far cheaper than the default +/// keys are 64-bit field elements that are effectively uniform, so one +/// multiplicative mix distributes well — far cheaper than the default /// SipHash across the interpreter's millions of reverse-index lookups/inserts /// (e.g. growing the index to `2^20` on a dynamic allocation). #[derive(Default)] @@ -23,7 +23,7 @@ impl std::hash::Hasher for GPowHasher { } #[inline] fn write(&mut self, bytes: &[u8]) { - // Fallback for non-u64 writes (F128's derived Hash uses `write_u64`, so + // Fallback for non-u64 writes (F64's derived Hash uses `write_u64`, so // this is not on the hot path). for &b in bytes { self.0 = (self.0 ^ b as u64).wrapping_mul(0x0100_0000_01b3); @@ -31,13 +31,13 @@ impl std::hash::Hasher for GPowHasher { } #[inline] fn write_u64(&mut self, i: u64) { - // F128 hashes its `lo` then `hi` limb through here; fold both. + // F64 hashes its single limb through here. self.0 = (self.0 ^ i).wrapping_mul(0x9E37_79B9_7F4A_7C15); } } -/// The g-power reverse index type: `F128 → u32` keyed by [`GPowHasher`]. -pub type GPowMap = HashMap>; +/// The g-power reverse index type: `F64 → u32` keyed by [`GPowHasher`]. +pub type GPowMap = HashMap>; /// A hint resolved to concrete offsets/sizes, keyed by global program counter. #[derive(Clone, Debug)] @@ -53,23 +53,31 @@ pub enum RHint { WitnessHeap { name: String, ptr: Off, lo: u32, len: u32 }, /// Write `g^max(log2_ceil(value), floor)` into `fp+dst`, where `value` is the /// integer reconstructed from the `nbits` bits at the buffer `m[fp+bits_ptr]`. - Log2Ceil { bits_ptr: Off, dst: Off, nbits: u32, floor: u32 }, + Log2Ceil { + bits_ptr: Off, + dst: Off, + nbits: u32, + floor: u32, + }, /// Write the `nbits` bits of `m[fp+value]` into the buffer `m[fp+bits_ptr]`. BitDecompose { value: Off, bits_ptr: Off, nbits: u32 }, /// Write the `nbits` bits of `n`, where `m[fp+value] = g^n` (a bounded /// discrete log at witness generation), into the buffer `m[fp+bits_ptr]`. BitDecomposeExp { value: Off, bits_ptr: Off, nbits: u32 }, + /// Write the first `len` K-coordinate limbs of `m[fp+value]` to + /// `m[fp+base..]`. Computed advice; callers constrain the result. + FieldLimbs { value: Off, base: Off, len: u32 }, /// Prover-side debug print (`print(...)` in the zkDSL): display the value /// of `m[fp+cell]` at this program point. Witness generation only. Print { label: String, cell: Off }, } /// Extend the `g^j` table and its reverse index `g^j ↦ j` to cover index `upto`. -pub fn grow_gpow(gpow: &mut Vec, gmap: &mut GPowMap, upto: usize) { +pub fn grow_gpow(gpow: &mut Vec, gmap: &mut GPowMap, upto: usize) { assert!(upto < (1 << 28), "address space overflow (program too large)"); while gpow.len() <= upto { - // ×g is ×x = `mul_by_x` (shift+fold), not a PMULL. - let next = primitives::field::mul_by_x(*gpow.last().unwrap()); + // ×g is ×x = `mul_by_g` (shift+fold), not a PMULL. + let next = primitives::field::mul_by_g(*gpow.last().unwrap()); gmap.insert(next, gpow.len() as u32); gpow.push(next); } diff --git a/crates/lean_vm/src/cpu/isa.rs b/crates/lean_vm/src/cpu/isa.rs index 533f94e7..0c6a0e7a 100644 --- a/crates/lean_vm/src/cpu/isa.rs +++ b/crates/lean_vm/src/cpu/isa.rs @@ -1,6 +1,6 @@ -//! The ISA: the six opcodes and the `DEREF` store modes. +//! The ISA and the `DEREF` store modes. -use primitives::field::F128; +use primitives::field::{F64, F192}; #[derive(Clone, Copy, Debug)] pub enum Op { @@ -16,7 +16,10 @@ pub enum Op { }, Set { o: u32, - k: F128, + /// The immediate stored into `mem[fp·o]`. A full 192-bit machine word + /// (`E = F192`); K-valued constants (addresses, small ints) ride the + /// low lane with `c1 = c2 = 0`. + k: F192, }, Deref { alpha: u32, @@ -29,24 +32,36 @@ pub enum Op { od: u32, of: u32, }, - /// `BLAKE3` (§7.6): compresses the four 16-byte input words `ins` (the two - /// 256-bit operands `ins[0]‖ins[1]` and `ins[2]‖ins[3]`) and writes the - /// 32-byte digest to the two consecutive words `out`, `out+1`. Each input - /// word is addressed independently (`fp+ins[i]`) — no forced contiguity, so - /// the caller need not assemble its operands into adjacent cells. The - /// compression relation is proven by flock. + /// Read two K-valued (64-bit) cells and pack them canonically into one + /// 128-bit cell: `c = (a.c0, b.c0, 0)`. The memory bus reads the sources as + /// `(lo, 0, 0)`, so executing this instruction also proves both source + /// words lie in K = F64. + Pack64x2 { + a: u32, + b: u32, + c: u32, + }, + /// `BLAKE3`: one standard BLAKE3 compression. The four 16-byte + /// message chunks `ins` (each a canonical 128-bit chunk in ONE 192-bit cell, + /// top limb zero) form the 64-byte block; the digest lands in the TWO + /// consecutive cells `out, out+1`. Each message chunk is addressed + /// independently — no forced contiguity, so the caller need not assemble + /// its operands into adjacent cells. The compression relation is proven by + /// flock. Blake3 { ins: [u32; 4], - /// Base of two consecutive words holding the 256-bit chaining value. + /// Base of two consecutive cells holding the 256-bit chaining value + /// (canonical 128-bit chunks, top limbs zero). cv: u32, out: u32, - /// `counter:u64 | block_len:u32 | flags:u32`, little-endian. - metadata: F128, + /// `counter:u64 | block_len:u32 | flags:u32`, little-endian, in the two + /// low K-lanes of a 192-bit immediate (top lane always zero). + metadata: F192, }, } -/// The source `DEREF` stores at `mem[loc_α·β]` (§1): a local cell, the return -/// address `pc+γ`, or the frame pointer. Encoded as two boolean flags `(f_pc, +/// The source `DEREF` stores at `mem[loc_α·β]`: a local cell, the return +/// address `g²·pc`, or the frame pointer. Encoded as two boolean flags `(f_pc, /// f_fp)` — `Cell=(0,0)`, `Pc=(1,0)`, `Fp=(0,1)` — keeping the store constraint degree 2. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum DerefMode { @@ -56,10 +71,10 @@ pub enum DerefMode { } impl DerefMode { - pub(crate) fn f_pc(self) -> F128 { - if self == DerefMode::Pc { F128::ONE } else { F128::ZERO } + pub(crate) fn f_pc(self) -> F64 { + if self == DerefMode::Pc { F64::ONE } else { F64::ZERO } } - pub(crate) fn f_fp(self) -> F128 { - if self == DerefMode::Fp { F128::ONE } else { F128::ZERO } + pub(crate) fn f_fp(self) -> F64 { + if self == DerefMode::Fp { F64::ONE } else { F64::ZERO } } } diff --git a/crates/lean_vm/src/cpu/layout.rs b/crates/lean_vm/src/cpu/layout.rs index 5019edbf..eb5a162a 100644 --- a/crates/lean_vm/src/cpu/layout.rs +++ b/crates/lean_vm/src/cpu/layout.rs @@ -1,6 +1,6 @@ //! The public column schema and bus layout: the committed-column indices, and //! the flush/count blocks the verifier reconstructs from the program + announced -//! sizes + public input (§7, §8). Plus the prover-side witness build. +//! sizes and public input. Plus the prover-side witness build. use super::*; @@ -9,25 +9,28 @@ use super::*; // Shared committed columns (indices `0..N_SHARED`). The program (opcode + // operands) is PUBLIC, not committed: it rides the bytecode seed/finalize blocks // as `Coord::Public`; only the witness-dependent finalize counts are committed. -pub const MEM: usize = 0; // the data-memory image -pub const MFCNT: usize = 1; // per-cell memory access count, g^{A[i]} -pub const BFCNT: usize = 2; // per-pc bytecode execution count, g^{A[pc]} +// The data-memory image, a 192-bit word per cell committed as three K-lane columns. +pub const MEM_LO: usize = 0; +pub const MEM_HI: usize = 1; +pub const MEM_TOP: usize = 2; +pub const MFCNT: usize = 3; // per-cell memory access count, g^{A[i]} +pub const BFCNT: usize = 4; // per-pc bytecode execution count, g^{A[pc]} // flock's packed BLAKE3 witness `q_pkd`, committed in the SAME stack as every -// other column (single PCS). Size `2^(K_LOG+n_log-7)`, always ≥ 1 instance (a -// no-BLAKE3 program commits one full padding instance). It is the SOLE copy of -// the input/output words: the VM's BLAKE3 value columns are virtual and their -// memory-bus claims route to `q_pkd` slots (§blake3_flock), so nothing duplicates -// them. flock's R1CS validity is discharged by the single stacked Ligerito -// opening over this commitment. -pub const QPKD: usize = 3; -pub const N_SHARED: usize = 4; +// other column (single PCS). Size `2^(K_LOG+n_log-6)` F64 words, always ≥ 1 +// instance (a no-BLAKE3 program commits one full padding instance). It is the +// SOLE copy of the input/output words: the VM's BLAKE3 value columns are +// virtual and their memory-bus claims route to `q_pkd` slots (§blake3_flock), so +// nothing duplicates them. flock's R1CS validity is discharged by the single +// stacked Ligerito opening over this commitment. +pub const QPKD: usize = 5; +pub const N_SHARED: usize = 6; /// Global column indexing: the shared columns occupy `0..N_SHARED`, then each /// table `t` (in [`tables::tables`] order) owns the contiguous block `[base[t], /// base[t] + n_committed_columns_t)`. Both prover and verifier derive this identically /// from the table set, so every column claim lines up. pub struct Schema { - pub base: [usize; 6], + pub base: [usize; tables::N_TABLES], pub n: usize, } @@ -35,7 +38,7 @@ pub struct Schema { pub fn schema() -> &'static Schema { static SCHEMA: std::sync::OnceLock = std::sync::OnceLock::new(); SCHEMA.get_or_init(|| { - let mut base = [0usize; 6]; + let mut base = [0usize; tables::N_TABLES]; let mut next = N_SHARED; for (t, table) in tables::tables().iter().enumerate() { base[t] = next; @@ -51,7 +54,7 @@ fn offset_coords(base: usize, coords: Vec) -> Vec { .into_iter() .map(|c| match c { Coord::Col(i) => Coord::Col(base + i), - Coord::GCol(i) => Coord::GCol(base + i), + Coord::GCol(i, k) => Coord::GCol(base + i, k), other => other, }) .collect() @@ -68,29 +71,29 @@ pub struct Layout { pub count: Vec, /// Per-column padding value (count columns pad with 1, else 0), so the verifier /// can form the default-padding surplus it divides out of the bus (§sec:gp). - pub pad: Vec, + pub pad: Vec, /// Per-column placement (offset + n_vars) in the stacked witness; from the /// columns' log-sizes alone, so reconstructable by the verifier. pub placements: Vec, /// `log2` of the stacked witness length. pub m: usize, - /// Public input: the first two memory cells `m[0], m[1]` (256 bits), bound to - /// the committed memory at verification (§8). - pub pi: [F128; 2], - pub taus: [usize; 6], // (xor, mul, set, deref, jump, blake3) log row counts + /// Public input: the first two memory cells `m[0], m[1]` (each a 192-bit + /// word), bound to the committed memory at verification (§8). + pub pi: [F192; 2], + pub taus: [usize; tables::N_TABLES], /// Real (non-padded) per-table row counts, as announced. `row_counts[5]` is /// the executed `BLAKE3` count, which gates the flock sub-proof. - pub row_counts: [usize; 6], + pub row_counts: [usize; tables::N_TABLES], } /// The prover's witness bundle: the committed column values + their stacked /// multilinear `q` + the public [`Layout`] (plus the sizes needed to announce it). pub(crate) struct Witness { pub(crate) cols: Vec, - pub(crate) q: Vec, + pub(crate) q: Vec, pub(crate) layout: Layout, pub(crate) log_mem: usize, - pub(crate) row_counts: [usize; 6], + pub(crate) row_counts: [usize; tables::N_TABLES], /// `Option` lets `prove` take and free the large reduction-only buffers /// immediately after reduction, before the mixed PCS opening. pub(crate) flock_reduction: Option, @@ -103,19 +106,18 @@ pub(crate) struct Witness { /// adj; used for the fixed-size columns and the program bytecode length, /// which the caller passes as `log_bytecode`), source 1 is log_mem, and /// source 2 + t is tau_t. `None` = virtual (never committed). Mirrors -/// [`col_kappas`] exactly; keep the two in lockstep. +/// `col_kappas` exactly; keep the two in lockstep. pub fn col_kappa_sources(log_bytecode: usize) -> Vec> { let sch = schema(); let mut k = vec![Some((0usize, 0usize)); sch.n]; - k[MEM] = Some((1, 0)); + k[MEM_LO] = Some((1, 0)); + k[MEM_HI] = Some((1, 0)); + k[MEM_TOP] = Some((1, 0)); k[MFCNT] = Some((1, 0)); k[BFCNT] = Some((0, log_bytecode)); // qpkd_kappa(n) = K_LOG + n_blocks_log - LOG_PACKING, and tau_5 IS // n_blocks_log (the announced-size certification uses the same floor). - k[QPKD] = Some(( - 2 + tables::BLAKE3_TABLE, - flock::blake3::K_LOG - ::pcs::LOG_PACKING, - )); + k[QPKD] = Some((2 + tables::BLAKE3_TABLE, flock::blake3::K_LOG - ::pcs::LOG_PACKING)); for (t, table) in tables::tables().iter().enumerate() { let base = sch.base[t]; k[base..base + table.n_committed_columns()].fill(Some((2 + t, 0))); @@ -149,22 +151,51 @@ pub fn block_kappa_sources(log_bytecode: usize) -> Vec<(usize, usize)> { push } -/// Resolve the symbolic column-size sources used by recursion into concrete -/// log-sizes. `None` marks a virtual (uncommitted) column. -fn col_kappas(log_mem: usize, log_bytecode: usize, taus: [usize; 6]) -> Vec> { - let values = [0, log_mem, taus[0], taus[1], taus[2], taus[3], taus[4], taus[5]]; - col_kappa_sources(log_bytecode) - .into_iter() - .map(|source| source.map(|(index, adjustment)| values[index] + adjustment)) - .collect() +/// Column → log-size (`kappa`) map: the shared MEM/MFCNT columns are `2^log_mem`, +/// the bytecode finalize count is `2^log_bytecode`, and every column of table `t` +/// is `2^taus[t]` (its padded log-row-count). `None` marks a **virtual** +/// (uncommitted) column. Depends only on the public sizes, so the verifier can +/// reconstruct the placements. +/// +/// The BLAKE3 value columns (`va0..vc3`) are always virtual: `q_pkd` +/// already holds those words at fixed packed slots, so committing them again is +/// redundant. Their memory-bus claims route directly to `q_pkd` slot evaluations +/// (see [`slot_claims`]), which both binds them to +/// the proven witness AND eliminates the separate value-binding sub-protocol. +fn col_kappas( + log_mem: usize, + log_bytecode: usize, + taus: [usize; tables::N_TABLES], + n_blake3: usize, +) -> Vec> { + let sch = schema(); + let mut k = vec![Some(0usize); sch.n]; + k[MEM_LO] = Some(log_mem); + k[MEM_HI] = Some(log_mem); + k[MEM_TOP] = Some(log_mem); + k[MFCNT] = Some(log_mem); + k[BFCNT] = Some(log_bytecode); + // q_pkd: `2^(K_LOG+n_log-6)` F64 words, always ≥ 1 instance (`qpkd_kappa` + // floors `n_blake3` at 1 — padding instance for a no-BLAKE3 program). + k[QPKD] = Some(crate::blake3_flock::qpkd_kappa(n_blake3)); + for (t, table) in tables::tables().iter().enumerate() { + let base = sch.base[t]; + k[base..base + table.n_committed_columns()].fill(Some(taus[t])); + } + // BLAKE3 value columns are ALWAYS virtual (read from q_pkd, never committed). + let b3 = sch.base[tables::BLAKE3_TABLE]; + for &c in &tables::BLAKE3_VALUE_COLS { + k[b3 + c] = None; + } + k } /// Build the public [`Layout`] from the program, the memory log-size `log_mem`, the -/// six tables' real row counts `row_counts`, and the public input `pi`. The flush +/// instruction tables' real row counts `row_counts`, and the public input `pi`. The flush /// blocks reference columns only by INDEX and the program only through its /// public columns, so this needs no committed witness — both prover and verifier -/// reconstruct exactly the same structure (§7, §8). -pub fn layout(prog: &[Op], log_mem: usize, row_counts: [usize; 6], pi: [F128; 2]) -> Layout { +/// reconstruct exactly the same structure. +pub fn layout(prog: &[Op], log_mem: usize, row_counts: [usize; tables::N_TABLES], pi: [F192; 2]) -> Layout { let bytecode_size = prog.len(); let log_bytecode = crate::log2_strict_usize(bytecode_size); let cells = 1usize << log_mem; @@ -172,7 +203,7 @@ pub fn layout(prog: &[Op], log_mem: usize, row_counts: [usize; 6], pi: [F128; 2] // Per-table padded log-row-counts (the boundary block is fixed). The real // (non-padded) `row_counts[t]` tell each flush how many of its 2^kappa rows // are padding (default rows divided out of the bus, §sec:gp). - let mut taus = [0usize; 6]; + let mut taus = [0usize; tables::N_TABLES]; for (i, &r) in row_counts.iter().enumerate() { taus[i] = crate::log2_ceil_usize(r.max(1)); } @@ -190,7 +221,7 @@ pub fn layout(prog: &[Op], log_mem: usize, row_counts: [usize; 6], pi: [F128; 2] let final_pc = (bytecode_size - 1) as u32; let final_fp = 0u32; - let one = F128::ONE; + let one = F64::ONE; // The public program columns map operand *offsets* (small, ≤ frame size) to // g-powers — not memory addresses — so precompute only up to the largest // operand, an O(1) lookup each, rather than over the whole 2^log_mem memory. @@ -201,6 +232,7 @@ pub fn layout(prog: &[Op], log_mem: usize, row_counts: [usize; 6], pi: [F128; 2] Op::Set { o, .. } => o, Op::Deref { alpha, beta, gamma, .. } => alpha.max(beta).max(gamma), Op::Jump { oc, od, of } => oc.max(od).max(of), + Op::Pack64x2 { a, b, c } => a.max(b).max(c), Op::Blake3 { ins, cv, out, .. } => ins[0].max(ins[1]).max(ins[2]).max(ins[3]).max(cv).max(out), }) .max() @@ -214,14 +246,18 @@ pub fn layout(prog: &[Op], log_mem: usize, row_counts: [usize; 6], pi: [F128; 2] Op::Set { .. } => OP_SET, Op::Deref { .. } => OP_DEREF, Op::Jump { .. } => OP_JUMP, + Op::Pack64x2 { .. } => tables::OP_PACK64X2, Op::Blake3 { .. } => OP_BLAKE3, }; - let operands = |op: &Op| -> (F128, F128, F128) { + let operands = |op: &Op| -> (F64, F64, F64) { match *op { Op::Xor { a, b, c } | Op::Mul { a, b, c } => (g_at(a), g_at(b), g_at(c)), - Op::Set { o, k } => (g_at(o), k, F128::ZERO), + // The immediate's first two K-limbs ride operand slots o2/o3; c2 + // rides the fpc slot below. + Op::Set { o, k } => (g_at(o), F64(k.c0), F64(k.c1)), Op::Deref { alpha, beta, gamma, .. } => (g_at(alpha), g_at(beta), g_at(gamma)), Op::Jump { oc, od, of } => (g_at(oc), g_at(od), g_at(of)), + Op::Pack64x2 { a, b, c } => (g_at(a), g_at(b), g_at(c)), // BLAKE3's first three input-word offsets; the last two ride the // fpc/ffp bytecode slots below. Op::Blake3 { ins, .. } => (g_at(ins[0]), g_at(ins[1]), g_at(ins[2])), @@ -232,31 +268,39 @@ pub fn layout(prog: &[Op], log_mem: usize, row_counts: [usize; 6], pi: [F128; 2] let fpc = |op: &Op| match op { Op::Deref { mode, .. } => mode.f_pc(), Op::Blake3 { ins, .. } => g_at(ins[3]), - _ => F128::ZERO, + Op::Set { k, .. } => F64(k.c2), + _ => F64::ZERO, }; let ffp = |op: &Op| match op { Op::Deref { mode, .. } => mode.f_fp(), Op::Blake3 { cv, .. } => g_at(*cv), - _ => F128::ZERO, + _ => F64::ZERO, }; + // The 6th/7th/8th bytecode operand slots: BLAKE3's output base and the two + // K-lanes of its metadata immediate (0 elsewhere). let extra0 = |op: &Op| match op { Op::Blake3 { out, .. } => g_at(*out), - _ => F128::ZERO, + _ => F64::ZERO, }; let extra1 = |op: &Op| match op { - Op::Blake3 { metadata, .. } => *metadata, - _ => F128::ZERO, + Op::Blake3 { metadata, .. } => F64(metadata.c0), + _ => F64::ZERO, + }; + let extra2 = |op: &Op| match op { + Op::Blake3 { metadata, .. } => F64(metadata.c1), + _ => F64::ZERO, }; - // The program is PUBLIC (not committed): eight public columns over the + // The program is PUBLIC (not committed): nine public columns over the // program cube, embedded in the bytecode seed/finalize blocks below. - let prog_op: Vec = prog.par_iter().map(opcode).collect(); - let prog_o1: Vec = prog.par_iter().map(|o| operands(o).0).collect(); - let prog_o2: Vec = prog.par_iter().map(|o| operands(o).1).collect(); - let prog_o3: Vec = prog.par_iter().map(|o| operands(o).2).collect(); - let prog_fpc: Vec = prog.par_iter().map(fpc).collect(); - let prog_ffp: Vec = prog.par_iter().map(ffp).collect(); - let prog_extra0: Vec = prog.par_iter().map(extra0).collect(); - let prog_extra1: Vec = prog.par_iter().map(extra1).collect(); + let prog_op: Vec = prog.par_iter().map(opcode).collect(); + let prog_o1: Vec = prog.par_iter().map(|o| operands(o).0).collect(); + let prog_o2: Vec = prog.par_iter().map(|o| operands(o).1).collect(); + let prog_o3: Vec = prog.par_iter().map(|o| operands(o).2).collect(); + let prog_fpc: Vec = prog.par_iter().map(fpc).collect(); + let prog_ffp: Vec = prog.par_iter().map(ffp).collect(); + let prog_extra0: Vec = prog.par_iter().map(extra0).collect(); + let prog_extra1: Vec = prog.par_iter().map(extra1).collect(); + let prog_extra2: Vec = prog.par_iter().map(extra2).collect(); // ---- bus blocks ---- use Coord::{Col, Const, Index, Public}; @@ -283,9 +327,32 @@ pub fn layout(prog: &[Op], log_mem: usize, row_counts: [usize; 6], pi: [F128; 2] Const(g_pow(final_fp as usize)), ], )); - // memory seed + finalize (every address real, no padding). - push.push(blk(log_mem, cells, vec![Const(SEP_MEM), Index, Const(one), Col(MEM)])); - pull.push(blk(log_mem, cells, vec![Const(SEP_MEM), Index, Col(MFCNT), Col(MEM)])); + // memory seed + finalize (every address real, no padding). The value is the + // full three-limb 192-bit word. + push.push(blk( + log_mem, + cells, + vec![ + Const(SEP_MEM), + Index, + Const(one), + Col(MEM_LO), + Col(MEM_HI), + Col(MEM_TOP), + ], + )); + pull.push(blk( + log_mem, + cells, + vec![ + Const(SEP_MEM), + Index, + Col(MFCNT), + Col(MEM_LO), + Col(MEM_HI), + Col(MEM_TOP), + ], + )); // bytecode seed + finalize (program columns are public; padding entries // self-cancel at count 1, so the whole 2^log_bytecode is "real"). push.push(blk( @@ -303,6 +370,7 @@ pub fn layout(prog: &[Op], log_mem: usize, row_counts: [usize; 6], pi: [F128; 2] Public(prog_ffp.clone()), Public(prog_extra0.clone()), Public(prog_extra1.clone()), + Public(prog_extra2.clone()), ], )); pull.push(blk( @@ -320,6 +388,7 @@ pub fn layout(prog: &[Op], log_mem: usize, row_counts: [usize; 6], pi: [F128; 2] Public(prog_ffp), Public(prog_extra0), Public(prog_extra1), + Public(prog_extra2), ], )); @@ -328,7 +397,7 @@ pub fn layout(prog: &[Op], log_mem: usize, row_counts: [usize; 6], pi: [F128; 2] // also fix the per-column padding to `1` (so they never zero the bus product). let sch = schema(); let mut count_blocks: Vec = Vec::new(); - let mut pad = vec![F128::ZERO; sch.n]; + let mut pad = vec![F64::ZERO; sch.n]; for (t, table) in tables::tables().iter().enumerate() { let base = sch.base[t]; let (kappa, real) = (taus[t], row_counts[t]); @@ -342,11 +411,11 @@ pub fn layout(prog: &[Op], log_mem: usize, row_counts: [usize; 6], pi: [F128; 2] } for &c in table.count_columns() { count_blocks.push(blk(kappa, real, vec![Col(base + c)])); - pad[base + c] = F128::ONE; + pad[base + c] = F64::ONE; } } // BLAKE3 padding rows must match flock's padding instance (the all-zero-input - // compression): zero inputs but a NONZERO output `out_lo`. So the two output + // compression): zero inputs but a NONZERO output `out_lo`. So the four output // value columns pad with that digest, not 0 — the memory bus flushes these // (virtual) columns, and their padding rows must equal `q_pkd`'s padding slots // so the default-padding surplus divides out and the routed claims agree. @@ -355,14 +424,21 @@ pub fn layout(prog: &[Op], log_mem: usize, row_counts: [usize; 6], pi: [F128; 2] { let b3 = sch.base[tables::BLAKE3_TABLE]; let pc = crate::blake3_flock::padding_digest(); - pad[b3 + tables::BLAKE3_VALUE_COLS[4]] = pc[0]; // c0 - pad[b3 + tables::BLAKE3_VALUE_COLS[5]] = pc[1]; // c1 - pad[b3 + tables::BLAKE3_VALUE_COLS[6]] = crate::blake3_flock::IV[0]; // cv0 - pad[b3 + tables::BLAKE3_VALUE_COLS[7]] = crate::blake3_flock::IV[1]; // cv1 - pad[b3 + tables::BLAKE3_VALUE_COLS[8]] = crate::blake3_flock::metadata(0, 64, crate::blake3_flock::FLAGS); + let md = crate::blake3_flock::metadata(0, 64, crate::blake3_flock::FLAGS); + for k in 0..4 { + pad[b3 + tables::BLAKE3_VALUE_COLS[8 + k]] = pc[k]; // c0..c3 + pad[b3 + tables::BLAKE3_VALUE_COLS[12 + k]] = crate::blake3_flock::IV[k]; // cv0..cv3 + } + pad[b3 + tables::BLAKE3_VALUE_COLS[16]] = F64(md.c0); // metadata counter lane + pad[b3 + tables::BLAKE3_VALUE_COLS[17]] = F64(md.c1); // metadata blen‖flags lane } - let (placements, m) = witness::placements_of(&col_kappas(log_mem, log_bytecode, taus)); + let (placements, m) = witness::placements_of(&col_kappas( + log_mem, + log_bytecode, + taus, + row_counts[tables::BLAKE3_TABLE], + )); Layout { push, pull, @@ -412,8 +488,10 @@ impl Program { let (base, n) = (sch.base[t], table.n_committed_columns()); table.fill(&ctx, &mut cols[base..base + n]); } - // Shared columns. - cols[MEM] = exec.mem.clone(); + // Shared columns. The 192-bit memory image splits into three K-limbs. + cols[MEM_LO] = exec.mem.par_iter().map(|w| F64(w.c0)).collect(); + cols[MEM_HI] = exec.mem.par_iter().map(|w| F64(w.c1)).collect(); + cols[MEM_TOP] = exec.mem.par_iter().map(|w| F64(w.c2)).collect(); cols[MFCNT] = tr.mem_count.clone(); // running counts ended at g^{A[i]} cols[BFCNT] = tr.bytecode_count.clone(); // running counts ended at g^{A[pc]} // flock's packed BLAKE3 witness q_pkd, ALWAYS committed in this same stack: @@ -426,14 +504,7 @@ impl Program { let blocks: Vec<_> = tr .blake3 .iter() - .map(|r| { - crate::blake3_flock::compression( - [r.va0, r.va1], - [r.vb0, r.vb1], - [r.vcv0, r.vcv1], - r.metadata, - ) - }) + .map(|r| crate::blake3_flock::compression(r.va, r.vb, r.vcv, r.metadata)) .collect(); crate::blake3_flock::build_qpkd_prepared(&blocks) }; @@ -448,7 +519,7 @@ impl Program { // The public layout (flush/count blocks, per-column padding, placements, // boundary, taus) is a pure function of the program + announced sizes + // public input, with no committed witness; reconstruct it here so the - // prover and verifier share exactly the same structure (§7, §8). + // prover and verifier share exactly the same structure. let row_counts = [ tr.xor.len(), tr.mul.len(), @@ -456,6 +527,7 @@ impl Program { tr.deref.len(), tr.jump.len(), tr.blake3.len(), + tr.pack64x2.len(), ]; assert!( row_counts.iter().all(|&r| r <= 1 << MAX_LOG_ROWS), @@ -483,7 +555,10 @@ impl Program { let t_stack = std::time::Instant::now(); let q = witness::stack_q(&cols, &l.placements, l.m); if prof { - eprintln!("[build] stack_q : {:>7.2} ms", t_stack.elapsed().as_secs_f64() * 1e3); + eprintln!( + "[build] stack_q : {:>7.2} ms", + t_stack.elapsed().as_secs_f64() * 1e3 + ); } Witness { cols, diff --git a/crates/lean_vm/src/cpu/mod.rs b/crates/lean_vm/src/cpu/mod.rs index 1b591187..ac2f0276 100644 --- a/crates/lean_vm/src/cpu/mod.rs +++ b/crates/lean_vm/src/cpu/mod.rs @@ -1,17 +1,18 @@ -//! Whole-program assembly over GF(2^128) (§7, §8): the six instruction tables +//! Whole-program assembly over GF(2^64) (`misc/doc.tex`): the instruction tables //! sharing the state / memory / bytecode buses, bound to one field-valued //! commitment and verified oracle-free. Addresses, the program counter, and read -//! counts are g-powers, so every increment is a free ×g; arithmetic is the field's -//! own (XOR = degree-1, MUL_NATIVE = degree-2). `BLAKE3` (§7.6) adds the -//! memory/state/bytecode plumbing for a 64→32-byte compression whose relation is -//! discharged by flock (see [`crate::blake3_flock`]). +//! counts are g-powers, so every increment is a free ×g. Machine-word arithmetic +//! is over `E = F192 = K[y]/(y³+y+1)` (XOR degree 1, MUL_NATIVE degree 2), +//! with each word carried by three committed `K = F64` limbs. `BLAKE3` +//! adds the memory/state/bytecode plumbing for a 64→32-byte compression +//! whose relation is discharged by flock (see [`crate::blake3_flock`]). All +//! Challenges and transcript scalars live in the same tower E. use std::collections::HashMap; use rayon::prelude::*; use crate::constraints; -use primitives::field::{F128, g_pow}; use crate::leaf::{self, Block, ColumnClaim, Coord}; use crate::pcs; use crate::tables::{ @@ -19,6 +20,7 @@ use crate::tables::{ }; use crate::transcript::{ProverState, VerifierState}; use crate::witness::{self, Column}; +use primitives::field::{F64, F192, g_pow}; mod execute; pub mod hints; @@ -30,27 +32,13 @@ pub use isa::{DerefMode, Op}; pub use layout::*; pub(crate) use trace::{Brow, Drow, Jrow, Srow, Trace, Xrow}; -/// Witness-gen `BLAKE3` compression (doc §7.6): the four message words are laid -/// out little-endian into 64 bytes, combined with the supplied chaining value -/// and metadata, and split back into `c = (vc0, vc1)`. Flock proves this same -/// compression relation ([`crate::blake3_flock`]). -fn blake3_compress( - va0: F128, - va1: F128, - vb0: F128, - vb1: F128, - cv0: F128, - cv1: F128, - metadata: F128, -) -> (F128, F128) { - let block = crate::blake3_flock::compression( - [va0, va1], - [vb0, vb1], - [cv0, cv1], - metadata, - ); - let [c0, c1] = crate::blake3_flock::digest(&block); - (c0, c1) +/// Witness-gen `BLAKE3` compression: the four message cells' eight +/// words are laid out little-endian into 64 bytes, combined with the supplied +/// chaining value and metadata, and the 32-byte result is split back into the +/// four output words `c`. Flock proves this same compression relation +/// ([`crate::blake3_flock`]). +fn blake3_compress(va: [F64; 4], vb: [F64; 4], vcv: [F64; 4], metadata: F192) -> [F64; 4] { + crate::blake3_flock::digest(&crate::blake3_flock::compression(va, vb, vcv, metadata)) } /// Data-memory size bounds (doc §Memory): memory is `2^h` cells with @@ -63,9 +51,17 @@ pub const MIN_LOG_MEM: usize = 16; const MAX_LOG_MEM: usize = 32; /// Each per-opcode table holds at most `2^MAX_LOG_ROWS` rows (executed -/// instructions of that opcode). +/// instructions of that opcode). Together with `MAX_LOG_MEM` and the bytecode +/// cap these are the instance caps from “Counts must not wrap” in `misc/doc.tex`: at `ord(g) = 2^64−1` +/// the memory-soundness and count-non-wrap counting arguments are theorems only +/// for instances whose total read-flush count stays far below `2^64`, so the +/// verifier rejects any announcement exceeding them before running a reduction. const MAX_LOG_ROWS: usize = 32; +/// Bytecode-length instance cap (see [`MAX_LOG_ROWS`]): programs are at most +/// `2^32` instructions. +const MAX_LOG_BYTECODE: usize = 32; + /// A binding digest of the program bytecode (BLAKE3 of every instruction's /// canonical encoding — opcode, operands, and the DEREF store-mode), as two field /// elements. Seeded into the transcript alongside the public input, so EVERY @@ -80,38 +76,48 @@ const MAX_LOG_ROWS: usize = 32; /// input) — bound up front, so a different program yields a different sponge from /// the very first squeeze. Both sides hold the program, so both compute this /// identically; the announced sizes ride the stream (`announce_public`). -fn program_digest(prog: &[Op]) -> [F128; 2] { +fn program_digest(prog: &[Op]) -> [F64; 4] { // VM-native: encode the program as a field-element slice and hash its exact // little-endian bytes with standard BLAKE3 ([`crate::vmhash::hash_slice`]). - let mut words: Vec = Vec::with_capacity(4 * prog.len() + 1); + let mut words: Vec = Vec::with_capacity(7 * prog.len() + 2); // Domain/version marker; standard BLAKE3 binds the total byte length. - words.push(F128::new(prog.len() as u64, 1)); + words.push(F64(prog.len() as u64)); + words.push(F64(3)); for op in prog { - // Fixed four-word encoding per instruction. The final two words carry - // BLAKE3's remaining offsets; they are zero for other opcodes. + // Fixed seven-word encoding per instruction: two operand-offset words + // packed with the tag, the 192-bit immediate's three lanes, then two + // words for BLAKE3's remaining offsets (zero for other opcodes). let (tag, a, b, c, k, x, y) = match *op { - Op::Xor { a, b, c } => (0u8, a, b, c, F128::ZERO, F128::ZERO, F128::ZERO), - Op::Mul { a, b, c } => (1, a, b, c, F128::ZERO, F128::ZERO, F128::ZERO), - Op::Set { o, k } => (2, o, 0, 0, k, F128::ZERO, F128::ZERO), + Op::Xor { a, b, c } => (0u8, a, b, c, F192::ZERO, 0u64, 0u64), + Op::Mul { a, b, c } => (1, a, b, c, F192::ZERO, 0, 0), + Op::Set { o, k } => (2, o, 0, 0, k, 0, 0), Op::Deref { alpha, beta, gamma, mode, } => { - (3 + mode as u8, alpha, beta, gamma, F128::ZERO, F128::ZERO, F128::ZERO) - } - Op::Jump { oc, od, of } => (6, oc, od, of, F128::ZERO, F128::ZERO, F128::ZERO), - Op::Blake3 { ins, cv, out, metadata } => { - (7, ins[0], ins[1], ins[2], metadata, F128::new(ins[3] as u64, cv as u64), F128::new(out as u64, 0)) + (3 + mode as u8, alpha, beta, gamma, F192::ZERO, 0, 0) // mode ∈ {Cell,Pc,Fp} ⇒ tag 3/4/5 } + Op::Jump { oc, od, of } => (6, oc, od, of, F192::ZERO, 0, 0), + Op::Blake3 { ins, cv, out, metadata } => ( + 7, + ins[0], + ins[1], + ins[2], + metadata, + ins[3] as u64 | ((cv as u64) << 32), + out as u64, + ), + Op::Pack64x2 { a, b, c } => (9, a, b, c, F192::ZERO, 0, 0), }; - let lo = a as u64 | ((b as u64) << 32); - let hi = c as u64 | ((tag as u64) << 32); - words.push(F128::new(lo, hi)); - words.push(k); - words.push(x); - words.push(y); + words.push(F64(a as u64 | ((b as u64) << 32))); + words.push(F64(c as u64 | ((tag as u64) << 32))); + words.push(F64(k.c0)); + words.push(F64(k.c1)); + words.push(F64(k.c2)); + words.push(F64(x)); + words.push(F64(y)); } crate::vmhash::hash_slice(&words) } @@ -123,64 +129,80 @@ fn program_digest(prog: &[Op]) -> [F128; 2] { /// challenges depend on the circuit version and the program before anything /// else; a recursion guest carries the INNER program's seed in its public /// input, pinning both with one word pair. -pub fn fs_seed(program: &Program) -> [F128; 2] { +pub fn fs_seed(program: &Program) -> [F192; 2] { let mut h = blake3::Hasher::new(); h.update(b"leanvm-b-fs-seed-v1"); h.update(&crate::blake3_flock::family_digest()); for w in program.digest { - h.update(&w.lo.to_le_bytes()); - h.update(&w.hi.to_le_bytes()); + h.update(&w.0.to_le_bytes()); } let d = *h.finalize().as_bytes(); let word = |o: usize| u64::from_le_bytes(d[o..o + 8].try_into().unwrap()); - [F128::new(word(0), word(8)), F128::new(word(16), word(24))] + [F192::new(word(0), word(8), 0), F192::new(word(16), word(24), 0)] } /// The transcript seed: the public statement bound before any challenge, the /// public input `pi` prefixed by the [`fs_seed`]. Both sides build it identically. -fn transcript_seed(program: &Program, pi: &[F128; 2]) -> [F128; 4] { +fn transcript_seed(program: &Program, pi: &[F192; 2]) -> [F192; 4] { let seed = fs_seed(program); [seed[0], seed[1], pi[0], pi[1]] } -/// Announce the prover's per-table log-sizes (`log_mem` + the six `row_counts`) by +/// Announce the prover's per-table log-sizes (`log_mem` + all `row_counts`) by /// writing them onto the scalar stream (which binds them into the sponge and lets /// the verifier reconstruct the layout). The public statement (program + input) is /// not announced here — it seeds the transcript at construction (see /// [`transcript_seed`]). The boundary states and per-table log-sizes (`taus`) are /// derived (constants from the program, and `padlen(row_counts)`), so they need no /// separate binding. -fn announce_public(ps: &mut ProverState, log_mem: usize, row_counts: [usize; 6]) { - ps.add_scalar(F128::new(log_mem as u64, 0)); +fn announce_public(ps: &mut ProverState, log_mem: usize, row_counts: [usize; tables::N_TABLES], log_inv_rate: usize) { + ps.add_scalar(F192::new(log_mem as u64, 0, 0)); for r in row_counts { - ps.add_scalar(F128::new(r as u64, 0)); + ps.add_scalar(F192::new(r as u64, 0, 0)); } + ps.add_scalar(F192::new(log_inv_rate as u64, 0, 0)); } -/// Verifier side of [`announce_public`]: read the seven announced sizes from the -/// stream and reconstruct the public [`Layout`] from the program + sizes + public -/// input. (The public input was already bound by seeding the transcript.) -fn read_public(vs: &mut VerifierState, prog: &Program, public_input: &[F128; 2]) -> Result { - let log_mem = vs.next_scalar().map_err(Error::Transcript)?.lo as usize; - let mut row_counts = [0usize; 6]; +/// Verifier side of [`announce_public`]: read the announced sizes and PCS +/// rate from the stream, validate them, and reconstruct the public [`Layout`] +/// from the program + sizes + public input. (The public input was already bound +/// by seeding the transcript.) +fn read_public(vs: &mut VerifierState, prog: &Program, public_input: &[F192; 2]) -> Result<(Layout, usize), Error> { + let read_size = |vs: &mut VerifierState| -> Result { + let word = vs.next_scalar().map_err(Error::Transcript)?; + if word.c1 != 0 || word.c2 != 0 { + return Err(Error::PublicInput); + } + usize::try_from(word.c0).map_err(|_| Error::PublicInput) + }; + + let log_mem = read_size(vs)?; + let mut row_counts = [0usize; tables::N_TABLES]; for r in &mut row_counts { - *r = vs.next_scalar().map_err(Error::Transcript)?.lo as usize; + *r = read_size(vs)?; } - // Sanity-bound the announced sizes (a table's row count is the number of times - // its opcode runs — unbounded by the bytecode size, since a small loop body - // runs many times — so cap it generously, not by `bytecode_size`). The bus balance and - // GKR pin the actual sizes; this only guards against absurd/overflowing values. + let log_inv_rate = read_size(vs)?; + // The public instance caps ensure that, with `ord(g) = 2^64 − 1`, the + // counting arguments (memory soundness, count non-wrap, exponent range checks) + // are theorems only when the announced instance keeps the total read-flush + // count provably below `2^64 − 1`, so reject any announcement exceeding the + // caps BEFORE running any reduction. (A table's row count is the number of + // times its opcode runs — unbounded by the bytecode size, since a small loop + // body runs many times — so it gets its own cap, not `bytecode_size`.) let bytecode_size = prog.prog.len(); if !bytecode_size.is_power_of_two() + || bytecode_size > (1usize << MAX_LOG_BYTECODE) || !(MIN_LOG_MEM..=MAX_LOG_MEM).contains(&log_mem) || row_counts.iter().any(|&r| r >= (1usize << MAX_LOG_ROWS)) + || ::pcs::ligerito::validate_log_inv_rate(log_inv_rate).is_err() { return Err(Error::PublicInput); } let l = layout(&prog.prog, log_mem, row_counts, *public_input); - Ok(l) + Ok((l, log_inv_rate)) } +#[derive(Clone)] pub struct Program { pub prog: Vec, // bytecode (size B, power of two) pub pc0: u32, @@ -190,7 +212,7 @@ pub struct Program { /// program. Trusted to match `prog` — always set by [`Program::assemble`] from /// the bytecode, so a `Program` value cannot carry a digest inconsistent with /// its own `prog`. - pub(crate) digest: [F128; 2], + pub(crate) digest: [F64; 4], /// Prover-side frame/buffer allocation hints (keyed by global pc) and the /// size of `main`'s frame — the nondeterminism [`Program::execute`] needs to /// run the program. Public verification (§ `verify`) ignores them. @@ -201,7 +223,7 @@ pub struct Program { /// slice of values per `hint_witness` call — the same symbol may be /// hinted many times); each call pops the next entry, whose length must /// match its destination. Prover-side only; verification ignores them. - pub(crate) witness: HashMap>>, + pub(crate) witness: HashMap>>, /// Function pc-ranges `(name, entry, len)` from the compiler, for the /// `DBG_PROF=1` per-function cycle profile ([`Program::execute`]). Purely /// diagnostic; empty for hand-assembled programs. @@ -209,7 +231,7 @@ pub struct Program { } impl Program { - /// Assemble a [`Program`], computing its bytecode [`digest`](Program::digest) + /// Assemble a [`Program`], computing its bytecode digest /// from `prog`. The single funnel for construction, so the digest is always /// consistent with the bytecode. pub fn assemble( @@ -236,7 +258,7 @@ impl Program { /// `hint_witness(dest, "name")` call, popped in order (the same symbol /// may be hinted many times). Prover-side data: entirely unconstrained, /// invisible to verification. - pub fn set_witness(&mut self, name: impl Into, entries: Vec>) { + pub fn set_witness(&mut self, name: impl Into, entries: Vec>) { self.witness.insert(name.into(), entries); } } @@ -276,7 +298,9 @@ pub enum Error { fn block_owners(log_bytecode: usize, sides: [usize; 3]) -> [Vec>; 3] { let sch = schema(); let src = block_kappa_sources(log_bytecode); - let mut it = src.into_iter().map(|(source, _)| source.checked_sub(2).map(|t| (t, sch.base[t]))); + let mut it = src + .into_iter() + .map(|(source, _)| source.checked_sub(2).map(|t| (t, sch.base[t]))); sides.map(|n| it.by_ref().take(n).collect()) } @@ -285,7 +309,11 @@ fn block_owners(log_bytecode: usize, sides: [usize; 3]) -> [Vec Vec<(usize, usize)> { let sch = schema(); - tables::tables().iter().enumerate().map(|(t, tb)| (sch.base[t], tb.n_committed_columns())).collect() + tables::tables() + .iter() + .enumerate() + .map(|(t, tb)| (sch.base[t], tb.n_committed_columns())) + .collect() } /// The per-table inputs to the batched zerocheck (§constraints), in schema order. @@ -296,9 +324,9 @@ fn table_spans() -> Vec<(usize, usize)> { /// evaluated on the same values. The identities take the air's own `η`-range; the /// three forms take the shared powers at [`eta_form_base`]. fn airs<'a>( - taus: &[usize; 6], + taus: &[usize; tables::N_TABLES], forms: &'a [Vec; 3], - form_pows: [F128; 3], + form_pows: [F192; 3], ) -> Vec> { tables::tables() .iter() @@ -312,7 +340,9 @@ fn airs<'a>( n_constraints: table.n_constraints(), eval: Box::new(move |p, vals| { let air = table.eval_constraint(p, &tables::Cols::new(vals)); - bus.iter().zip(form_pows).fold(air, |acc, (form, w)| acc + w * form.eval(vals)) + bus.iter() + .zip(form_pows) + .fold(air, |acc, (form, w)| acc + w * form.eval(vals)) }), } }) @@ -322,9 +352,9 @@ fn airs<'a>( /// Each table's claimed sum: its identities vanish, so what its summand comes to /// is its three bus forms, `η`-weighted. Prover-side only, to build the waiting /// line each round; the verifier needs just their total, which it derives. -fn sigmas(bus: &[Vec; 3], form_pows: [F128; 3]) -> Vec { +fn sigmas(bus: &[Vec; 3], form_pows: [F192; 3]) -> Vec { (0..tables::tables().len()) - .map(|t| (0..3).fold(F128::ZERO, |acc, s| acc + form_pows[s] * bus[s][t])) + .map(|t| (0..3).fold(F192::ZERO, |acc, s| acc + form_pows[s] * bus[s][t])) .collect() } @@ -342,7 +372,7 @@ pub fn eta_form_base() -> usize { } /// The three shared form powers `η^{base}, η^{base+1}, η^{base+2}`. -fn eta_form_pows(eta: F128) -> [F128; 3] { +fn eta_form_pows(eta: F192) -> [F192; 3] { let base = eta_form_base(); let pows = constraints::eta_powers(eta, base + 3); [pows[base], pows[base + 1], pows[base + 2]] @@ -376,16 +406,17 @@ fn blake3_value_slot(col: usize) -> Option { tables::BLAKE3_VALUE_COLS .iter() .position(|&c| base + c == col) - .map(|i| crate::blake3_flock::VM_SLOTS[i]) + .map(|i| crate::blake3_flock::SLOTS[i]) } /// Run statistics returned alongside the proof: the cycle count (total executed -/// instructions), the per-opcode counts `[XOR, MUL, SET, DEREF, JUMP, BLAKE3]`, and the +/// instructions), the per-opcode counts +/// `[XOR, MUL, SET, DEREF, JUMP, BLAKE3, PACK64X2]`, and the /// committed witness size — the sum of the column lengths, i.e. the real data /// before the stacked witness is zero-padded to a power of two `2^m`. pub struct Stats { pub cycles: usize, - pub counts: [usize; 6], + pub counts: [usize; tables::N_TABLES], pub committed: usize, /// Data memory is `2^log_mem` cells (the padded write-once image). pub log_mem: usize, @@ -397,9 +428,11 @@ pub struct Stats { /// Prove the program on the given public input: run it (witness generation), /// then emit everything the verifier needs through the returned [`Proof`] /// (scalar stream + PCS commitment / opening hints). Returns the proof and the -/// run [`Stats`]. -#[tracing::instrument(name = "Prove", skip_all)] -pub fn prove(program: &Program, public_input: [F128; 2]) -> (Proof, Stats) { +/// run [`Stats`]. `log_inv_rate` selects the PCS rate and is announced in the +/// Fiat–Shamir transcript before the commitment. +#[tracing::instrument(name = "Prove", skip_all, fields(log_inv_rate))] +pub fn prove(program: &Program, public_input: [F192; 2], log_inv_rate: usize) -> (Proof, Stats) { + ::pcs::ligerito::validate_log_inv_rate(log_inv_rate).expect("valid log_inv_rate"); let prof = std::env::var("LEANVM_PROFILE").is_ok(); let ms = |t: std::time::Instant| t.elapsed().as_secs_f64() * 1e3; let t = std::time::Instant::now(); @@ -435,9 +468,9 @@ pub fn prove(program: &Program, public_input: [F128; 2]) -> (Proof, Stats) { let mut ps = ProverState::new(b"leanvm-b", &transcript_seed(program, &public_input)); // Announce the prover's sizes, then commit, before sampling any challenge. - announce_public(&mut ps, w.log_mem, w.row_counts); + announce_public(&mut ps, w.log_mem, w.row_counts, log_inv_rate); let t = std::time::Instant::now(); - let committed = tracing::info_span!("Commit").in_scope(|| pcs::commit(&mut ps, &w.q)); + let committed = tracing::info_span!("Commit").in_scope(|| pcs::commit(&mut ps, &w.q, log_inv_rate)); if prof { eprintln!("[prove] commit : {:>7.2} ms", ms(t)); } @@ -463,7 +496,7 @@ pub fn prove(program: &Program, public_input: [F128; 2]) -> (Proof, Stats) { } let t = std::time::Instant::now(); let table_claims = tracing::info_span!("Prove constraints").in_scope(|| { - // ONE sumcheck for all six tables (§constraints). + // One sumcheck for all seven tables (§constraints). // MOVE the columns out: the batch folds them destructively and nothing // reads them again (`prove_balance` is done, and `QPKD < N_SHARED` is never // a table column), so copying them would be ~300 MB for nothing. @@ -476,7 +509,14 @@ pub fn prove(program: &Program, public_input: [F128; 2]) -> (Proof, Stats) { let eta = ps.sample(); let form_pows = eta_form_pows(eta); let sigma = sigmas(&bus.sigmas, form_pows); - constraints::prove(&airs(&l.taus, &bus.forms, form_pows), &mut cols, eta, &bus.point, &sigma, &mut ps) + constraints::prove( + &airs(&l.taus, &bus.forms, form_pows), + &mut cols, + eta, + &bus.point, + &sigma, + &mut ps, + ) }); if prof { eprintln!("[prove] constraints : {:>7.2} ms", ms(t)); @@ -484,40 +524,50 @@ pub fn prove(program: &Program, public_input: [F128; 2]) -> (Proof, Stats) { let mut claims = bus.claims; claims.extend(constraint_claims(&table_claims)); - claims.push(bind_pi_claim(ps.sample(), &w.layout.placements, &w.layout.pi)); + // The PI binding transmits the low/high memory-limb evaluations. The full + // F192 public-input interpolation then determines the top-limb evaluation. + let r_pi = ps.sample(); + let pi_lo = primitives::multilinear::interp_k(F64(w.layout.pi[0].c0), F64(w.layout.pi[1].c0), r_pi); + let pi_hi = primitives::multilinear::interp_k(F64(w.layout.pi[0].c1), F64(w.layout.pi[1].c1), r_pi); + ps.add_scalar(pi_lo); + ps.add_scalar(pi_hi); + claims.extend(bind_pi_claim(r_pi, &w.layout.placements, &w.layout.pi, pi_lo, pi_hi)); // The input/output words bind via the memory bus (value columns are virtual and // route to q_pkd, see `slot_claims`); cv/counter/blen/flags are constants baked // into flock's per-block matrices, so no pin claims are needed. let slots = slot_claims(&w.layout, &claims); - // Run flock's reduction (zerocheck + lincheck) over the prepared witness + // Run flock's reduction (zerocheck + lincheck) over the prepared native // layouts retained from the fused q_pkd build pass; it returns the `(ab, c)` // validity claims on the committed `q_pkd`, discharged by the PCS below in the // SAME Ligerito as every leanVM point claim (the point claims become the - // opener's `stack_pd`). + // opener's `point_claims`). let t = std::time::Instant::now(); let flock_reduction = w .flock_reduction .take() .expect("prepared flock reduction witness is present"); - let reduced = tracing::info_span!("Flock reduction") - .in_scope(|| flock_reduction.prove(&w.cols[QPKD], &mut ps)); + let reduced = tracing::info_span!("Flock reduction").in_scope(|| flock_reduction.prove(&mut ps)); let n_blocks = flock_reduction.n_blocks(); drop(flock_reduction); + if prof { + eprintln!("[prove] reduction : {:>7.2} ms", ms(t)); + } + let t = std::time::Instant::now(); let offset = w.layout.placements[QPKD].offset; let ring = tracing::info_span!("Package ring switch") .in_scope(|| crate::blake3_flock::ring_switch_open(n_blocks, offset, &reduced)); if prof { - eprintln!("[open] reduction : {:>7.2} ms", ms(t)); + eprintln!("[prove] ring pkg : {:>7.2} ms", ms(t)); } - let t_pcs = std::time::Instant::now(); - let mixed_open = tracing::info_span!("PCS open") - .in_scope(|| pcs::open(&mut ps, &committed, &w.q, &slots, &ring)); + let t = std::time::Instant::now(); + let mixed_open = tracing::info_span!("PCS open").in_scope(|| pcs::open(&mut ps, &committed, &w.q, &slots, &ring)); if prof { - eprintln!("[open] pcs::open : {:>7.2} ms", ms(t_pcs)); + eprintln!("[prove] stack open: {:>7.2} ms", ms(t)); } // flock's scalar sub-proof already rode the shared stream (add_scalar at its - // protocol points); only the Merkle-bearing Ligerito needs the hint channel. + // protocol points); only the Merkle-bearing stacked opening needs the hint + // channel. ps.hint_opening(mixed_open); if prof { eprintln!("[prove] open : {:>7.2} ms", ms(t)); @@ -534,21 +584,40 @@ pub fn prove(program: &Program, public_input: [F128; 2]) -> (Proof, Stats) { ) } -/// The public-input binding claim (§8): `MEM(r, 0,…,0) = interp(m[0], m[1], r)`. -/// The value is a deterministic function of the (seeded) public input `pi` and the -/// challenge `r`, so it is NOT transmitted — both sides compute it, and the single -/// opening proves the committed `MEM` really evaluates to it (a memory whose first -/// two cells disagree with `pi` then fails the opening). `pi` is already bound (the -/// seed), so `r` is sampled directly. `placements`/`pi` come from the prover's or -/// verifier's layout; both build the byte-identical claim. -fn bind_pi_claim(r: F128, placements: &[witness::Placement], pi: &[F128; 2]) -> ColumnClaim { - let mut point = vec![F128::ZERO; placements[MEM].n_vars]; +/// The public-input binding (§8): the committed `MEM` at `(r, 0,…,0)` must equal +/// `interp(pi[0], pi[1], r)`, split into its three physical `K` limbs. The +/// prover transmits `MEM_LO(r)` and `MEM_HI(r)`; both sides derive `MEM_TOP(r)` +/// from the full F192 interpolation. The opening discharges all three claims. +/// `placements` and `pi` come from the prover's or verifier's layout, so both +/// sides build byte-identical claims. +fn bind_pi_claim( + r: F192, + placements: &[witness::Placement], + pi: &[F192; 2], + v_lo: F192, + v_hi: F192, +) -> [ColumnClaim; 3] { + let mut point = vec![F192::ZERO; placements[MEM_LO].n_vars]; point[0] = r; - ColumnClaim { - col: MEM, - point, - value: primitives::multilinear::interp(pi[0], pi[1], r), - } + let y2 = F192::Y * F192::Y; + let v_top = (primitives::multilinear::interp(pi[0], pi[1], r) + v_lo + F192::Y * v_hi) * y2.inv(); + [ + ColumnClaim { + col: MEM_LO, + point: point.clone(), + value: v_lo, + }, + ColumnClaim { + col: MEM_HI, + point: point.clone(), + value: v_hi, + }, + ColumnClaim { + col: MEM_TOP, + point, + value: v_top, + }, + ] } /// Everything a recursion harness needs from an accepting verify run, named @@ -558,11 +627,13 @@ fn bind_pi_claim(r: F128, placements: &[witness::Placement], pi: &[F128; 2]) -> /// fold/query data). The sub-proof scalars themselves live on `proof.stream` /// at fixed offsets from its tail. Ordinary callers just `?`-discard it. pub struct VerifySummary { + /// Transcript-bound inverse-rate logarithm used by this proof's PCS. + pub log_inv_rate: usize, pub bytecode_claims: Vec, - pub count_root: F128, + pub count_root: F192, /// Sponge states after: the bus, the zerochecks, the PI sample, and the /// flock reduction. - pub checkpoints: [[F128; 2]; 4], + pub checkpoints: [[F64; 4]; 4], pub zc_claim: flock::zerocheck::ZerocheckClaim, pub lc_claim: flock::lincheck::LincheckClaim, pub opening: pcs::StackedOpeningSummary, @@ -573,13 +644,9 @@ pub struct VerifySummary { /// every scalar the prover wrote and pull the PCS hints, then assert the stream /// was fully consumed. Takes only public inputs — never the prover's witness. #[tracing::instrument(name = "Verify", skip_all)] -pub fn verify( - program: &Program, - public_input: &[F128; 2], - proof: &Proof, -) -> Result { +pub fn verify(program: &Program, public_input: &[F192; 2], proof: &Proof) -> Result { let mut vs = VerifierState::new(b"leanvm-b", proof, &transcript_seed(program, public_input)); - let l = read_public(&mut vs, program, public_input)?; + let (l, log_inv_rate) = read_public(&mut vs, program, public_input)?; let root = pcs::read_commitment(&mut vs).map_err(Error::Transcript)?; // BLAKE3 ↔ flock (single PCS): flock's R1CS validity and every leanVM point @@ -596,8 +663,7 @@ pub fn verify( [l.push.len(), l.pull.len(), l.count.len()], ); let spans = table_spans(); - let bus = leaf::verify_balance(&l.push, &l.pull, &l.count, &l.pad, &owners, &spans, &mut vs) - .map_err(Error::Bus)?; + let bus = leaf::verify_balance(&l.push, &l.pull, &l.count, &l.pad, &owners, &spans, &mut vs).map_err(Error::Bus)?; let checkpoint_bus = vs.sponge_state(); let zc_eta = vs.sample(); @@ -609,14 +675,23 @@ pub fn verify( // are fixed, hitting that one number forces `Σ_t σ_{s,t} = R_s` on all three // sides. A transmitted target would be a free value in its own check, and the // tables' bus blocks would be settled by nothing at all. - let target = (0..3).fold(F128::ZERO, |a, s| a + form_pows[s] * bus.totals[s]); - let table_claims = constraints::verify(&airs(&l.taus, &bus.forms, form_pows), zc_eta, &bus.point, target, &mut vs) - .map_err(Error::Constraint)?; + let target = (0..3).fold(F192::ZERO, |a, s| a + form_pows[s] * bus.totals[s]); + let table_claims = constraints::verify( + &airs(&l.taus, &bus.forms, form_pows), + zc_eta, + &bus.point, + target, + &mut vs, + ) + .map_err(Error::Constraint)?; let checkpoint_zerochecks = vs.sponge_state(); let mut claims = bus.claims; claims.extend(constraint_claims(&table_claims)); - claims.push(bind_pi_claim(vs.sample(), &l.placements, &l.pi)); + let r_pi = vs.sample(); + let pi_lo = vs.next_scalar().map_err(Error::Transcript)?; + let pi_hi = vs.next_scalar().map_err(Error::Transcript)?; + claims.extend(bind_pi_claim(r_pi, &l.placements, &l.pi, pi_lo, pi_hi)); let checkpoint_pi = vs.sponge_state(); let slots = slot_claims(&l, &claims); @@ -626,12 +701,11 @@ pub fn verify( // (mirroring `prove`). `n_blocks = max(n_b3, 1)` — always ≥ 1 instance. let n_blocks = n_b3.max(1); let offset = l.placements[QPKD].offset; - let replay = crate::blake3_flock::verify_reduction(n_blocks, &root, l.m, &mut vs) - .map_err(Error::Blake3)?; + let replay = crate::blake3_flock::verify_reduction(n_blocks, &root, l.m, &mut vs).map_err(Error::Blake3)?; let checkpoint_flock = vs.sponge_state(); let open = vs.next_opening().map_err(Error::Transcript)?; let ring = crate::blake3_flock::ring_switch_verify(n_blocks, offset, replay.ab, replay.c); - let opening = pcs::verify(&mut vs, &slots, &ring, open, l.m, &root).map_err(Error::Open)?; + let opening = pcs::verify(&mut vs, &slots, &ring, open, l.m, log_inv_rate, &root).map_err(Error::Open)?; vs.finish().map_err(Error::Transcript)?; Ok(VerifySummary { bytecode_claims: bus.bytecode_claims, @@ -640,6 +714,7 @@ pub fn verify( zc_claim: replay.zc_claim, lc_claim: replay.lc_claim, opening, + log_inv_rate, }) } @@ -649,7 +724,7 @@ pub fn verify( /// BLAKE3 value columns are virtual — they have no committed placement. A bus /// claim `value_col(r) = v` (at the `n_log`-dim instance point `r`) is re-routed /// to the equal `q_pkd` slot evaluation: an ordinary claim on the committed -/// `QPKD` column at the point freezing the low 7 coords to the slot's bits and +/// `QPKD` column at the point freezing the low 8 coords to the slot's bits and /// the high coords to `r`. No downstream special-casing — it folds into the /// one opening like every other point claim. fn slot_claims(l: &Layout, claims: &[ColumnClaim]) -> Vec { @@ -658,7 +733,7 @@ fn slot_claims(l: &Layout, claims: &[ColumnClaim]) -> Vec { .map(|c| { // A virtual BLAKE3 value column (always virtual): its bus claim at // instance point `c.point` is the q_pkd slot value — a boolean-selector - // (strided) claim on QPKD, folded sparsely (2^n_log, not the 2^(7+n_log) + // (strided) claim on QPKD, folded sparsely (2^n_log, not the 2^(8+n_log) // dense QPKD block). if let Some(slot) = blake3_value_slot(c.col) { return pcs::SlotClaim::Strided { @@ -669,7 +744,7 @@ fn slot_claims(l: &Layout, claims: &[ColumnClaim]) -> Vec { value: c.value, }; } - pcs::SlotClaim::Slot { + pcs::SlotClaim::Point { offset: l.placements[c.col].offset, low_point: c.point.clone(), value: c.value, @@ -682,50 +757,101 @@ fn slot_claims(l: &Layout, claims: &[ColumnClaim]) -> Vec { mod tests { use super::*; - /// A hand-built straight-line program exercising the `BLAKE3` table: set up - /// the two 256-bit inputs (`a` at cells 2,3 and `b` at cells 4,5), hash them - /// into the output `c` (cells 6,7), and halt at the sentinel. The flock - /// validity sub-proof plus the memory / state / bytecode bus interactions - /// are verified end-to-end (the proof carries the Ligerito opening they - /// assert on). - #[test] - fn blake3_proves_and_verifies() { - let x0 = F128::new(0x0123_4567_89ab_cdef, 0xfedc_ba98_7654_3210); - let x1 = F128::new(0x1111_2222_3333_4444, 0x5555_6666_7777_8888); - let y0 = F128::new(0xdead_beef_cafe_babe, 0x0badf00d_0badf00d); - let y1 = F128::new(0x9999_aaaa_bbbb_cccc, 0xdddd_eeee_ffff_0000); - - // 8 slots (power of two). Slots 4 and 6 are filler SETs whose only job is to - // step the pc so the last executed instruction lands at slot 6 (→ pc 7, - // halt). Slot 7 is the never-executed sentinel. - let prog = vec![ - Op::Set { o: 2, k: x0 }, - Op::Set { o: 3, k: x1 }, - Op::Set { o: 4, k: y0 }, - Op::Set { o: 5, k: y1 }, - Op::Set { o: 8, k: F128::ONE }, + /// A K-embedded immediate (both extension limbs zero). + fn w(x: u64) -> F192 { + F192::new(x, 0, 0) + } + + /// Pack two 64-bit flock words into the canonical BLAKE3 subspace of F192. + fn cell(lo: F64, hi: F64) -> F192 { + F192::new(lo.0, hi.0, 0) + } + + /// The default one-block-root metadata for a hand-built BLAKE3 op. + fn md() -> F192 { + crate::blake3_flock::metadata(0, 64, crate::blake3_flock::FLAGS) + } + + /// The four chaining-value lanes of the two cv cells. + fn cv_lanes(cv0: F192, cv1: F192) -> [F64; 4] { + [F64(cv0.c0), F64(cv0.c1), F64(cv1.c0), F64(cv1.c1)] + } + + /// A hand-built straight-line program with one BLAKE3 row: set up the two + /// 256-bit inputs (`a` at cells 2,3, `b` at cells 4,5 — one 128-bit word per + /// cell), hash them into the output `c` (cells 6,7), pad with filler SETs so + /// the last executed instruction lands one before the sentinel, and halt + /// there. The flock validity sub-proof plus the memory / state / bytecode bus + /// interactions are verified end-to-end (the proof carries the Ligerito + /// opening they assert on). + fn blake3_program(a: [F64; 4], b: [F64; 4]) -> Program { + // a → cells 2,3 and b → cells 4,5 (two flock lanes per BLAKE3 cell). + let mut prog = vec![ + Op::Set { + o: 2, + k: cell(a[0], a[1]), + }, + Op::Set { + o: 3, + k: cell(a[2], a[3]), + }, + Op::Set { + o: 4, + k: cell(b[0], b[1]), + }, + Op::Set { + o: 5, + k: cell(b[2], b[3]), + }, + // The chaining value reads cells 0,1 (the public input); any + // canonical cv is legal. Op::Blake3 { ins: [2, 3, 4, 5], cv: 0, out: 6, metadata: crate::blake3_flock::metadata(0, 64, crate::blake3_flock::FLAGS), }, - Op::Set { o: 9, k: F128::ONE }, - Op::Xor { a: 0, b: 0, c: 0 }, // sentinel (never executed) + ]; // c → cells 6,7 + // 16 slots: 5 executed so far; 10 filler SETs step the pc to 15 (halt); + // slot 15 is the never-executed sentinel. + for k in 0..10u32 { + prog.push(Op::Set { + o: 16 + k, + k: F192::ONE, + }); + } + prog.push(Op::Xor { a: 0, b: 0, c: 0 }); // sentinel (never executed) + assert_eq!(prog.len(), 16); + Program::from_bytecode(prog, 32) + } + + #[test] + fn blake3_proves_and_verifies() { + let a: [F64; 4] = [ + F64(0x0123_4567_89ab_cdef), + F64(0xfedc_ba98_7654_3210), + F64(0x1111_2222_3333_4444), + F64(0x5555_6666_7777_8888), ]; - let program = Program::assemble(prog, 0, 0, HashMap::new(), 10); + let b: [F64; 4] = [ + F64(0xdead_beef_cafe_babe), + F64(0x0badf00d_0badf00d), + F64(0x9999_aaaa_bbbb_cccc), + F64(0xdddd_eeee_ffff_0000), + ]; + let program = blake3_program(a, b); - let pi = crate::blake3_flock::IV; + let pi = [w(7), w(11)]; let exec = program.execute(pi); - // The output cells hold the digest of the two inputs (the prover computes - // a definite value even though nothing constrains it). - let (d0, d1) = blake3_compress(x0, x1, y0, y1, pi[0], pi[1], crate::blake3_flock::metadata(0, 64, crate::blake3_flock::FLAGS)); - assert_eq!(exec.mem[6], d0); - assert_eq!(exec.mem[7], d1); + // The output cells hold the compression of the two inputs under the + // pi-supplied chaining value (two 128-bit chunks). + let d = blake3_compress(a, b, cv_lanes(pi[0], pi[1]), md()); + assert_eq!(exec.mem[6], cell(d[0], d[1])); + assert_eq!(exec.mem[7], cell(d[2], d[3])); assert_eq!(exec.trace.blake3.len(), 1); - let (proof, stats) = prove(&program, pi); + let (proof, stats) = prove(&program, pi, pcs::LOG_INV_RATE); assert_eq!(stats.counts[5], 1, "one BLAKE3 row"); // flock's sub-proof rides the shared channels: its Ligerito is the proof's // one opening, its scalar reduction trails the `stream`. @@ -733,85 +859,87 @@ mod tests { verify(&program, &pi, &proof).expect("BLAKE3 program verifies"); } - /// A self-hash `BLAKE3(h, h)` (the hash-chain step) passes the *same* operand - /// base as both `a` and `b` (`a == b`), so one 256-bit pair feeds both inputs - /// with no copy. The row reads those two cells twice; the running access counts - /// thread through and the bus still balances. This is the aliasing the - /// consecutive-pair DSL lowering relies on. + /// BLAKE consumes the `(c0,c1,0)` embedding. This is not an extra AIR + /// constraint: the full three-limb memory bus makes a request carrying a + /// literal zero in limb 2 match only such a stored word. + #[test] + #[should_panic(expected = "BLAKE3 input cell must be a canonical 128-bit embedding")] + fn blake3_requires_zero_third_limb() { + let mut program = blake3_program([F64::ZERO; 4], [F64::ZERO; 4]); + program.prog[0] = Op::Set { + o: 2, + k: F192::new(0, 0, 1), + }; + let _ = program.execute([w(7), w(11)]); + } + + /// A self-hash `BLAKE3(h, h)` (the hash-chain step) passes the *same* input + /// chunks as both `a` and `b` (`ins[0..2] == ins[2..4]`), so one 256-bit quad + /// feeds both inputs with no copy. The row reads those cells twice; the + /// running access counts thread through and the bus still balances. This is + /// the aliasing the DSL's hash-chain lowering relies on. #[test] fn blake3_self_hash_aliased_operands() { - let h0 = F128::new(0xfeed_face_dead_beef, 0x0123_4567_89ab_cdef); - let h1 = F128::new(0xcafe_d00d_1337_c0de, 0x8877_6655_4433_2211); - // 8 slots (power of two). Slots 2,3,6 are filler SETs stepping the pc so the - // last executed instruction (slot 6) lands at pc 7 (the sentinel, halt). - let prog = vec![ - Op::Set { o: 2, k: h0 }, // operand pair h = (cell 2, cell 3) - Op::Set { o: 3, k: h1 }, - Op::Set { o: 8, k: F128::ONE }, // filler - Op::Set { o: 9, k: F128::ONE }, // filler - Op::Set { o: 10, k: F128::ONE }, // filler - Op::Blake3 { - ins: [2, 3, 2, 3], - cv: 0, - out: 6, - metadata: crate::blake3_flock::metadata(0, 64, crate::blake3_flock::FLAGS), - }, // a == b: hash h ‖ h into cells 6,7 - Op::Set { o: 11, k: F128::ONE }, // filler - Op::Xor { a: 0, b: 0, c: 0 }, // sentinel + let h: [F64; 4] = [ + F64(0xfeed_face_dead_beef), + F64(0x0123_4567_89ab_cdef), + F64(0xcafe_d00d_1337_c0de), + F64(0x8877_6655_4433_2211), ]; + // 8 slots: 2 SETs (h at cells 2,3), the aliased BLAKE3 (output 4,5), + // 2 filler SETs stepping the pc to 7 (the sentinel, halt). + let mut prog = Vec::new(); + prog.push(Op::Set { + o: 2, + k: cell(h[0], h[1]), + }); + prog.push(Op::Set { + o: 3, + k: cell(h[2], h[3]), + }); + prog.push(Op::Blake3 { + ins: [2, 3, 2, 3], + cv: 0, + out: 4, + metadata: crate::blake3_flock::metadata(0, 64, crate::blake3_flock::FLAGS), + }); // a == b: hash h ‖ h into cells 4,5 + for k in 0..4u32 { + prog.push(Op::Set { + o: 12 + k, + k: F192::ONE, + }); // fillers step pc to the sentinel + } + prog.push(Op::Xor { a: 0, b: 0, c: 0 }); // sentinel + assert_eq!(prog.len(), 8); let program = Program::from_bytecode(prog, 16); - let pi = crate::blake3_flock::IV; + let pi = [w(3), w(5)]; let exec = program.execute(pi); - let (d0, d1) = blake3_compress(h0, h1, h0, h1, pi[0], pi[1], crate::blake3_flock::metadata(0, 64, crate::blake3_flock::FLAGS)); - assert_eq!(exec.mem[6], d0); - assert_eq!(exec.mem[7], d1); + let d = blake3_compress(h, h, cv_lanes(pi[0], pi[1]), md()); + assert_eq!(exec.mem[4], cell(d[0], d[1])); + assert_eq!(exec.mem[5], cell(d[2], d[3])); - let (proof, stats) = prove(&program, pi); + let (proof, stats) = prove(&program, pi, pcs::LOG_INV_RATE); assert_eq!(stats.counts[5], 1, "one BLAKE3 row"); verify(&program, &pi, &proof).expect("self-hash BLAKE3 verifies"); } - /// Tampering flock's validity sub-proof (its Ligerito `final_b`, opened over - /// the same stacked commitment) must make verification fail. + /// Tampering flock's validity sub-proof (its Ligerito, opened over the same + /// stacked commitment) must make verification fail. #[test] fn blake3_rejects_tampered_validity() { - let prog = vec![ - Op::Set { - o: 2, - k: F128::new(0xABCD, 0x1234), - }, - Op::Set { - o: 3, - k: F128::new(0x5678, 0x9999), - }, - Op::Set { - o: 4, - k: F128::new(0x1111, 0x2222), - }, - Op::Set { - o: 5, - k: F128::new(0x3333, 0x4444), - }, - Op::Set { o: 8, k: F128::ONE }, - Op::Blake3 { - ins: [2, 3, 4, 5], cv: 0, out: 6, - metadata: crate::blake3_flock::metadata(0, 64, crate::blake3_flock::FLAGS), - }, - Op::Set { o: 9, k: F128::ONE }, - Op::Xor { a: 0, b: 0, c: 0 }, // sentinel - ]; - let program = Program::from_bytecode(prog, 10); - let pi = crate::blake3_flock::IV; - let (mut proof, _) = prove(&program, pi); + let program = blake3_program( + [F64(0xABCD), F64(0x1234), F64(0x5678), F64(0x9999)], + [F64(0x1111), F64(0x2222), F64(0x3333), F64(0x4444)], + ); + let pi = [w(7), w(11)]; + let (mut proof, _) = prove(&program, pi, pcs::LOG_INV_RATE); verify(&program, &pi, &proof).expect("honest proof verifies"); - // flock's Ligerito opening is the proof's one hint; tamper an opened - // row (the Merkle-bound query data) — must be rejected. (The sumcheck - // round messages ride the stream and are covered by the stream-tamper - // test below.) - let lig = proof.openings.last_mut().expect("flock Ligerito opening"); - lig.initial_proof.opened_rows[0][0] += F128::ONE; + // The stacked opening is the proof's one hint; tamper a sumcheck + // round message (the inner-product transcript) — must be rejected. + let lig = proof.openings.last_mut().expect("stacked Ligerito opening"); + lig.ligerito.sumcheck_transcript[0].u_0 += F192::ONE; assert!( verify(&program, &pi, &proof).is_err(), "tampered BLAKE3 validity proof must be rejected" @@ -826,34 +954,12 @@ mod tests { /// which tampers the Ligerito opening.) #[test] fn blake3_rejects_tampered_reduction() { - let prog = vec![ - Op::Set { - o: 2, - k: F128::new(0xABCD, 0x1234), - }, - Op::Set { - o: 3, - k: F128::new(0x5678, 0x9999), - }, - Op::Set { - o: 4, - k: F128::new(0x1111, 0x2222), - }, - Op::Set { - o: 5, - k: F128::new(0x3333, 0x4444), - }, - Op::Set { o: 8, k: F128::ONE }, - Op::Blake3 { - ins: [2, 3, 4, 5], cv: 0, out: 6, - metadata: crate::blake3_flock::metadata(0, 64, crate::blake3_flock::FLAGS), - }, - Op::Set { o: 9, k: F128::ONE }, - Op::Xor { a: 0, b: 0, c: 0 }, // sentinel - ]; - let program = Program::from_bytecode(prog, 10); - let pi = crate::blake3_flock::IV; - let (proof, _) = prove(&program, pi); + let program = blake3_program( + [F64(0xABCD), F64(0x1234), F64(0x5678), F64(0x9999)], + [F64(0x1111), F64(0x2222), F64(0x3333), F64(0x4444)], + ); + let pi = [w(7), w(11)]; + let (proof, _) = prove(&program, pi, pcs::LOG_INV_RATE); verify(&program, &pi, &proof).expect("honest proof verifies"); // The reduction is serialized onto the stream tail (after the last bound @@ -861,7 +967,7 @@ mod tests { // always meaningful bytes (only the final word may be zero-padded). let mut tampered = proof.clone(); let n = tampered.stream.len(); - tampered.stream[n - 2] += F128::ONE; + tampered.stream[n - 2] += F192::ONE; assert!( verify(&program, &pi, &tampered).is_err(), "tampered reduction transport must be rejected" @@ -875,26 +981,40 @@ mod tests { #[test] fn non_blake3_program_verifies() { let prog = vec![ - Op::Set { - o: 2, - k: F128::new(5, 0), - }, - Op::Set { - o: 3, - k: F128::new(6, 0), - }, + Op::Set { o: 2, k: w(5) }, + Op::Set { o: 3, k: w(6) }, Op::Xor { a: 2, b: 3, c: 4 }, Op::Xor { a: 0, b: 0, c: 0 }, // sentinel ]; let program = Program::from_bytecode(prog, 5); - let pi = [F128::new(1, 0), F128::new(2, 0)]; - let (proof, stats) = prove(&program, pi); + let pi = [F192::new(1, 2, 3), F192::new(4, 5, 6)]; + let (proof, stats) = prove(&program, pi, pcs::LOG_INV_RATE); assert_eq!(stats.counts[5], 0, "no real BLAKE3 rows"); // The proof still carries exactly one Ligerito opening (over the padding). assert_eq!(proof.openings.len(), 1, "unified path: one opening always"); verify(&program, &pi, &proof).expect("non-BLAKE3 program verifies"); } + /// A 192-bit-word MUL: the E-product of two full machine words is proven and + /// verified. Exercises the tower-product constraint (all limbs nonzero). + #[test] + fn mul_192bit_word() { + let x = F192::new(0x0123_4567_89ab_cdef, 0xfeed_face_dead_beef, 0x1111_2222_3333_4444); + let y = F192::new(0x9999_aaaa_bbbb_cccc, 0x1357_9bdf_2468_ace0, 0x5555_6666_7777_8888); + let prog = vec![ + Op::Set { o: 2, k: x }, + Op::Set { o: 3, k: y }, + Op::Mul { a: 2, b: 3, c: 4 }, + Op::Xor { a: 0, b: 0, c: 0 }, // sentinel + ]; + let program = Program::from_bytecode(prog, 5); + let pi = [w(1), w(2)]; + let exec = program.execute(pi); + assert_eq!(exec.mem[4], x * y, "MUL computes the E product"); + let (proof, _) = prove(&program, pi, pcs::LOG_INV_RATE); + verify(&program, &pi, &proof).expect("192-bit MUL verifies"); + } + /// A proof is bound to its exact program: presenting it against a *different* /// program (same sizes/layout, one instruction constant changed) must be /// rejected — the program digest seeds the transcript, so a modified program @@ -903,28 +1023,22 @@ mod tests { #[test] fn proof_bound_to_program() { let prog = vec![ - Op::Set { - o: 2, - k: F128::new(5, 0), - }, - Op::Set { - o: 3, - k: F128::new(6, 0), - }, + Op::Set { o: 2, k: w(5) }, + Op::Set { o: 3, k: w(6) }, Op::Xor { a: 2, b: 3, c: 4 }, Op::Xor { a: 0, b: 0, c: 0 }, // sentinel ]; let program = Program::from_bytecode(prog.clone(), 5); - let pi = [F128::new(1, 0), F128::new(2, 0)]; - let (proof, _) = prove(&program, pi); + let pi = [w(1), w(2)]; + let (proof, _) = prove(&program, pi, pcs::LOG_INV_RATE); verify(&program, &pi, &proof).expect("honest proof verifies"); // Same shape (4 ops, same opcodes/operands, so identical layout + announced - // sizes) but one SET constant changed. Must be rejected. + // sizes) but only the SET constant's third limb changed. Must be rejected. let mut prog2 = prog; prog2[0] = Op::Set { o: 2, - k: F128::new(99, 0), + k: F192::new(5, 0, 1), }; let program2 = Program::from_bytecode(prog2, 5); assert!( @@ -933,45 +1047,55 @@ mod tests { ); } - /// Out-of-process verification: a BLAKE3 proof (whose flock sub-proof now rides + /// Out-of-process verification: a BLAKE3 proof (whose flock sub-proof rides /// the shared `stream` + `openings`, no side field) serializes to bytes, /// deserializes on the other side, and verifies — everything travels in the two /// channels, nothing out of band. A flipped encoded byte must not verify. #[test] fn proof_roundtrips_through_bytes_and_verifies() { - let prog = vec![ - Op::Set { - o: 2, - k: F128::new(0xABCD, 0x1234), - }, - Op::Set { - o: 3, - k: F128::new(0x5678, 0x9999), - }, - Op::Set { - o: 4, - k: F128::new(0x1111, 0x2222), - }, - Op::Set { - o: 5, - k: F128::new(0x3333, 0x4444), - }, - Op::Set { o: 8, k: F128::ONE }, - Op::Blake3 { - ins: [2, 3, 4, 5], cv: 0, out: 6, - metadata: crate::blake3_flock::metadata(0, 64, crate::blake3_flock::FLAGS), - }, - Op::Set { o: 9, k: F128::ONE }, - Op::Xor { a: 0, b: 0, c: 0 }, // sentinel - ]; - let program = Program::from_bytecode(prog, 10); - let pi = crate::blake3_flock::IV; - let (proof, _) = prove(&program, pi); + let program = blake3_program( + [F64(0xABCD), F64(0x1234), F64(0x5678), F64(0x9999)], + [F64(0x1111), F64(0x2222), F64(0x3333), F64(0x4444)], + ); + let pi = [w(7), w(11)]; + let (proof, _) = prove(&program, pi, pcs::LOG_INV_RATE); let bytes = bincode::serialize(&proof).expect("proof serializes"); let decoded: Proof = bincode::deserialize(&bytes).expect("proof deserializes"); verify(&program, &pi, &decoded).expect("deserialized BLAKE3 proof verifies"); + let mut bad_rate = decoded.clone(); + bad_rate.stream[1 + tables::N_TABLES] = F192::new(5, 0, 0); + assert!( + matches!(verify(&program, &pi, &bad_rate), Err(Error::PublicInput)), + "the transcript-announced PCS rate must be in 1..=4" + ); + + for announcement in 0..=tables::N_TABLES + 1 { + for high_limb in [F192::new(0, 1, 0), F192::new(0, 0, 1)] { + let mut malformed = decoded.clone(); + malformed.stream[announcement] += high_limb; + assert!( + matches!(verify(&program, &pi, &malformed), Err(Error::PublicInput)), + "announcement {announcement} with a nonzero high limb must be rejected" + ); + } + } + + let root_offset = tables::N_TABLES + 2; + for root_word in root_offset..root_offset + 2 { + let mut malformed = decoded.clone(); + malformed.stream[root_word].c2 = 1; + assert!( + matches!( + verify(&program, &pi, &malformed), + Err(Error::Transcript(crate::transcript::Error::NonCanonicalEncoding)) + ), + "commitment root word {} with a nonzero top limb must be rejected", + root_word - root_offset + ); + } + let mut tampered = bytes.clone(); let i = tampered.len() / 2; tampered[i] ^= 0x01; diff --git a/crates/lean_vm/src/cpu/trace.rs b/crates/lean_vm/src/cpu/trace.rs index 0b07d5d6..b50aa1a9 100644 --- a/crates/lean_vm/src/cpu/trace.rs +++ b/crates/lean_vm/src/cpu/trace.rs @@ -1,7 +1,7 @@ //! Per-opcode trace rows, emitted during execution and assembled into a [`Trace`]. use super::DerefMode; -use primitives::field::F128; +use primitives::field::{F64, F192}; pub(crate) struct Xrow { pub(crate) pc: u32, @@ -9,19 +9,19 @@ pub(crate) struct Xrow { pub(crate) aa: u32, pub(crate) ab: u32, pub(crate) ac: u32, - pub(crate) ra: F128, - pub(crate) rb: F128, - pub(crate) rc: F128, - pub(crate) bytecode_read: F128, + pub(crate) ra: F64, + pub(crate) rb: F64, + pub(crate) rc: F64, + pub(crate) bytecode_read: F64, } pub(crate) struct Srow { pub(crate) pc: u32, pub(crate) fp: u32, pub(crate) o: u32, pub(crate) a: u32, - pub(crate) k: F128, - pub(crate) r: F128, - pub(crate) bytecode_read: F128, + pub(crate) k: F192, // the stored immediate, a 192-bit machine word + pub(crate) r: F64, + pub(crate) bytecode_read: F64, } pub(crate) struct Drow { pub(crate) pc: u32, @@ -31,41 +31,44 @@ pub(crate) struct Drow { pub(crate) gamma: u32, pub(crate) mode: DerefMode, pub(crate) a1: u32, - pub(crate) p: F128, + pub(crate) p: F192, // mem[a1], the pointer word (a K-valued address, read as a full word) pub(crate) a2: usize, pub(crate) a3: u32, - pub(crate) v2: F128, // mem[a2], the store target - pub(crate) v3: F128, // mem[a3], the local cell - pub(crate) r1: F128, - pub(crate) r2: F128, - pub(crate) r3: F128, - pub(crate) bytecode_read: F128, + pub(crate) v2: F192, // mem[a2], the store target + pub(crate) v3: F192, // mem[a3], the local cell + pub(crate) r1: F64, + pub(crate) r2: F64, + pub(crate) r3: F64, + pub(crate) bytecode_read: F64, } pub(crate) struct Jrow { pub(crate) pc: u32, pub(crate) fp: u32, - pub(crate) npc: F128, - pub(crate) nfp: F128, + pub(crate) npc: F64, // next pc — a K-valued address + pub(crate) nfp: F64, // next fp — a K-valued address pub(crate) oc: u32, pub(crate) od: u32, pub(crate) of: u32, pub(crate) ac: u32, pub(crate) ad: u32, pub(crate) af: u32, - pub(crate) c: F128, - pub(crate) d: F128, - pub(crate) f: F128, - pub(crate) w: F128, // inverse hint (is-nonzero witness): c⁻¹ when c ≠ 0, else 0 - pub(crate) b: F128, // taken indicator b = [c ≠ 0] - pub(crate) rc: F128, - pub(crate) rd: F128, - pub(crate) rf: F128, - pub(crate) bytecode_read: F128, + pub(crate) c: F192, // condition, an arbitrary 192-bit word + pub(crate) d: F192, // destination word (a K-valued code address, read as a full word) + pub(crate) f: F192, // new frame word (a K-valued frame pointer, read as a full word) + pub(crate) w: F192, // inverse hint (is-nonzero witness): c⁻¹ when c ≠ 0, else 0 + pub(crate) b: F64, // taken indicator b = [c ≠ 0] + pub(crate) rc: F64, + pub(crate) rd: F64, + pub(crate) rf: F64, + pub(crate) bytecode_read: F64, } -/// `BLAKE3` row: four independent message-word addresses, the chaining-value -/// base `acv`, and the output base `ac`. CV and output each span two consecutive -/// words; all eight memory words carry independent offline-memory counts. +/// `BLAKE3` row: the four independent message-chunk addresses `aa0, aa1, ab0, +/// ab1` (each a canonical 128-bit chunk in one 192-bit cell), the +/// chaining-value base `acv`, and the output base `ac` (CV and output each +/// span two consecutive cells); the eighteen flock words (message `a`/`b`, +/// chaining value `cv`, output `c`, metadata — two 64-bit lanes per chunk), +/// and the eight per-cell memory access counts. pub(crate) struct Brow { pub(crate) pc: u32, pub(crate) fp: u32, @@ -75,24 +78,16 @@ pub(crate) struct Brow { pub(crate) ab1: u32, pub(crate) acv: u32, pub(crate) ac: u32, - pub(crate) va0: F128, - pub(crate) va1: F128, - pub(crate) vb0: F128, - pub(crate) vb1: F128, - pub(crate) vcv0: F128, - pub(crate) vcv1: F128, - pub(crate) metadata: F128, - pub(crate) vc0: F128, - pub(crate) vc1: F128, - pub(crate) ra0: F128, - pub(crate) ra1: F128, - pub(crate) rb0: F128, - pub(crate) rb1: F128, - pub(crate) rcv0: F128, - pub(crate) rcv1: F128, - pub(crate) rc0: F128, - pub(crate) rc1: F128, - pub(crate) bytecode_read: F128, + pub(crate) va: [F64; 4], // a's four flock words = cells (aa0, aa1), lanes (lo, hi) + pub(crate) vb: [F64; 4], // b's four flock words = cells (ab0, ab1) + pub(crate) vcv: [F64; 4], // cv's four flock words = cells (acv, acv+1) + pub(crate) vc: [F64; 4], // c's four flock words = cells (ac, ac+1) + pub(crate) metadata: F192, // counter:u64 | block_len:u32 | flags:u32 (top lane zero) + pub(crate) ra: [F64; 2], // per-cell counts for the two a input cells + pub(crate) rb: [F64; 2], // … the two b input cells + pub(crate) rcv: [F64; 2], // … the two cv input cells + pub(crate) rc: [F64; 2], // … the two c output cells + pub(crate) bytecode_read: F64, } pub(crate) struct Trace { @@ -102,6 +97,7 @@ pub(crate) struct Trace { pub(crate) deref: Vec, pub(crate) jump: Vec, pub(crate) blake3: Vec, - pub(crate) mem_count: Vec, // per-cell running access count g^{count}; final = g^{A[i]} - pub(crate) bytecode_count: Vec, // per-pc running execution count g^{count}; final = g^{A[pc]} + pub(crate) pack64x2: Vec, + pub(crate) mem_count: Vec, // per-cell running access count g^{count}; final = g^{A[i]} + pub(crate) bytecode_count: Vec, // per-pc running execution count g^{count}; final = g^{A[pc]} } diff --git a/crates/lean_vm/src/gkr.rs b/crates/lean_vm/src/gkr.rs index 3c3e6f5a..e412b88d 100644 --- a/crates/lean_vm/src/gkr.rs +++ b/crates/lean_vm/src/gkr.rs @@ -3,25 +3,32 @@ //! `Ṽ_0(ζ)`. Two binary levels are contracted at a time: a radix-four layer has //! relation `V_i(x)=∏_{a,b∈{0,1}}V_{i-2}(a,b,x)`. Its normalized eq-trick //! sumcheck has degree four. An odd-depth tree starts with one binary layer. +//! Leaves and every layer are `E`-valued (the bus fingerprints mix `K`-columns +//! into `E` upstream, [`crate::leaf`]). -use crate::transcript::{ProverState, VerifierState}; use crate::PAR_THRESHOLD; -use primitives::field::{F256Unreduced, F128}; -use primitives::multilinear::{build_eq, interp, quartic_eval_from_eq, shrink_eq_low}; +use crate::transcript::{ProverState, VerifierState}; +use primitives::field::{F192, F192Unreduced}; +use primitives::multilinear::{eq_table, interp, quartic_eval_from_eq, shrink_eq_low}; use rayon::prelude::*; use std::ops::Range; #[inline] -fn mul_pair(a: [F128; 2], b: [F128; 2]) -> [F128; 2] { +fn mul_pair(a: [F192; 2], b: [F192; 2]) -> [F192; 2] { #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2"))] { - // SAFETY: both required features are enabled for the whole crate. - unsafe { primitives::field::gf2_128::x86_64::ghash_mul_vec2_clmul(a, b) } + // SAFETY: both features are enabled for the whole crate. + unsafe { primitives::field::gf2_64x3::x86_64::mul_vec2(a, b) } } #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] { // SAFETY: PMULL is enabled for the whole crate. - unsafe { primitives::field::gf2_128::aarch64::ghash_mul_vec2_neon(a, b) } + unsafe { + [ + primitives::field::gf2_64x3::aarch64::mul_karatsuba(a[0], b[0]), + primitives::field::gf2_64x3::aarch64::mul_karatsuba(a[1], b[1]), + ] + } } #[cfg(not(any( all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2"), @@ -33,23 +40,13 @@ fn mul_pair(a: [F128; 2], b: [F128; 2]) -> [F128; 2] { } #[inline] -fn mul_four(a: [F128; 4], b: [F128; 4]) -> [F128; 4] { - #[cfg(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx512f", - target_feature = "avx512bw" - ))] +fn mul_four(a: [F192; 4], b: [F192; 4]) -> [F192; 4] { + #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] { - // SAFETY: all required features are enabled for the whole crate. - unsafe { primitives::field::gf2_128::x86_64::ghash_mul_vec4_clmul(a, b) } + // SAFETY: both features are enabled for the whole crate. + unsafe { primitives::field::gf2_64x3::x86_64::mul_vec4(a, b) } } - #[cfg(not(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx512f", - target_feature = "avx512bw" - )))] + #[cfg(not(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f")))] { let low = mul_pair([a[0], a[1]], [b[0], b[1]]); let high = mul_pair([a[2], a[3]], [b[2], b[3]]); @@ -58,11 +55,11 @@ fn mul_four(a: [F128; 4], b: [F128; 4]) -> [F128; 4] { } #[inline] -fn mul_unreduced_four(a: F128, b: [F128; 4]) -> [F256Unreduced; 4] { +fn mul_unreduced_four(a: F192, b: [F192; 4]) -> [F192Unreduced; 4] { #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] { - // SAFETY: all required features are enabled for the whole crate. - unsafe { primitives::field::gf2_128::x86_64::ghash_mul_unreduced_vec4_clmul([a; 4], b) } + // SAFETY: both features are enabled for the whole crate. + unsafe { primitives::field::gf2_64x3::x86_64::mul_unreduced_vec4([a; 4], b) } } #[cfg(all( target_arch = "x86_64", @@ -71,21 +68,17 @@ fn mul_unreduced_four(a: F128, b: [F128; 4]) -> [F256Unreduced; 4] { not(target_feature = "avx512f") ))] { - // SAFETY: both required features are enabled for the whole crate. + // SAFETY: both features are enabled for the whole crate. unsafe { - let low = primitives::field::gf2_128::x86_64::ghash_mul_unreduced_vec2_clmul([a; 2], [b[0], b[1]]); - let high = primitives::field::gf2_128::x86_64::ghash_mul_unreduced_vec2_clmul([a; 2], [b[2], b[3]]); + let low = primitives::field::gf2_64x3::x86_64::mul_unreduced_vec2([a; 2], [b[0], b[1]]); + let high = primitives::field::gf2_64x3::x86_64::mul_unreduced_vec2([a; 2], [b[2], b[3]]); [low[0], low[1], high[0], high[1]] } } #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] { // SAFETY: PMULL is enabled for the whole crate. - unsafe { - let low = primitives::field::gf2_128::aarch64::ghash_mul_unreduced_vec2_neon([a; 2], [b[0], b[1]]); - let high = primitives::field::gf2_128::aarch64::ghash_mul_unreduced_vec2_neon([a; 2], [b[2], b[3]]); - [low[0], low[1], high[0], high[1]] - } + unsafe { b.map(|value| primitives::field::gf2_64x3::aarch64::mul_unreduced_neon(a, value)) } } #[cfg(not(any( all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"), @@ -103,54 +96,35 @@ pub enum GkrError { LayerMismatch { layer: usize }, } -/// Rows `[start, end)` of a layer that all hold `value`: an aligned constant -/// run, which the prover skips. Both rows of a fold pair are then flat in the -/// round variable, so their summand is constant in it and moves only `q`'s -/// constant coefficient, which the verifier reconstructs from the incoming claim -/// (`T = (1+s)q(0)+s·q(1)`, see [`quartic_eval_from_eq`]). None of the four -/// transmitted coefficients moves, so skipping is exact and the transcript is -/// the dense one. Every layer is stored densely all the same, so a run list only -/// ever removes multiplications: forgetting a run costs work, never soundness. +/// Rows `[start, end)` of a layer that all hold `value`. Such an aligned +/// constant run can be skipped in the layer sumcheck: a constant fold pair only +/// changes the constant coefficient, which the verifier reconstructs from the +/// incoming claim. #[derive(Clone, Copy, Debug)] pub struct PadRun { start: usize, end: usize, - value: F128, + value: F192, } impl PadRun { - /// Rows `[start, end)` of the leaf vector, all holding `value`. - pub fn new(start: usize, end: usize, value: F128) -> Self { + pub fn new(start: usize, end: usize, value: F192) -> Self { Self { start, end, value } } - /// The run after `1 << log_group` consecutive rows collapse into one of - /// value `value`: the rows lying wholly inside, `[⌈start/g⌉, ⌊end/g⌋)`, and - /// `None` once none does. A straddling row is left to the dense path. - fn regroup(self, log_group: usize, value: F128) -> Option { + fn regroup(self, log_group: usize, value: F192) -> Option { let start = self.start.div_ceil(1 << log_group); let end = self.end >> log_group; (start < end).then_some(Self { start, end, value }) } } -/// Rows per rayon task, given the rows to cover. The runs are skipped *inside* a -/// task rather than by handing each run its own parallel region: the blocks -/// differ in size by orders of magnitude, so per-run regions leave the small ones -/// running one at a time while the pool idles. Uniform windows instead keep the -/// parallel decomposition a run-free prover would use, and work stealing absorbs -/// windows that turn out to be all padding. The count is scaled to the pool so a -/// mid-sized round, of which the layers walk through many, still spreads over it. fn window_rows(total: usize) -> usize { let tasks = (rayon::current_num_threads() * 4).max(1); total.div_ceil(tasks).clamp(32, 1 << 12) } -/// Cut `window` into the alternating pieces the runs induce: `visit(piece, None)` -/// for rows to compute, `visit(piece, Some(value))` for a run to fill or skip. -/// The runs are sorted and disjoint, so a window binary-searches to its first one -/// and then walks only the runs it meets, usually none or one. -fn for_each_piece(window: Range, runs: &[PadRun], mut visit: impl FnMut(Range, Option)) { +fn for_each_piece(window: Range, runs: &[PadRun], mut visit: impl FnMut(Range, Option)) { let first = runs.partition_point(|run| run.end <= window.start); let mut cursor = window.start; for run in &runs[first..] { @@ -169,52 +143,37 @@ fn for_each_piece(window: Range, runs: &[PadRun], mut visit: impl FnMut(R } } -/// One tree's leaves, plus the aligned runs of them that hold a known constant. -/// On the bus those are a block's padding rows, whose tuple is the fixed default -/// of §e2e-pad, so one value covers the run. +/// One tree's dense leaves and the aligned runs known to be constant. pub struct LeafVector { - leaves: Vec, + leaves: Vec, pads: Vec, } impl LeafVector { - /// Sorts the runs; they must be disjoint and really constant. - pub fn new(leaves: Vec, mut pads: Vec) -> Self { + pub fn new(leaves: Vec, mut pads: Vec) -> Self { pads.sort_unstable_by_key(|run| run.start); - debug_assert!( - pads.iter().all(|run| run.start < run.end && run.end <= leaves.len()), - "a run must be a nonempty range of the leaves" - ); - debug_assert!( - pads.windows(2).all(|pair| pair[0].end <= pair[1].start), - "runs must be disjoint" - ); + debug_assert!(pads.iter().all(|run| run.start < run.end && run.end <= leaves.len())); + debug_assert!(pads.windows(2).all(|pair| pair[0].end <= pair[1].start)); debug_assert!( pads.iter() - .all(|run| leaves[run.start..run.end].iter().all(|&leaf| leaf == run.value)), - "a run must hold its declared value" + .all(|run| leaves[run.start..run.end].iter().all(|&leaf| leaf == run.value)) ); Self { leaves, pads } } - /// A leaf vector with no declared run: every row is folded explicitly. - pub fn dense(leaves: Vec) -> Self { + pub fn dense(leaves: Vec) -> Self { Self::new(leaves, Vec::new()) } } -/// One materialized product-tree level: its nodes, and the runs of them that are -/// a known constant. #[derive(Default)] struct Layer { - values: Vec, + values: Vec, pads: Vec, } /// Build only the levels consumed by radix four: `0,2,4,…`, plus a final -/// binary root when the logical depth is odd. A constant run of leaves is a -/// constant run of nodes at every level, its value raised to the level's arity, -/// which is what carries the runs up to the layer sumchecks. +/// binary root when the logical depth is odd. fn build_layers(leaves: LeafVector, mu: usize) -> Vec { assert!(!leaves.leaves.is_empty()); assert!(leaves.leaves.len() <= 1usize << mu); @@ -234,9 +193,7 @@ fn build_layers(leaves: LeafVector, mu: usize) -> Vec { ); left * right }; - // Four children of one value give the fourth power, so a run survives - // the level whenever four of its nodes share a parent. - let next_pads: Vec = pads + let next_pads = pads .iter() .filter_map(|run| { let square = run.value * run.value; @@ -248,20 +205,13 @@ fn build_layers(leaves: LeafVector, mu: usize) -> Vec { } else if current.len() == 2 { vec![current[0] * current[1]] } else if full_rows >= PAR_THRESHOLD { - // Every node is computed, runs included. Filling a run here would - // save its three multiplications and four loads, but a filled level - // has to be preallocated rather than collected, and the allocation's - // zero is then a second store on *every* node. Measured on a trace - // whose leaves are 41% padding, that loses: the level build is bound - // by its writes, not its multiplies. Skipping pays only where the - // arithmetic is dense, which is the layer sumcheck below. (0..full_rows).into_par_iter().map(product).collect() } else { (0..full_rows).map(product).collect() }; if current.len() % 4 != 0 && current.len() > 2 { let row = full_rows; - let child = |index| current.get(4 * row + index).copied().unwrap_or(F128::ONE); + let child = |index| current.get(4 * row + index).copied().unwrap_or(F192::ONE); let [left, right] = mul_pair([child(0), child(2)], [child(1), child(3)]); next.push(left * right); } @@ -285,7 +235,7 @@ fn build_layers(leaves: LeafVector, mu: usize) -> Vec { } #[inline] -fn quartic_summand(lines: [[F128; 2]; 4], equality: F128) -> [F256Unreduced; 4] { +fn quartic_summand(lines: [[F192; 2]; 4], equality: F192) -> [F192Unreduced; 4] { let [left0, left2, right0, right2] = mul_four( [lines[0][0], lines[0][1], lines[2][0], lines[2][1]], [lines[1][0], lines[1][1], lines[3][0], lines[3][1]], @@ -309,26 +259,22 @@ fn quartic_summand(lines: [[F128; 2]; 4], equality: F128) -> [F256Unreduced; 4] struct QuaternaryLayerState { /// Four child tables interleaved in their original order. This lets the /// prover consume a product-tree level without first transposing it. - values: Vec, - next: Vec, + values: Vec, + next: Vec, /// Logical row count after identity padding. `values` stores an arbitrary /// prefix; every omitted row is the constant four-tuple one. logical_rows: usize, - /// Rows of `values` all four of whose children hold one known value. They - /// are skipped by [`Self::round_message`] and refilled rather than folded. + /// Rows whose four children are one known value. pads: Vec, } impl QuaternaryLayerState { - fn new(mut values: Vec, pads: Vec, width: usize) -> Self { + fn new(mut values: Vec, pads: Vec, width: usize) -> Self { // Materialize only the incomplete final four-tuple. Every complete // all-one row after the arbitrary explicit prefix remains implicit. - values.resize(4 * values.len().max(1).div_ceil(4), F128::ONE); + values.resize(4 * values.len().max(1).div_ceil(4), F192::ONE); debug_assert_eq!(values.len() % 4, 0); debug_assert!(values.len() <= 4 * width); - // Four consecutive nodes are one row, whose four children then all hold - // the run's value. A row the run only partly covers falls to the dense - // path, as does the final row the resize above completed with ones. let pads = pads.iter().filter_map(|run| run.regroup(2, run.value)).collect(); Self { values, @@ -338,18 +284,15 @@ impl QuaternaryLayerState { } } - /// The runs after this round's fold: a pair of constant rows folds to one - /// row of the same value, `v + ρ(v+v) = v`. These are exactly the pairs - /// [`Self::round_message`] may skip. fn folded_pads(&self) -> Vec { self.pads.iter().filter_map(|run| run.regroup(1, run.value)).collect() } /// `(q(0)+q(1), [X²]q, [X³]q, [X⁴]q)`. - fn round_message(&self, equality: &[F128]) -> [F128; 4] { + fn round_message(&self, equality: &[F192]) -> [F192; 4] { let stored_rows = self.values.len() / 4; let full_pairs = stored_rows / 2; - let summand = |row: usize| -> [F256Unreduced; 4] { + let summand = |row: usize| -> [F192Unreduced; 4] { let (lo, hi) = (8 * row, 8 * row + 4); let lines = [0, 1, 2, 3].map(|child| { let at_zero = self.values[lo + child]; @@ -357,7 +300,7 @@ impl QuaternaryLayerState { }); quartic_summand(lines, equality[row]) }; - let xor = |mut left: [F256Unreduced; 4], right: [F256Unreduced; 4]| { + let xor = |mut left: [F192Unreduced; 4], right: [F192Unreduced; 4]| { for coefficient in 0..4 { left[coefficient] ^= right[coefficient]; } @@ -365,12 +308,10 @@ impl QuaternaryLayerState { }; let pads = self.folded_pads(); let rows = window_rows(full_pairs); - let window = |index: usize| -> [F256Unreduced; 4] { + let window = |index: usize| -> [F192Unreduced; 4] { let base = index * rows; - let mut acc = [F256Unreduced::ZERO; 4]; + let mut acc = [F192Unreduced::ZERO; 4]; for_each_piece(base..(base + rows).min(full_pairs), &pads, |piece, constant| { - // A constant pair is flat in the round variable, so it moves only - // `q(0)`, which the verifier derives from the incoming claim. if constant.is_none() { acc = piece.fold(acc, |sum, row| xor(sum, summand(row))); } @@ -382,47 +323,40 @@ impl QuaternaryLayerState { (0..windows) .into_par_iter() .map(window) - .reduce(|| [F256Unreduced::ZERO; 4], xor) + .reduce(|| [F192Unreduced::ZERO; 4], xor) } else { - (0..windows).map(window).fold([F256Unreduced::ZERO; 4], xor) + (0..windows).map(window).fold([F192Unreduced::ZERO; 4], xor) }; if stored_rows % 2 != 0 { let lo = 8 * full_pairs; let lines = [0, 1, 2, 3].map(|child| { let at_zero = self.values[lo + child]; - [at_zero, at_zero + F128::ONE] + [at_zero, at_zero + F192::ONE] }); message = xor(message, quartic_summand(lines, equality[full_pairs])); } - message.map(F256Unreduced::reduce) + message.map(F192Unreduced::reduce) } - fn fold(&mut self, challenge: F128) { + fn fold(&mut self, challenge: F192) { let stored_rows = self.values.len() / 4; let full_rows = stored_rows / 2; let rows = stored_rows.div_ceil(2); - self.next.resize(4 * rows, F128::ZERO); + self.next.resize(4 * rows, F192::ZERO); let folded = self.folded_pads(); let (values, next) = (&self.values, &mut self.next); - let fold_row = |row: usize, destination: &mut [F128]| { + let fold_row = |row: usize, destination: &mut [F192]| { let lo = 8 * row; let hi = lo + 4; - let [fold0, fold1] = mul_pair( - [values[lo] + values[hi], values[lo + 1] + values[hi + 1]], - [challenge; 2], + let folds = mul_four( + [0, 1, 2, 3].map(|child| values[lo + child] + values[hi + child]), + [challenge; 4], ); - let [fold2, fold3] = mul_pair( - [values[lo + 2] + values[hi + 2], values[lo + 3] + values[hi + 3]], - [challenge; 2], - ); - destination[0] = values[lo] + fold0; - destination[1] = values[lo + 1] + fold1; - destination[2] = values[lo + 2] + fold2; - destination[3] = values[lo + 3] + fold3; + for child in 0..4 { + destination[child] = values[lo + child] + folds[child]; + } }; - // A constant pair folds to its own value, so the run is refilled rather - // than folded, keeping the layer dense. - let window = |base: usize, destination: &mut [F128]| { + let window = |base: usize, destination: &mut [F192]| { for_each_piece(base..base + destination.len() / 4, &folded, |piece, constant| { let (lo, hi) = (4 * (piece.start - base), 4 * (piece.end - base)); match constant { @@ -447,24 +381,19 @@ impl QuaternaryLayerState { self.pads = folded; if stored_rows % 2 != 0 { let lo = 8 * full_rows; - let [fold0, fold1] = mul_pair( - [self.values[lo] + F128::ONE, self.values[lo + 1] + F128::ONE], - [challenge; 2], - ); - let [fold2, fold3] = mul_pair( - [self.values[lo + 2] + F128::ONE, self.values[lo + 3] + F128::ONE], - [challenge; 2], + let folds = mul_four( + [0, 1, 2, 3].map(|child| self.values[lo + child] + F192::ONE), + [challenge; 4], ); - self.next[4 * full_rows] = self.values[lo] + fold0; - self.next[4 * full_rows + 1] = self.values[lo + 1] + fold1; - self.next[4 * full_rows + 2] = self.values[lo + 2] + fold2; - self.next[4 * full_rows + 3] = self.values[lo + 3] + fold3; + for child in 0..4 { + self.next[4 * full_rows + child] = self.values[lo + child] + folds[child]; + } } std::mem::swap(&mut self.values, &mut self.next); self.logical_rows /= 2; } - fn children(&self) -> [F128; 4] { + fn children(&self) -> [F192; 4] { debug_assert_eq!(self.values.len(), 4); debug_assert_eq!(self.logical_rows, 1); self.values[..4].try_into().unwrap() @@ -474,9 +403,9 @@ impl QuaternaryLayerState { /// The result of a batched grand-product proof: the three roots and leaf /// evaluations, all reduced to one shared point. pub struct ProductTriple { - pub roots: [F128; 3], - pub point: Vec, - pub values: [F128; 3], + pub roots: [F192; 3], + pub point: Vec, + pub values: [F192; 3], } /// Prove three identity-padded grand products as one RLC-batched radix-four GKR. @@ -510,7 +439,7 @@ pub fn prove_product_triple(leaves: [LeafVector; 3], ps: &mut ProverState) -> Pr let below = &layers[tree][layer - 1].values; match below.as_slice() { [left, right] => [*left, *right], - [left] => [*left, F128::ONE], + [left] => [*left, F192::ONE], _ => unreachable!("the root's children have at most two explicit nodes"), } }); @@ -533,7 +462,7 @@ pub fn prove_product_triple(leaves: [LeafVector; 3], ps: &mut ProverState) -> Pr QuaternaryLayerState::new(values, pads, width) }); let mut equality = if round_count > 0 { - build_eq(&point[1..]) + eq_table(&point[1..]) } else { Vec::new() }; @@ -575,7 +504,7 @@ pub fn prove_product_triple(leaves: [LeafVector; 3], ps: &mut ProverState) -> Pr /// Verify the RLC-batched radix-four proof. pub fn verify_product_triple(mu: usize, vs: &mut VerifierState) -> Result { - let mut roots = [F128::ZERO; 3]; + let mut roots = [F192::ZERO; 3]; for root in &mut roots { *root = vs.next_scalar().map_err(|_| GkrError::Truncated)?; } @@ -589,7 +518,7 @@ pub fn verify_product_triple(mu: usize, vs: &mut VerifierState) -> Result Result Result F192 { + assert_eq!(table.len(), 1 << point.len()); + let mut folded = table.to_vec(); + for &challenge in point { + let half = folded.len() / 2; + for row in 0..half { + folded[row] = interp(folded[2 * row], folded[2 * row + 1], challenge); + } + folded.truncate(half); + } + folded[0] + } #[test] fn quartic_round_message_matches_direct_evaluation() { for width in [2, 4, 8, 16] { - let below: Vec = (0..4 * width) - .map(|i| F128::new((17 * i + width + 1) as u64, (i * i + 3) as u64)) + let below: Vec = (0..4 * width) + .map(|i| F192::new((17 * i + width + 1) as u64, (i * i + 3) as u64, (5 * i + 7) as u64)) .collect(); let state = QuaternaryLayerState::new(below, Vec::new(), width); - let equality: Vec = (0..width / 2) - .map(|i| F128::new((31 * i + 5) as u64, (7 * i + 1) as u64)) + let equality: Vec = (0..width / 2) + .map(|i| F192::new((31 * i + 5) as u64, (7 * i + 1) as u64, (11 * i + 9) as u64)) .collect(); let [difference, c2, c3, c4] = state.round_message(&equality); - let direct = |point: F128| { - (0..width / 2).fold(F128::ZERO, |sum, row| { + let direct = |point: F192| { + (0..width / 2).fold(F192::ZERO, |sum, row| { let values = [0, 1, 2, 3] .map(|child| interp(state.values[8 * row + child], state.values[8 * row + 4 + child], point)); sum + equality[row] * values[0] * values[1] * values[2] * values[3] }) }; - let c0 = direct(F128::ZERO); + let c0 = direct(F192::ZERO); let c1 = difference + c2 + c3 + c4; - for point in [ - F128::ZERO, - F128::ONE, - F128::generator(), - F128::generator() * F128::generator(), - ] { + for point in [F192::ZERO, F192::ONE, F192::Y, F192::Y.square()] { assert_eq!( c0 + point * (c1 + point * (c2 + point * (c3 + point * c4))), direct(point) @@ -690,19 +626,19 @@ mod tests { #[test] fn radix_four_roundtrip_at_even_and_odd_depths() { for mu in 0..=10 { - let leaves: [Vec; 3] = [0, 1, 2].map(|lane| { + let leaves: [Vec; 3] = [0, 1, 2].map(|lane| { (0..1usize << mu) - .map(|row| F128::new((1 + row + lane * 100_003) as u64, 0)) + .map(|row| F192::new((1 + row + lane * 100_003) as u64, row as u64, lane as u64)) .collect() }); let expected_roots = leaves .each_ref() - .map(|lane| lane.iter().copied().fold(F128::ONE, |product, value| product * value)); + .map(|lane| lane.iter().copied().fold(F192::ONE, |product, value| product * value)); let mut ps = ProverState::new(b"radix-four-gkr-test", &[]); let proved = prove_product_triple(leaves.clone().map(LeafVector::dense), &mut ps); assert_eq!(proved.roots, expected_roots); for lane in 0..3 { - assert_eq!(proved.values[lane], mle_eval(&leaves[lane], &proved.point)); + assert_eq!(proved.values[lane], mle_eval_e(&leaves[lane], &proved.point)); } let proof = ps.into_proof(); @@ -715,26 +651,25 @@ mod tests { } } - /// The bus's leaf layout: power-of-two blocks at aligned offsets, largest - /// first, each a real prefix followed by its own default-tuple constant, - /// then the identity tail. Returns the leaves and the runs to declare. - fn padded_blocks(mu: usize, lane: usize) -> (Vec, Vec) { + fn padded_blocks(mu: usize, lane: usize) -> (Vec, Vec) { let (half, quarter, eighth) = (1usize << (mu - 1), 1usize << (mu - 2), 1usize << (mu - 3)); - // (offset, rows, real): a mostly-real big block, a full one, and one - // that is almost all padding. let blocks = [ (0usize, half, half - 3 - lane), (half, quarter, quarter), (half + quarter, eighth, 1), ]; - let mut leaves = vec![F128::ONE; half + quarter + eighth]; + let mut leaves = vec![F192::ONE; half + quarter + eighth]; let mut pads = Vec::new(); for (index, &(off, rows, real)) in blocks.iter().enumerate() { for z in 0..real { - leaves[off + z] = F128::new((3 + off + z + 1_000 * lane) as u64, (z * z + 1) as u64); + leaves[off + z] = F192::new( + (3 + off + z + 1_000 * lane) as u64, + (z * z + 1) as u64, + (7 + z + lane) as u64, + ); } if real < rows { - let default = F128::new((17 + index + 31 * lane) as u64, (5 + index) as u64); + let default = F192::new((17 + index + 31 * lane) as u64, (5 + index) as u64, (11 + lane) as u64); leaves[off + real..off + rows].fill(default); pads.push(PadRun::new(off + real, off + rows, default)); } @@ -745,11 +680,7 @@ mod tests { #[test] fn declared_pad_runs_match_the_dense_prover() { for mu in 4..=11 { - let lanes: [(Vec, Vec); 3] = std::array::from_fn(|lane| padded_blocks(mu, lane)); - assert!( - lanes.iter().all(|(_, pads)| pads.len() == 2), - "the runs must be exercised" - ); + let lanes: [(Vec, Vec); 3] = std::array::from_fn(|lane| padded_blocks(mu, lane)); let mut skipped_ps = ProverState::new(b"pad-run-gkr-test", &[]); let skipped = prove_product_triple( @@ -762,8 +693,6 @@ mod tests { &mut dense_ps, ); - // Skipping a constant run moves no transmitted coefficient, so the - // two provers agree byte for byte. assert_eq!(skipped.roots, dense.roots); assert_eq!(skipped.point, dense.point); assert_eq!(skipped.values, dense.values); @@ -772,11 +701,11 @@ mod tests { for (lane, (leaves, _)) in lanes.iter().enumerate() { let mut padded = leaves.clone(); - padded.resize(1 << mu, F128::ONE); - assert_eq!(skipped.values[lane], mle_eval(&padded, &skipped.point)); + padded.resize(1 << mu, F192::ONE); + assert_eq!(skipped.values[lane], mle_eval_e(&padded, &skipped.point)); assert_eq!( skipped.roots[lane], - padded.iter().copied().fold(F128::ONE, |product, value| product * value) + padded.iter().copied().fold(F192::ONE, |product, value| product * value) ); } let mut vs = VerifierState::new(b"pad-run-gkr-test", &proof, &[]); @@ -790,26 +719,26 @@ mod tests { fn implicit_identity_suffix_matches_dense_padding() { for mu in 3..=10 { let lengths = [(1usize << mu) - 3, (1usize << (mu - 1)) + 1, (1usize << (mu - 2)) + 3]; - let leaves: [Vec; 3] = std::array::from_fn(|lane| { + let leaves: [Vec; 3] = std::array::from_fn(|lane| { (0..lengths[lane]) - .map(|row| F128::new((3 + row + lane * 10_007) as u64, 0)) + .map(|row| F192::new((3 + row + lane * 10_007) as u64, row as u64, lane as u64)) .collect() }); let dense = leaves.each_ref().map(|lane| { let mut padded = lane.clone(); - padded.resize(1 << mu, F128::ONE); + padded.resize(1 << mu, F192::ONE); padded }); let mut sparse_ps = ProverState::new(b"sparse-radix-four-gkr-test", &[]); let proved = prove_product_triple(leaves.map(LeafVector::dense), &mut sparse_ps); for lane in 0..3 { - assert_eq!(proved.values[lane], mle_eval(&dense[lane], &proved.point)); + assert_eq!(proved.values[lane], mle_eval_e(&dense[lane], &proved.point)); assert_eq!( proved.roots[lane], dense[lane] .iter() .copied() - .fold(F128::ONE, |product, value| product * value) + .fold(F192::ONE, |product, value| product * value) ); } let proof = sparse_ps.into_proof(); diff --git a/crates/lean_vm/src/leaf.rs b/crates/lean_vm/src/leaf.rs index fef7d92b..7a3f32ef 100644 --- a/crates/lean_vm/src/leaf.rs +++ b/crates/lean_vm/src/leaf.rs @@ -3,30 +3,33 @@ //! direction; the bus balances when pushed and pulled tuples form the same //! multiset, proven by two GKR passes over the leaf vectors `γ − π_α(σ)`. Each pass //! reduces to a leaf claim `Ṽ₀(ζ)`, decomposed into evaluation claims on the -//! committed columns. +//! committed columns. Tuple coordinates `σ_i` are `K`-valued (column entries, +//! g-powers, separators); the fingerprint challenges `α, γ` are `E`-valued, so a +//! leaf accumulates via the mixed `mul_base` product (2 PMULL per coordinate). use crate::PAR_THRESHOLD; -use primitives::field::{F128, G, index_mle}; use crate::gkr; -use primitives::multilinear::{eq_eval, mle_eval}; use crate::transcript::{ProverState, VerifierState}; use crate::witness::Column; +use primitives::field::{F64, F192, F192BaseUnreduced, g_pow, index_mle}; +use primitives::multilinear::{eq_eval, mle_eval}; use rayon::prelude::*; /// One tuple coordinate as a function of the block's row `z`. #[derive(Clone, Debug)] pub enum Coord { /// A public constant (domain separator, opcode, the seed count `1`). - Const(F128), + Const(F64), /// A committed column, value `col[z]`. Col(usize), - /// The free increment `g · col[z]` (a virtual column, §1). - GCol(usize), + /// The free increment `g^k · col[z]` (a virtual column, §1): `k = 1` for the + /// count/state steps, `k ∈ {1,2,3}` for BLAKE3's consecutive-word successors. + GCol(usize, u32), /// The index column `g^z` (§5.3), free via the factored MLE. Index, /// A public column (the bytecode program, §8): not committed; both parties form /// its MLE directly, so it raises no claim. - Public(Vec), + Public(Vec), } /// A flushing rule: `2^kappa` rows, each a tuple of coordinates. `real` is the @@ -51,8 +54,8 @@ pub struct Layout { #[derive(Clone, Debug, PartialEq, Eq)] pub struct ColumnClaim { pub col: usize, - pub point: Vec, - pub value: F128, + pub point: Vec, + pub value: F192, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -62,43 +65,50 @@ pub enum Error { /// A read count is zero, so a read self-cancels on the bus (§sec:memchan). ZeroCount, Gkr(gkr::GkrError), - /// The bus grinding nonce (before the multiset challenge γ) failed its PoW. - PowFailed, } -/// Proof-of-work bits to grind before the multiset challenge γ, so the bus -/// grand-product phase clears [`crate::SECURITY_BITS`]. Two Schwartz–Zippel -/// failure events share this randomness; union-bound over them: -/// -/// - the push/pull **balance** `push_root · d_pull = pull_root · d_push` — one -/// identity in γ whose difference has degree `max(push factors, pull -/// factors)` (the larger of the two sides; within a side the default-padding -/// factors are a single high-multiplicity root, so it is `max` not sum); -/// - the **count** channel `count_root ≠ 0` — a *separate* grand product of -/// `count factors`. -/// -/// So `N = max(2^push_mu, 2^pull_mu) + 2^count_mu`, and a false phase passes a -/// random challenge with probability ≤ `N / 2^128`, i.e. `128 − log2(N)` bits. -/// Two structural facts collapse this. The push and pull sides emit their bus -/// blocks in matched pairs — every [`FlushBuilder`] call appends one block to -/// each side with equal `κ`, and the three framework blocks (boundary, memory, -/// bytecode) are paired the same way — so the two sides have identical -/// `κ`-multisets and `push_mu == pull_mu`. And each count column is the count -/// coordinate of exactly one bytecode/memory flush while the state flush -/// carries none, so the count side sums strictly fewer `2^κ` than push and -/// `count_mu ≤ push_mu`. Hence `N = 2^push_mu + 2^count_mu` with -/// `count_mu ≤ push_mu`, so `⌈log2 N⌉ = push_mu + 1` exactly and the grind is -/// simply `SECURITY_BITS + push_mu + 1 − 128`. Grinding adds that deficit back -/// (the prover must redo the PoW to re-roll γ). -/// -/// The fingerprint challenge α is sampled AFTER the grind, so re-rolling it -/// also costs the PoW (besides the older argument that a fresh commitment to -/// re-roll α already costs `≥ 2^MIN_MU` Merkle hashes, above the target for -/// every admitted witness size). -fn grand_product_grinding_bits(push: &Layout, pull: &Layout, count: &Layout) -> u32 { - assert_eq!(push.mu, pull.mu, "push/pull bus blocks are paired, so their layouts match"); +/// Conservative sum of the degree bounds for every random-challenge failure in +/// the bus argument. A side contains at most `2^mu` leaf factors; multiplying by +/// the other side's padding surplus can double that count. Each factor has total +/// `(alpha, gamma)` degree at most the tuple width. The second term covers all +/// radix-four GKR batching and sumcheck challenges. +fn soundness_degree_bound(mu: usize, tuple_width: usize) -> u128 { + assert!(mu < u128::BITS as usize, "bus layout is too large to bound"); + let fingerprint = 2u128 * tuple_width as u128 * (1u128 << mu); + let gkr = 8u128 * (mu as u128 + 1).pow(2); + fingerprint + gkr +} + +fn soundness_bits(mu: usize, tuple_width: usize) -> u32 { + let degree = soundness_degree_bound(mu, tuple_width); + 192u32.saturating_sub(u128::BITS - degree.leading_zeros()) +} + +/// Check that the 192-bit challenge field supplies the target bus soundness. +/// Push and pull have the same logical height, and the count product is checked +/// by the same GKR rather than by a separate root-at-random test. +fn assert_grinding_unnecessary( + push_blocks: &[Block], + pull_blocks: &[Block], + push: &Layout, + pull: &Layout, + count: &Layout, +) { + assert_eq!( + push.mu, pull.mu, + "push/pull bus blocks are paired, so their layouts match" + ); assert!(count.mu <= push.mu, "count sums fewer bus messages than push"); - (crate::SECURITY_BITS + push.mu as u32 + 1).saturating_sub(128) + let tuple_width = push_blocks + .iter() + .chain(pull_blocks) + .map(|block| block.coords.len()) + .max() + .unwrap_or(0); + assert!( + soundness_bits(push.mu, tuple_width) >= crate::SECURITY_BITS, + "bus layout exceeds the unground F192 soundness budget" + ); } /// Stack blocks largest-first at aligned offsets; `μ = ⌈log2 Σ 2^{κ_b}⌉`. @@ -117,24 +127,19 @@ pub fn layout(blocks: &[Block]) -> Layout { } /// A non-constant coordinate as `(source, coefficient)`: its leaf contribution is -/// `coeff · source(z)`. `GCol` folds the `g` factor into the coefficient. +/// the mixed product `coeff · source(z)` with `source(z) ∈ K`, `coeff ∈ E`. +/// `GCol` folds the `g^k` factor into the coefficient. enum Term<'a> { - Col(usize, F128), - Index(F128), - Public(&'a [F128], F128), + Col(usize, F192), + Index(F192), + Public(&'a [F64], F192), } /// Build one side's leaf vector: block `b` row `z` holds `γ − Σ_i α^i c_i(z)`, /// followed implicitly by the identity `1` up to `2^μ`. The row-invariant /// `α`-power chain and constant coordinates are folded once per block into /// `const_part`. -/// -/// A block's padding rows are one fixed default (§e2e-pad), so its whole padded -/// suffix is a single leaf value: it is computed once and declared to the GKR as -/// a [`gkr::PadRun`], which the product tree then skips instead of folding -/// (§sec:gkr). The leaves themselves stay dense, so the statement proven is the -/// padded one and the verifier still divides the default surplus out. -pub fn build_leaves(blocks: &[Block], lay: &Layout, cols: &[Column], alpha: F128, gamma: F128) -> gkr::LeafVector { +pub fn build_leaves(blocks: &[Block], lay: &Layout, cols: &[Column], alpha: F192, gamma: F192) -> gkr::LeafVector { let explicit = blocks .iter() .enumerate() @@ -142,34 +147,38 @@ pub fn build_leaves(blocks: &[Block], lay: &Layout, cols: &[Column], alpha: F128 .max() .unwrap_or(1); debug_assert!(explicit <= 1usize << lay.mu); - let mut leaves = vec![F128::ONE; explicit]; - let mut pads: Vec = Vec::new(); + let mut leaves = vec![F192::ONE; explicit]; + let mut pads = Vec::new(); let maxk = blocks.iter().map(|b| b.kappa).max().unwrap_or(0); let gpow = primitives::field::g_powers(1usize << maxk); for (b, blk) in blocks.iter().enumerate() { let mut const_part = gamma; let mut terms: Vec = Vec::with_capacity(blk.coords.len()); - let mut alpha_pow = F128::ONE; + let mut alpha_pow = F192::ONE; for c in &blk.coords { match c { - Coord::Const(v) => const_part += alpha_pow * *v, + Coord::Const(v) => const_part += alpha_pow.mul_base(*v), Coord::Col(i) => terms.push(Term::Col(*i, alpha_pow)), - Coord::GCol(i) => terms.push(Term::Col(*i, alpha_pow * G)), + Coord::GCol(i, k) => terms.push(Term::Col(*i, alpha_pow.mul_base(g_pow(*k as usize)))), Coord::Index => terms.push(Term::Index(alpha_pow)), Coord::Public(vals) => terms.push(Term::Public(vals, alpha_pow)), } alpha_pow *= alpha; } - let row = |z: usize| -> F128 { - let mut acc = const_part; + let row = |z: usize| -> F192 { + // The α-weighted coordinate sum defers its reductions: each mixed + // product contributes its three raw limb products (3 PMULL, no + // reduction tail), one combined reduction per row at the end — + // bit-identical to summing reduced `mul_base` terms. + let mut acc = F192BaseUnreduced::ZERO; for t in &terms { - acc += match t { - Term::Col(i, c) => *c * cols[*i][z], - Term::Index(c) => *c * gpow[z], - Term::Public(vals, c) => *c * vals[z], + acc ^= match t { + Term::Col(i, c) => c.mul_base_unreduced(cols[*i][z]), + Term::Index(c) => c.mul_base_unreduced(gpow[z]), + Term::Public(vals, c) => c.mul_base_unreduced(vals[z]), }; } - acc + const_part + acc.reduce() }; let off = lay.offsets[b]; let rows = 1usize << blk.kappa; @@ -203,21 +212,21 @@ pub fn build_leaves(blocks: &[Block], lay: &Layout, cols: &[Column], alpha: F128 /// is why those per-column claims no longer reach the PCS. #[derive(Clone, Debug)] pub struct BusForm { - pub coeffs: Vec, - pub constant: F128, + pub coeffs: Vec, + pub constant: F192, } impl BusForm { fn new(n_cols: usize) -> Self { Self { - coeffs: vec![F128::ZERO; n_cols], - constant: F128::ZERO, + coeffs: vec![F192::ZERO; n_cols], + constant: F192::ZERO, } } /// `Σ_c coeffs[c]·evals[c] + constant`: the form at one row, or at a point when /// `evals` are column evaluations there. - pub fn eval(&self, evals: &[F128]) -> F128 { + pub fn eval(&self, evals: &[F192]) -> F192 { self.coeffs .iter() .zip(evals) @@ -230,39 +239,27 @@ impl BusForm { /// claims as before, `fresh` supplying values not already in `claims`. Returns the /// framework blocks' contribution to `Ṽ₀(ζ)` plus the padding mass, so the caller /// can settle the side once the zerocheck has proven the tables' forms. -fn decompose_formula Result>( +fn decompose_formula Result>( blocks: &[Block], lay: &Layout, - zeta: &[F128], - alpha: F128, - gamma: F128, + zeta: &[F192], + alpha: F192, + gamma: F192, owners: &[Option<(usize, usize)>], forms: &mut [BusForm], claims: &mut Vec, mut fresh: F, -) -> Result { +) -> Result { assert_eq!(zeta.len(), lay.mu); - let mut col_val = |col, point: &[F128]| { - if let Some(claim) = claims.iter().find(|c| c.col == col && c.point == point) { - return Ok(claim.value); - } - let value = fresh(col, point)?; - claims.push(ColumnClaim { - col, - point: point.to_vec(), - value, - }); - Ok(value) - }; - let mut acc = F128::ZERO; - let mut sel_sum = F128::ZERO; + let mut acc = F192::ZERO; + let mut sel_sum = F192::ZERO; for (b, blk) in blocks.iter().enumerate() { let kappa = blk.kappa; let zeta_lo = &zeta[..kappa]; let zeta_hi = &zeta[kappa..]; let sel = lay.offsets[b] >> kappa; - let sel_bits: Vec = (0..(lay.mu - kappa)) - .map(|k| F128::new(((sel >> k) & 1) as u64, 0)) + let sel_bits: Vec = (0..(lay.mu - kappa)) + .map(|k| F192::new(((sel >> k) & 1) as u64, 0, 0)) .collect(); let eq_hi = eq_eval(&sel_bits, zeta_hi); sel_sum += eq_hi; @@ -272,12 +269,12 @@ fn decompose_formula Result>( if let Some((t, base)) = owners[b] { let form = &mut forms[t]; form.constant += eq_hi * gamma; - let mut alpha_pow = F128::ONE; + let mut alpha_pow = F192::ONE; for c in &blk.coords { match c { - Coord::Const(v) => form.constant += eq_hi * alpha_pow * *v, + Coord::Const(v) => form.constant += eq_hi * alpha_pow.mul_base(*v), Coord::Col(i) => form.coeffs[*i - base] += eq_hi * alpha_pow, - Coord::GCol(i) => form.coeffs[*i - base] += eq_hi * alpha_pow * G, + Coord::GCol(i, k) => form.coeffs[*i - base] += eq_hi * alpha_pow.mul_base(g_pow(*k as usize)), Coord::Index | Coord::Public(_) => { unreachable!("a table's bus block carries no virtual coordinate") } @@ -287,14 +284,39 @@ fn decompose_formula Result>( continue; } - let mut inner = F128::ZERO; - let mut alpha_pow = F128::ONE; + let mut inner = F192::ZERO; + let mut alpha_pow = F192::ONE; for c in &blk.coords { let coord_val = match c { - Coord::Const(v) => *v, + Coord::Const(v) => F192::from(*v), Coord::Index => index_mle(zeta_lo), - Coord::Col(i) => col_val(*i, zeta_lo)?, - Coord::GCol(i) => G * col_val(*i, zeta_lo)?, + Coord::Col(i) => { + if let Some(v) = known_claim(claims, *i, zeta_lo) { + v + } else { + let v = fresh(*i, zeta_lo)?; + claims.push(ColumnClaim { + col: *i, + point: zeta_lo.to_vec(), + value: v, + }); + v + } + } + Coord::GCol(i, k) => { + let v = if let Some(v) = known_claim(claims, *i, zeta_lo) { + v + } else { + let v = fresh(*i, zeta_lo)?; + claims.push(ColumnClaim { + col: *i, + point: zeta_lo.to_vec(), + value: v, + }); + v + }; + v.mul_base(g_pow(*k as usize)) + } Coord::Public(vals) => mle_eval(vals, zeta_lo), }; inner += alpha_pow * coord_val; @@ -303,55 +325,113 @@ fn decompose_formula Result>( acc += eq_hi * (gamma + inner); } // The padding rows (identity `1`) contribute the leftover mass `1 - Σ_b sel_b`. - Ok(acc + (F128::ONE + sel_sum)) + Ok(acc + (F192::ONE + sel_sum)) +} + +/// Look up an already-recorded claim on `(col, point)`. Push and pull share +/// their GKR point, so a column read by both sides (or by two same-κ blocks of +/// one side) is streamed and opened ONCE; later occurrences reuse the value. +fn known_claim(claims: &[ColumnClaim], col: usize, point: &[F192]) -> Option { + claims + .iter() + .find(|c| c.col == col && c.point == point) + .map(|c| c.value) } /// Prover-side decomposition: reads the real columns, writing each FRESH /// committed value onto the stream and recording the matching claim /// (block/coord order); duplicates reuse the recorded value. +/// +/// The fresh column MLE evaluations run in a parallel first pass: within one +/// `decompose_formula` call no challenge is sampled between claims (`zeta`, +/// `alpha`, `gamma` are fixed arguments and each claim's point is +/// `zeta[..kappa]` of its block), so the values are independent of the +/// transcript and only their `add_scalar` ORDER matters. The second pass +/// replays them through the transcript in the original block/coord order, +/// keeping the stream byte-identical to the serial form. #[allow(clippy::too_many_arguments)] // the shared dedup + form context fn decompose_prove( blocks: &[Block], lay: &Layout, cols: &[Column], - zeta: &[F128], - alpha: F128, - gamma: F128, + zeta: &[F192], + alpha: F192, + gamma: F192, owners: &[Option<(usize, usize)>], forms: &mut [BusForm], claims: &mut Vec, ps: &mut ProverState, -) -> F128 { - decompose_formula(blocks, lay, zeta, alpha, gamma, owners, forms, claims, |col, zeta_lo| { - let v = mle_eval(&cols[col], zeta_lo); - ps.add_scalar(v); - Ok(v) - }) +) -> F192 { + // Pass 1: enumerate the FRESH committed coords exactly as `decompose_formula` + // visits them (blocks in order, coords in order, Col/GCol only, first + // occurrence per `(col, point)` — the same dedup as `known_claim`), then + // evaluate the column MLEs in parallel. + let mut jobs: Vec<(usize, usize)> = Vec::new(); + for (b, blk) in blocks.iter().enumerate() { + if owners[b].is_some() { + continue; + } + for c in &blk.coords { + if let Coord::Col(i) | Coord::GCol(i, _) = c { + let fresh = known_claim(claims, *i, &zeta[..blk.kappa]).is_none() && !jobs.contains(&(*i, blk.kappa)); + if fresh { + jobs.push((*i, blk.kappa)); + } + } + } + } + let vals: Vec = jobs + .par_iter() + .map(|&(col, kappa)| mle_eval(&cols[col], &zeta[..kappa])) + .collect(); + + // Pass 2: replay in the original order; duplicates reuse the recorded claim. + let mut fresh_iter = jobs.iter().zip(vals.iter()); + decompose_formula( + blocks, + lay, + zeta, + alpha, + gamma, + owners, + forms, + claims, + |col, zeta_lo| { + let (&(jc, jk), &v) = fresh_iter + .next() + .expect("job enumeration matches decompose_formula's col_val order"); + debug_assert_eq!((jc, jk), (col, zeta_lo.len()), "job/coord order drift"); + debug_assert_eq!(v, mle_eval(&cols[col], zeta_lo), "job/coord order drift"); + ps.add_scalar(v); + Ok(v) + }, + ) .expect("prover decomposition is infallible") } /// Verifier-side decomposition: reads each FRESH committed value from the /// stream (duplicates reuse the recorded claim), recomputes `Ṽ₀(ζ)`, and -/// records the fresh claims in one block/coordinate scan. +/// records the fresh claims. A pre-pass mirrors the formula's block/coord scan +/// so the stream reads stay sequential. fn decompose_verify( blocks: &[Block], lay: &Layout, - zeta: &[F128], - alpha: F128, - gamma: F128, + zeta: &[F192], + alpha: F192, + gamma: F192, owners: &[Option<(usize, usize)>], forms: &mut [BusForm], claims: &mut Vec, vs: &mut VerifierState, -) -> Result { +) -> Result { decompose_formula(blocks, lay, zeta, alpha, gamma, owners, forms, claims, |_, _| { vs.next_scalar().map_err(|_| Error::Truncated) }) } /// `base^e` by repeated squaring. -fn fpow(base: F128, mut e: usize) -> F128 { - let (mut r, mut b) = (F128::ONE, base); +fn fpow(base: F192, mut e: usize) -> F192 { + let (mut r, mut b) = (F192::ONE, base); while e > 0 { if e & 1 == 1 { r *= b; @@ -364,17 +444,17 @@ fn fpow(base: F128, mut e: usize) -> F128 { /// `π_α` of a block's padding-row tuple (every column zero but the read counts, /// value `1`). Only padded blocks are queried, and those carry only `Const`/`Col`/`GCol`. -fn default_fingerprint(block: &Block, pad: &[F128], alpha: F128) -> F128 { - let mut fingerprint = F128::ZERO; - let mut alpha_pow = F128::ONE; +fn default_fingerprint(block: &Block, pad: &[F64], alpha: F192) -> F192 { + let mut fingerprint = F192::ZERO; + let mut alpha_pow = F192::ONE; for c in &block.coords { let coord_val = match c { Coord::Const(v) => *v, Coord::Col(i) => pad[*i], - Coord::GCol(i) => G * pad[*i], - Coord::Index | Coord::Public(_) => F128::ZERO, + Coord::GCol(i, k) => g_pow(*k as usize) * pad[*i], + Coord::Index | Coord::Public(_) => F64::ZERO, }; - fingerprint += alpha_pow * coord_val; + fingerprint += alpha_pow.mul_base(coord_val); alpha_pow *= alpha; } fingerprint @@ -382,8 +462,8 @@ fn default_fingerprint(block: &Block, pad: &[F128], alpha: F128) -> F128 { /// The default-padding surplus on one side: `∏_b (γ − π_α(default_b))^{2^{κ_b} − /// real_b}`. The verifier divides it out before comparing the two sides (§sec:gp). -fn default_surplus(blocks: &[Block], pad: &[F128], alpha: F128, gamma: F128) -> F128 { - let mut acc = F128::ONE; +fn default_surplus(blocks: &[Block], pad: &[F64], alpha: F192, gamma: F192) -> F192 { + let mut acc = F192::ONE; for b in blocks { let delta = (1usize << b.kappa) - b.real; if delta != 0 { @@ -393,26 +473,26 @@ fn default_surplus(blocks: &[Block], pad: &[F128], alpha: F128, gamma: F128) -> acc } -/// One reduced claim on the bytecode polynomial. The eight public encoding -/// columns (opcode plus seven operand/immediate slots), stacked along three selector bits, form -/// ONE multilinear polynomial B̃ in `κ_bc + 3` variables; after the -/// decompositions both parties absorb the eight per-column evaluations (push and -/// pull share the GKR point ζ, so the columns are evaluated once), sample -/// three selector challenges `s`, and reduce the eight values to +/// One reduced claim on the bytecode polynomial. The nine public encoding +/// columns (opcode plus eight operand/immediate slots), padded to sixteen slots +/// along four selector bits, form one multilinear polynomial B̃ in `κ_bc + 4` +/// variables. After decomposition both parties absorb the nine column +/// evaluations (push and pull share the GKR point ζ), sample four selector +/// challenges `s`, and reduce them to /// `B̃(ζ_lo, s) = Σ_c eq(s, c)·v_c`. Natively the claim is /// true by construction (the verifier evaluated the columns itself); a /// recursive verifier defers exactly this one claim to its public input. #[derive(Clone, Debug)] pub struct BytecodeClaim { - /// `ζ_side_lo ++ s` — a point in `κ_bc + 3` variables. - pub point: Vec, + /// `ζ_side_lo ++ s` — a point in `κ_bc + 4` variables. + pub point: Vec, /// `B̃(point)`. - pub value: F128, + pub value: F192, } /// The public (bytecode) coordinate evaluations of a side at its GKR point, /// block/coord order, with the bytecode block's `κ`. -pub fn public_evals(blocks: &[Block], zeta: &[F128]) -> (usize, Vec) { +pub fn public_evals(blocks: &[Block], zeta: &[F192]) -> (usize, Vec) { let mut kappa = 0; let mut out = Vec::new(); for blk in blocks { @@ -426,13 +506,12 @@ pub fn public_evals(blocks: &[Block], zeta: &[F128]) -> (usize, Vec) { (kappa, out) } -/// The stacked bytecode polynomial as a dense table: the eight public encoding -/// columns along three selector bits (`B̃`'s evaluations on the cube). This is -/// the polynomial [`BytecodeClaim`]s are claims about; the outermost native -/// verifier evaluates it here. -pub fn stacked_bytecode_table(blocks: &[Block]) -> Vec { +/// The stacked bytecode polynomial as a dense table: nine public encoding +/// columns padded to sixteen selector slots. This is the polynomial +/// [`BytecodeClaim`]s are claims about; the outermost verifier evaluates it. +pub fn stacked_bytecode_table(blocks: &[Block]) -> Vec { let mut kbc = 0; - let mut cols: Vec<&Vec> = Vec::new(); + let mut cols: Vec<&Vec> = Vec::new(); for blk in blocks { for c in &blk.coords { if let Coord::Public(vals) = c { @@ -441,7 +520,8 @@ pub fn stacked_bytecode_table(blocks: &[Block]) -> Vec { } } } - let mut table = vec![F128::ZERO; 8 << kbc]; + assert!(cols.len() <= 1 << N_BYTECODE_SELECTORS); + let mut table = vec![F64::ZERO; 1 << (N_BYTECODE_SELECTORS + kbc)]; for (c_idx, vals) in cols.iter().enumerate() { assert_eq!(vals.len(), 1 << kbc); table[(c_idx << kbc)..((c_idx + 1) << kbc)].copy_from_slice(vals); @@ -450,13 +530,19 @@ pub fn stacked_bytecode_table(blocks: &[Block]) -> Vec { } /// `Σ_c eq(s, c)·v_c`: one side's public-column evaluations reduced to the +/// Selector bits of the stacked bytecode polynomial: the public encoding +/// columns (opcode + eight operand/immediate slots = nine) stack along +/// `2^N_BYTECODE_SELECTORS` slots. +pub const N_BYTECODE_SELECTORS: usize = 4; + /// stacked-polynomial value at selector point `s`. -pub fn stacked_bytecode_value(evals: &[F128], s: &[F128; 3]) -> F128 { - let mut acc = F128::ZERO; +pub fn stacked_bytecode_value(evals: &[F192], s: &[F192]) -> F192 { + debug_assert_eq!(s.len(), N_BYTECODE_SELECTORS); + let mut acc = F192::ZERO; for (c, &v) in evals.iter().enumerate() { - let mut e = F128::ONE; + let mut e = F192::ONE; for (t, &st) in s.iter().enumerate() { - e *= if (c >> t) & 1 == 1 { st } else { F128::ONE + st }; + e *= if (c >> t) & 1 == 1 { st } else { F192::ONE + st }; } acc += e * v; } @@ -473,14 +559,14 @@ pub struct BusProof { pub claims: Vec, pub bytecode_claims: Vec, /// The GKR point ζ: the zerocheck reuses it, so no fresh point is sampled. - pub point: Vec, + pub point: Vec, /// `forms[side][table]`, in `[push, pull, count]` order. pub forms: [Vec; 3], /// `sigmas[side][table]`: each form's eq-weighted sum over its table's rows. /// Prover-side only. NOTHING here travels: the batch's target is the caller's /// derived `Σ_s η^·totals[s]`, and the shares serve only to build each round's /// waiting line, which rides inside the round polynomial. - pub sigmas: [Vec; 3], + pub sigmas: [Vec; 3], } pub fn prove_balance( @@ -495,9 +581,7 @@ pub fn prove_balance( let push_lay = layout(push); let pull_lay = layout(pull); let mut count_lay = layout(count); - // Grind FIRST, so the PoW covers both bus challenges α and γ - // ([`grand_product_grinding_bits`]): re-rolling either means redoing it. - ps.grind(grand_product_grinding_bits(&push_lay, &pull_lay, &count_lay)); + assert_grinding_unnecessary(push, pull, &push_lay, &pull_lay, &count_lay); let alpha = ps.sample(); // The GKR treats the count tree as identity-padded to the pair's depth, but // its prover keeps that all-one suffix implicit. Retain the smaller layout @@ -507,26 +591,36 @@ pub fn prove_balance( let gamma = ps.sample(); // Independent leaf vectors; build concurrently. The count channel's leaf is the // count itself (a single `Col`, `γ=0`, `α=1`), so its root is the product of all counts. + let prof = std::env::var("LEANVM_PROFILE").is_ok(); + let t0 = std::time::Instant::now(); let (push_leaves, (pull_leaves, count_leaves)) = rayon::join( || build_leaves(push, &push_lay, cols, alpha, gamma), || { rayon::join( || build_leaves(pull, &pull_lay, cols, alpha, gamma), - || build_leaves(count, &count_build_lay, cols, F128::ONE, F128::ZERO), + || build_leaves(count, &count_build_lay, cols, F192::ONE, F192::ZERO), ) }, ); + if prof { + eprintln!("[bus] leaves : {:>7.2} ms", t0.elapsed().as_secs_f64() * 1e3); + } + let t0 = std::time::Instant::now(); // All three trees run as ONE RLC-batched GKR (equal μ: push/pull match // block-for-block, count is padded), so every claim lands on ONE point ζ. let bus_gkr = gkr::prove_product_triple([push_leaves, pull_leaves, count_leaves], ps); + if prof { + eprintln!("[bus] gkr : {:>7.2} ms", t0.elapsed().as_secs_f64() * 1e3); + } + let t0 = std::time::Instant::now(); // Framework blocks keep their per-column claims (deduped: push/pull share ζ); // every table block becomes a linear form for the zerocheck instead. let mut claims: Vec = Vec::new(); - let sides: [(&[Block], &Layout, F128, F128); 3] = [ + let sides: [(&[Block], &Layout, F192, F192); 3] = [ (push, &push_lay, alpha, gamma), (pull, &pull_lay, alpha, gamma), - (count, &count_lay, F128::ONE, F128::ZERO), + (count, &count_lay, F192::ONE, F192::ZERO), ]; // Each table's columns at ζ[..τ], computed once and shared by the three sides // (a form is linear in them). Nothing here travels, neither the evaluations nor @@ -536,10 +630,20 @@ pub fn prove_balance( // solved to satisfy, and would settle nothing. let table_evals = tables_at(cols, tables, &bus_gkr.point); let mut forms = std::array::from_fn(|_| tables.iter().map(|&(_, n)| BusForm::new(n)).collect::>()); - let mut sigmas: [Vec; 3] = std::array::from_fn(|_| Vec::new()); + let mut sigmas: [Vec; 3] = std::array::from_fn(|_| Vec::new()); for (s, &(blocks, lay, a, g)) in sides.iter().enumerate() { - let framework = - decompose_prove(blocks, lay, cols, &bus_gkr.point, a, g, &owners[s], &mut forms[s], &mut claims, ps); + let framework = decompose_prove( + blocks, + lay, + cols, + &bus_gkr.point, + a, + g, + &owners[s], + &mut forms[s], + &mut claims, + ps, + ); sigmas[s] = forms[s].iter().zip(&table_evals).map(|(f, e)| f.eval(e)).collect(); // Completeness only: the verifier derives this identity rather than checking // it, so a mismatch here is a prover bug, not a rejection path. @@ -549,15 +653,18 @@ pub fn prove_balance( "side {s} must decompose into its leaf value" ); } + if prof { + eprintln!("[bus] decompose : {:>7.2} ms", t0.elapsed().as_secs_f64() * 1e3); + } // Bytecode = ONE polynomial, and push/pull now share the point ζ, so the - // eight public columns are opened ONCE: bind the evaluations, sample the + // nine public columns are opened ONCE: bind the evaluations, sample the // selector challenges, emit the single reduced claim. let (kbc, pv) = public_evals(push, &bus_gkr.point); for &v in &pv { ps.observe_scalar(v); } - let s = [ps.sample(), ps.sample(), ps.sample()]; + let s: Vec = (0..N_BYTECODE_SELECTORS).map(|_| ps.sample()).collect(); let bytecode_claims = vec![BytecodeClaim { point: [&bus_gkr.point[..kbc], &s[..]].concat(), value: stacked_bytecode_value(&pv, &s), @@ -573,15 +680,14 @@ pub fn prove_balance( /// Every table's committed columns at `ζ[..τ_t]`: one `eq` table per table, then an /// inner product per column. `tables[t] = (base, n_cols)` in the global schema. -fn tables_at(cols: &[Column], tables: &[(usize, usize)], zeta: &[F128]) -> Vec> { +fn tables_at(cols: &[Column], tables: &[(usize, usize)], zeta: &[F192]) -> Vec> { tables .iter() .map(|&(base, n_cols)| { let tau = crate::log2_strict_usize(cols[base].len()); - let eq = primitives::multilinear::build_eq(&zeta[..tau]); (0..n_cols) .into_par_iter() - .map(|c| primitives::multilinear::inner_product(&eq, &cols[base + c])) + .map(|c| mle_eval(&cols[base + c], &zeta[..tau])) .collect() }) .collect() @@ -594,14 +700,14 @@ fn tables_at(cols: &[Column], tables: &[(usize, usize)], zeta: &[F128]) -> Vec, pub bytecode_claims: Vec, - pub count_root: F128, + pub count_root: F192, /// The GKR point ζ, reused as the batched zerocheck's eq point. - pub point: Vec, + pub point: Vec, /// `forms[side][table]`, for the zerocheck to settle. pub forms: [Vec; 3], /// Per side, what the tables' blocks owe its leaf claim: `Ṽ₀(ζ)` less the /// framework blocks' decomposition. Derived here, pinned by the batch's target. - pub totals: [F128; 3], + pub totals: [F192; 3], } /// Verify the bus balances, oracle-free (the prover's committed values arrive on @@ -610,20 +716,15 @@ pub fn verify_balance( push: &[Block], pull: &[Block], count: &[Block], - pad: &[F128], + pad: &[F64], owners: &[Vec>; 3], tables: &[(usize, usize)], vs: &mut VerifierState, ) -> Result { - // Check the grinding nonce FIRST: the PoW covers both bus challenges - // α and γ (mirror of prove_balance). let push_lay = layout(push); let pull_lay = layout(pull); let mut count_lay = layout(count); - vs.grind_check(grand_product_grinding_bits(&push_lay, &pull_lay, &count_lay)).map_err(|e| match e { - crate::transcript::Error::PowFailed => Error::PowFailed, - _ => Error::Truncated, - })?; + assert_grinding_unnecessary(push, pull, &push_lay, &pull_lay, &count_lay); let alpha = vs.sample(); // The count tree is padded to the pair's depth (identity leaves), so all // three verify as ONE RLC-batched GKR at ONE shared point. @@ -633,7 +734,7 @@ pub fn verify_balance( let [push_root, pull_root, count_root] = bus_gkr.roots; // Every read count is nonzero iff this product is (§sec:memchan); a zero would // let a read self-cancel and free its value from memory. - if count_root == F128::ZERO { + if count_root == F192::ZERO { return Err(Error::ZeroCount); } // The two sides differ by the default-padding surplus; divide each out @@ -649,15 +750,24 @@ pub fn verify_balance( // here, the batch's target being what pins it, so no table column is opened at ζ. let mut claims: Vec = Vec::new(); let mut forms = std::array::from_fn(|_| tables.iter().map(|&(_, n)| BusForm::new(n)).collect::>()); - let sides: [(&[Block], &Layout, F128, F128); 3] = [ + let sides: [(&[Block], &Layout, F192, F192); 3] = [ (push, &push_lay, alpha, gamma), (pull, &pull_lay, alpha, gamma), - (count, &count_lay, F128::ONE, F128::ZERO), + (count, &count_lay, F192::ONE, F192::ZERO), ]; - let mut totals = [F128::ZERO; 3]; + let mut totals = [F192::ZERO; 3]; for (s, &(blocks, lay, a, g)) in sides.iter().enumerate() { - let framework = - decompose_verify(blocks, lay, &bus_gkr.point, a, g, &owners[s], &mut forms[s], &mut claims, vs)?; + let framework = decompose_verify( + blocks, + lay, + &bus_gkr.point, + a, + g, + &owners[s], + &mut forms[s], + &mut claims, + vs, + )?; // What the tables owe this side: DERIVED, never read. A transmitted total // would be a free variable in its own check and would settle nothing; the // caller instead pins these against the batch's target. @@ -671,7 +781,7 @@ pub fn verify_balance( for &v in &pv { vs.observe_scalar(v); } - let s = [vs.sample(), vs.sample(), vs.sample()]; + let s: Vec = (0..N_BYTECODE_SELECTORS).map(|_| vs.sample()).collect(); let bytecode_claims = vec![BytecodeClaim { point: [&bus_gkr.point[..kbc], &s[..]].concat(), value: stacked_bytecode_value(&pv, &s), @@ -685,3 +795,16 @@ pub fn verify_balance( totals, }) } + +#[cfg(test)] +mod tests { + use super::soundness_bits; + + #[test] + fn bus_soundness_accounts_for_tuple_width() { + assert!(soundness_bits(38, 12) >= crate::SECURITY_BITS); + assert!(soundness_bits(59, 12) >= crate::SECURITY_BITS); + assert!(soundness_bits(60, 12) < crate::SECURITY_BITS); + assert!(soundness_bits(58, 16) < soundness_bits(58, 1)); + } +} diff --git a/crates/lean_vm/src/lib.rs b/crates/lean_vm/src/lib.rs index 0e62c464..e0d8d925 100644 --- a/crates/lean_vm/src/lib.rs +++ b/crates/lean_vm/src/lib.rs @@ -1,18 +1,21 @@ -//! leanVM-b — arithmetization of a minimal binary-field zkVM (see `doc.tex`). +//! leanVM-b — arithmetization of a minimal zkVM (see `misc/doc.tex`). //! -//! Every machine value is an element of GF(2^128), and logical indices are powers -//! of a fixed generator `g`, so incrementing an index is a multiplication by `g` — -//! a free virtual operation needing no addition gadget. The witness is field-valued -//! and committed directly by a dense multilinear PCS (no bit-decomposition). +//! Machine words are `c0 + c1*y + c2*y² ∈ E = K[y]/(y³ + y + 1)`. +//! Addresses, pc/fp, read counters, and logical indices live in +//! `K = GF(2^64)`; indices are powers of a fixed generator `g`, so incrementing +//! one is a multiplication by `g`, a free virtual operation. Every physical +//! witness column is K-valued (an E-valued word is three K-lane columns) and is +//! committed directly by a dense multilinear PCS. Challenges and transcript +//! scalars live in `E = GF(2^192)`, leaving ample margin for 128-bit soundness. //! //! - [`transcript`] — the shared Fiat–Shamir transcript (re-exported from `fiat_shamir`). -//! - [`pcs`] — field-valued witness commitment via the stacked Ligerito (§3). -//! - [`witness`] — field-valued columns stacked into one committed witness. +//! - [`pcs`] — `K`-committed witness, `E`-opened, via the stacked Ligerito (§3). +//! - [`witness`] — `K`-valued columns stacked into one committed witness. //! - [`gkr`] — the grand product via GKR (§4.3), balancing the bus. //! - [`leaf`] — the shared bus: grand-product balance, decomposed to per-column claims (§4.2–§4.4, §5). -//! - [`constraints`] — ONE back-loaded batched zerocheck over all six tables' degree-2 -//! identities plus their three bus forms (§4.1). -//! - [`tables`] — the six instruction tables (columns, flushes, constraints). +//! - [`constraints`] — one back-loaded batched zerocheck over all seven tables' +//! degree-2 identities plus their three bus forms (§4.1). +//! - [`tables`] — the seven instruction tables (columns, flushes, constraints). //! - [`cpu`] — whole-program assembly, control flow, and the prove/verify entry points. //! - [`blake3_flock`] — the `BLAKE3` glue: flock's R1CS validity proof over the same commitment. //! - [`vmhash`]: VM-native hashing (one-block compression and standard BLAKE3 slice hashing). @@ -69,12 +72,10 @@ fn set_qos_user_interactive() { } } -/// Target soundness of the whole proof, in bits. Every round is designed to clear -/// this: the PCS runs Ligerito's one shipped configuration ([`pcs`]: rate 1/2, the -/// unique-decoding regime, 120-bit round-by-round), and the bus grand product grinds -/// up to it before its multiset challenge ([`leaf`]). Raising it means bumping BOTH -/// (a stronger configuration and more grinding). -pub const SECURITY_BITS: u32 = 120; +/// Target soundness of the whole proof, in bits. Every algebraic challenge is +/// sampled in F192, and the PCS derives a Ligerito configuration whose query, +/// proximity-gap, and OOD-binding terms each clear this target. +pub const SECURITY_BITS: u32 = 128; /// Below this many parallelizable items a pass runs serially: rayon's fan-out /// overhead is not worth it for small inputs. Shared by [`constraints`], [`gkr`], [`leaf`]. diff --git a/crates/lean_vm/src/pcs.rs b/crates/lean_vm/src/pcs.rs index 81b826de..a4924b05 100644 --- a/crates/lean_vm/src/pcs.rs +++ b/crates/lean_vm/src/pcs.rs @@ -1,194 +1,82 @@ -//! Witness commitment: an inner-product PCS over F_{2^128} (doc §3), reusing -//! flock's **Ligerito**. An opening proves `Σ_x q(x)·W(x) = C` against any -//! verifier-evaluable weight `W` (a point evaluation `q̂(r)` is `W = eq(r,·)`). A -//! batch of claims `q̂(point_j) = value_j` folds with a random `λ` into one weight -//! `W_λ = Σ_j λ^j eq(point_j,·)` and target `C_λ = Σ_j λ^j value_j`, opened in a -//! single Ligerito run — the verifier evaluates `W_λ` itself, so it never travels. +//! Witness commitment: an inner-product PCS committing over `K = F_{2^64}` and +//! opening over `E = F_{2^192}` (doc §3), reusing flock's **Ligerito**. An +//! opening proves `Σ_x q(x)·W(x) = C` against any verifier-evaluable `E`-valued +//! weight `W` (a point evaluation `q̂(r)` is `W = eq(r,·)`). A batch of claims +//! `q̂(point_j) = value_j` folds with random `γ`s into one weight and target, +//! opened in a single Ligerito run — the verifier evaluates the weight itself, +//! so it never travels. flock's ring-switched `q_pkd` claims join the same batch +//! ([`::pcs::stack_open`]). //! -//! Security: Ligerito's one shipped configuration — rate 1/2, unique-decoding -//! regime (list size 1, no out-of-domain binding), 120-bit round-by-round soundness -//! ([`::pcs::ligerito::SECURITY_BITS`]). +//! Security: the K configs use rate-1/2 Johnson list decoding with OOD binding +//! and 128-bit round-by-round soundness ([`::pcs::ligerito::SECURITY_BITS`]). +//! L0's opening claim supplies its binding evaluation; each deeper commitment +//! takes one explicit OOD sample. The base-field +//! commitment only shrinks the level-0 symbols to 8 bytes; every random +//! ingredient is sampled from `E` with the same error terms as before. -use primitives::field::F128; use crate::transcript::{ProverState, VerifierState}; - -use ::pcs::ligerito::{LigeritoConfig, LigeritoSecurityConfig}; -pub use ::pcs::{Commitment, PcsParams, ProverData, StackedOpeningSummary}; -use ::pcs::{StackClaim, open_batch_mixed_ligerito_stacked, verify_opening_batch_mixed_ligerito_stacked}; -use ::pcs::PaddingSpec; - -/// flock frames `commit` as `m = log2(len) + LOG_PACKING`; the message length is -/// `2^(m - LOG_PACKING)`, so for an F-valued witness of `2^μ` elements we set -/// `m = μ + LOG_PACKING`. -const LOG_PACKING: usize = 7; -/// Row-batch lanes `2^LOG_BATCH`: the Merkle leaf width (`2^LOG_BATCH` -/// F128/leaf) IS Ligerito's INITIAL folding factor — the L0 commit is reused, -/// so the two are one knob ([`::pcs::ligerito::INITIAL_K`]). Larger ⇒ far -/// fewer Merkle nodes to hash at the cost of fatter query openings. -const LOG_BATCH: usize = ::pcs::ligerito::INITIAL_FOLDING_FATOR; -/// L0 rate (doc §3) — the one knob [`::pcs::ligerito::LOG_INV_RATE_0`]. +use primitives::field::{F64, F192}; + +use ::pcs::ligerito::{Commitment, ProverData, commit as ligerito_commit, configs_for_rate}; +use ::pcs::ligerito::{ProverConfig, VerifierConfig}; +pub use ::pcs::stack_open::{ + BatchOpeningProof, RingSwitchClaim, RingSwitchOpen, RingSwitchVerify, StackClaim as SlotClaim, + StackedOpeningSummary, +}; +use ::pcs::stack_open::{open_batch_mixed_ligerito_stacked, verify_opening_batch_mixed_ligerito_stacked}; + +/// The bit-packing width of `q_pkd` (`2^6` bits per committed `F64` word); only +/// bookkeeping here, since the K configs take the witness log-size directly. +pub const LOG_PACKING: usize = ::pcs::pack::LOG_PACKING; +/// Row-batch lanes `2^LOG_BATCH`: the Merkle leaf width (`2^LOG_BATCH` F64 +/// = 512 bytes/leaf) IS Ligerito's INITIAL folding factor — the L0 commit is +/// reused, so the two are one knob ([`::pcs::ligerito::INITIAL_FOLDING_FACTOR`]). +/// Larger ⇒ far fewer Merkle nodes to hash at the cost of fatter query openings. +const LOG_BATCH: usize = ::pcs::ligerito::INITIAL_FOLDING_FACTOR; +/// Default L0 rate index. pub const LOG_INV_RATE: usize = ::pcs::ligerito::LOG_INV_RATE_0; -// The PCS and the bus grinding both target `SECURITY_BITS`; keep them in -// sync — a stronger PCS target without bumping the constant (or vice versa) -// would leave one round below the intended level. +// The PCS and the unground F192 bus argument both target `SECURITY_BITS`. const _: () = assert!(::pcs::ligerito::SECURITY_BITS == crate::SECURITY_BITS as usize); /// Minimum committed-witness log-size: Ligerito's level ladder needs every level's -/// block length to accommodate its query count under the unique-decoding -/// regime's query counts — feasible from `μ = 14` (flock `m = 21`); we set -/// `μ = 15` (`m = 22`) for a one-level margin. `witness::placements_of` -/// zero-pads smaller stacks up to this floor (512 KB — negligible; real -/// workloads are far above it). +/// block length to accommodate its Johnson-radius query count — feasible from +/// `μ = 14`; we set `μ = 15` for a +/// one-level margin. `witness::placements_of` zero-pads smaller stacks up to +/// this floor (256 KB of F64 — negligible; real workloads are far above it). pub const MIN_MU: usize = 15; -fn params_for(mu: usize) -> PcsParams { - assert!( - mu >= MIN_MU, - "witness must be ≥ 2^{MIN_MU} elements (padded by placements_of)" - ); - PcsParams { - m: mu + LOG_PACKING, - log_inv_rate: LOG_INV_RATE, - log_batch_size: LOG_BATCH, - } -} - -/// The Ligerito config for a `2^μ`-element witness, -/// derived from the security analysis and memoized per `μ` (the derivation is -/// a pure function of `m`, so both sides agree). -fn lig_config(mu: usize) -> std::sync::Arc { +/// The Ligerito (prover, verifier) config pair for a `2^μ`-word witness, +/// derived from the security analysis and memoized per `(μ, log_inv_rate)`. +fn lig_configs(mu: usize, log_inv_rate: usize) -> std::sync::Arc<(ProverConfig, VerifierConfig)> { use std::collections::HashMap; use std::sync::{Arc, Mutex, OnceLock}; - type Cache = Mutex>>; + type Cache = Mutex>>; static CACHE: OnceLock = OnceLock::new(); let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new())); let mut map = cache.lock().expect("ligerito config cache poisoned"); - Arc::clone(map.entry(mu).or_insert_with(|| { - let config = LigeritoSecurityConfig::derive_config(mu + LOG_PACKING) - .and_then(|sec| sec.to_config()) - .unwrap_or_else(|e| panic!("ligerito config for mu={mu}: {e}")); - Arc::new(config) + Arc::clone(map.entry((mu, log_inv_rate)).or_insert_with(|| { + assert!( + mu >= MIN_MU, + "witness must be ≥ 2^{MIN_MU} elements (padded by placements_of)" + ); + let pair = configs_for_rate(mu, log_inv_rate) + .unwrap_or_else(|e| panic!("ligerito config for mu={mu}, log_inv_rate={log_inv_rate}: {e}")); + Arc::new(pair) })) } -/// Rebuild the public [`Commitment`] (root + params) for a witness of `2^mu` -/// elements. The verifier reconstructs the params from `mu` exactly as `commit` -/// did, so the single stacked Ligerito opening (which discharges flock's -/// `(ab, c)` claims together with leanVM's point claims, §blake3_flock) verifies -/// against this same commitment. -pub fn commitment_from_root(root: [u8; 32], mu: usize) -> Commitment { - Commitment { - root, - params: params_for(mu), - } -} - -/// A committed field-valued witness plus the data needed to open it. The witness +/// A committed `K`-valued witness plus the data needed to open it. The witness /// itself is not retained (the caller still owns it and passes it back to /// [`open`]), so committing costs no extra full-trace copy. pub struct Committed { pub commitment: Commitment, /// Codeword + Merkle tree retained for opening. Public so the single stacked - /// Ligerito opening (which also discharges flock's `(ab, c)` claims over this - /// same commitment, §blake3_flock) can reuse it. + /// Ligerito opening (which also discharges flock's `(ab, c)` claims over + /// this same commitment, §blake3_flock) can reuse it. pub prover_data: ProverData, - /// `log2` of the witness length. + /// `log2` of the witness length in F64 words. pub mu: usize, -} - -/// An evaluation claim located in a sub-cube (column slot) of the witness: -/// `q̂(low_point, sel) = value`, where the slot occupies `[offset, offset + -/// 2^{low_point.len()})` and `sel = offset >> low_point.len()` are its (boolean) -/// high-bit selector coordinates. Because the selector is boolean, the claim's -/// weight `eq(point,·)` is supported only inside the slot, where it equals -/// `eq(low_point,·)` — so W_λ is built block-sparsely in O(Σ_j 2^{n_vars_j}) -/// rather than O(J·2^μ). -/// -/// A [`SlotClaim::Strided`] is a further-sparse special case for **boolean-selector -/// slots on a packed column** (a BLAKE3 value word inside `q_pkd`): the low -/// `stride_log` within-block coords are frozen to `slot`'s bits, so `eq` is nonzero -/// only at `offset + slot + j·2^stride_log` — folded in `O(2^{point.len()})` rather -/// than the full `O(2^{stride_log + point.len()})` block. Equivalent to a `Slot` -/// with `low_point = slot_bits ++ point`. -#[derive(Clone, Debug)] -pub enum SlotClaim { - Slot { - offset: usize, - low_point: Vec, - value: F128, - }, - Strided { - offset: usize, - slot: usize, - stride_log: usize, - point: Vec, - value: F128, - }, -} - -impl SlotClaim { - /// This claim as a borrowed flock [`StackClaim`] — `Strided` maps to the sparse - /// [`StackClaim::StridedSlot`], `Slot` to the dense [`StackClaim::Slot`]. - fn as_stack(&self) -> StackClaim<'_> { - match self { - SlotClaim::Slot { - offset, - low_point, - value, - } => StackClaim::Slot { - offset: *offset, - low_point, - value: *value, - }, - SlotClaim::Strided { - offset, - slot, - stride_log, - point, - value, - } => StackClaim::StridedSlot { - offset: *offset, - slot: *slot, - stride_log: *stride_log, - point, - value: *value, - }, - } - } -} - -/// A batch of **ring-switched** evaluation claims discharged in the SAME opening -/// as the plain [`SlotClaim`]s (prover side). Unlike a `SlotClaim` — a plain -/// `eq`-point evaluation of the committed stack — these are claims on a packed -/// sub-block `qpkd` produced at the univariate-skip/packed level (flock's BLAKE3 -/// R1CS validity `(ab, c)`), so they carry the ring-switch tensor front-end. -/// [`crate::blake3_flock`] builds this from its reduction's claims; [`open`] -/// slices `qpkd` from the committed stack and folds it into the one Ligerito. -pub struct RingSwitchOpen { - /// `qpkd`'s offset inside the committed stack. - pub offset: usize, - /// `log2` of `qpkd`'s length; the opener slices `qpkd = stack[offset .. - /// offset + 2^qpkd_vars]` (the committed sub-block, so no separate copy). - pub qpkd_vars: usize, - /// Per-claim `x_outer_full` (the multilinear tail of each quirky point). - pub x_outers: Vec>, - /// Per-claim optional precomputed ring-switch weight `s_hat_v`. - pub s_hat_v: Vec>>, - /// flock's padding spec for the ring-switch weight (`k_log`, `useful_bits`). - pub padding: PaddingSpec, -} - -/// Verifier counterpart of [`RingSwitchOpen`]: the recovered `(ab, c)` claims. -/// The Ligerito opening proof travels separately (read off the `openings` hint -/// channel by the caller and passed to [`verify`] directly). -pub struct RingSwitchVerify { - /// `qpkd`'s offset inside the committed stack. - pub offset: usize, - /// `log2` of `qpkd`'s length (flock's `m − LOG_PACKING`). - pub qpkd_vars: usize, - /// Per-claim value, univariate-skip coord, and `x_outer_full`. - pub values: Vec, - pub z_skips: Vec, - pub x_outers: Vec>, + /// L0 inverse-rate logarithm bound into the transcript before this commitment. + pub log_inv_rate: usize, } #[derive(Clone, Debug, PartialEq, Eq)] @@ -196,74 +84,81 @@ pub enum Error { Ligerito, } -/// Commit a field-valued witness of length `2^μ` (`μ ≥ MIN_MU`; smaller stacks -/// are zero-padded up by [`crate::witness::placements_of`]) and bind its root into -/// the transcript, before any challenge is sampled. The verifier reads it with -/// [`read_commitment`]. -pub fn commit(ps: &mut ProverState, witness: &[F128]) -> Committed { +/// Commit a `K`-valued witness of length `2^μ` (`μ ≥ MIN_MU`; smaller stacks +/// are zero-padded up by [`crate::witness::placements_of`]) and bind its root +/// into the transcript, before any challenge is sampled. The verifier reads it +/// with [`read_commitment`]. +pub fn commit(ps: &mut ProverState, witness: &[F64], log_inv_rate: usize) -> Committed { let n = witness.len(); let mu = crate::log2_strict_usize(n); - let params = params_for(mu); - let (commitment, prover_data) = ::pcs::commit(witness, ¶ms); - ps.add_scalars(&::pcs::merkle::hash_to_scalars(&commitment.root)); + assert!( + mu >= MIN_MU, + "witness must be ≥ 2^{MIN_MU} elements (padded by placements_of)" + ); + let (commitment, prover_data) = ligerito_commit(witness, LOG_BATCH, log_inv_rate); + ps.add_scalars(&root_to_scalars(&commitment.root)); Committed { commitment, prover_data, mu, + log_inv_rate, } } -// The folding scalar `λ` is just `sample()`d: every claim it combines is already -// bound — the values rode the stream (`add_scalar`) during the bus / constraint / -// public-input sub-protocols, the points are prior challenges, and the offsets are -// public (reconstructed identically from the announced layout). So `λ` sampled -// here is already bound to all of them; no re-observe is needed. +// The batching challenges are just `sample()`d inside the stacked opener: every +// claim they combine is already bound — the values rode the stream +// (`add_scalar`) during the bus / constraint / public-input sub-protocols, the +// points are prior challenges, and the offsets are public (reconstructed +// identically from the announced layout). + +/// A Merkle root (32 bytes) as two field scalars, so it travels the transcript +/// stream like any other transmitted value (leanVM parses its root the same way). +fn root_to_scalars(root: &[u8; 32]) -> [F192; 2] { + let w = |o: usize| u64::from_le_bytes(root[o..o + 8].try_into().unwrap()); + [F192::new(w(0), w(8), 0), F192::new(w(16), w(24), 0)] +} + +fn scalars_to_root(s: &[F192]) -> Result<[u8; 32], crate::transcript::Error> { + assert_eq!(s.len(), 2, "a Merkle root is exactly two field words"); + if s.iter().any(|word| word.c2 != 0) { + return Err(crate::transcript::Error::NonCanonicalEncoding); + } + let mut root = [0u8; 32]; + root[0..8].copy_from_slice(&s[0].c0.to_le_bytes()); + root[8..16].copy_from_slice(&s[0].c1.to_le_bytes()); + root[16..24].copy_from_slice(&s[1].c0.to_le_bytes()); + root[24..32].copy_from_slice(&s[1].c1.to_le_bytes()); + Ok(root) +} /// Verifier counterpart of [`commit`]'s root binding: read the committed root /// from the stream at the start of verification, before sampling any challenge. pub fn read_commitment(vs: &mut VerifierState) -> Result<[u8; 32], crate::transcript::Error> { let root_s = vs.next_scalars(2)?; - Ok(::pcs::merkle::scalars_to_hash(&root_s)) + scalars_to_root(&root_s) } /// Open the committed witness: discharge the `points` (leanVM's bus / constraint / -/// public-input / pin claims, as block-sparse slot evaluations) AND flock's -/// ring-switched BLAKE3 `(ab, c)` validity (`ring`) in ONE stacked Ligerito. The -/// points become the opener's `stack_pd`; the returned proof is placed on the -/// `openings` hint channel by the caller (`ps.hint_opening`), not on the scalar -/// stream. The commitment root was already bound by [`commit`], and the point -/// *values* rode the stream during their sub-protocols, so nothing extra is -/// bound here. +/// public-input claims, as block-sparse slot evaluations) AND flock's +/// ring-switched BLAKE3 `(ab, c)` validity (`ring`) in ONE stacked Ligerito. +/// The points become the opener's `point_claims`; the returned proof is placed +/// on the `openings` hint channel by the caller (`ps.hint_opening`), not on the +/// scalar stream. The commitment root was already bound by [`commit`], and the +/// point *values* rode the stream during their sub-protocols, so nothing extra +/// is bound here. /// /// There is no plain (non-ring-switch) path: the witness ALWAYS carries a `q_pkd` /// sub-block (≥ 1 padding instance, §cpu), so every opening is stacked. pub fn open( ps: &mut ProverState, c: &Committed, - q: &[F128], + q: &[F64], points: &[SlotClaim], ring: &RingSwitchOpen, -) -> ::pcs::ligerito::LigeritoProof { +) -> BatchOpeningProof { debug_assert_eq!(q.len(), 1usize << c.mu, "witness length must match the commitment"); - // The packed sub-block is exactly the committed slice — no separate copy. - let qpkd = &q[ring.offset..ring.offset + (1usize << ring.qpkd_vars)]; - let stack_pd: Vec = points.iter().map(|s| s.as_stack()).collect(); - let x_refs: Vec<&[F128]> = ring.x_outers.iter().map(|v| v.as_slice()).collect(); - let s_refs: Vec> = ring.s_hat_v.iter().map(|o| o.as_deref()).collect(); - let cfg = lig_config(c.mu); - open_batch_mixed_ligerito_stacked( - qpkd, - &x_refs, - &s_refs, - &ring.padding, - q, - ring.offset, - &c.prover_data, - &c.commitment, - &stack_pd, - &cfg, - ps, - ) + let cfg = lig_configs(c.mu, c.log_inv_rate); + open_batch_mixed_ligerito_stacked(ps.sponge_mut(), q, &c.prover_data, &cfg.0, points, ring) } /// Verify the opening (mirror of [`open`]): flock's ring-switched `(ab, c)` claims @@ -274,25 +169,12 @@ pub fn verify( vs: &mut VerifierState, points: &[SlotClaim], ring: &RingSwitchVerify, - open: &::pcs::ligerito::LigeritoProof, + open: &BatchOpeningProof, mu: usize, + log_inv_rate: usize, root: &[u8; 32], ) -> Result { - let commitment = commitment_from_root(*root, mu); - let stack_pd: Vec = points.iter().map(|s| s.as_stack()).collect(); - let x_refs: Vec<&[F128]> = ring.x_outers.iter().map(|v| v.as_slice()).collect(); - let cfg = lig_config(mu); - verify_opening_batch_mixed_ligerito_stacked( - &commitment, - ring.offset, - ring.qpkd_vars, - &ring.values, - &ring.z_skips, - &x_refs, - &stack_pd, - open, - &cfg, - vs, - ) - .map_err(|_| Error::Ligerito) + let cfg = lig_configs(mu, log_inv_rate); + verify_opening_batch_mixed_ligerito_stacked(vs.sponge_mut(), &cfg.1, mu, root, points, ring, open) + .ok_or(Error::Ligerito) } diff --git a/crates/lean_vm/src/tables.rs b/crates/lean_vm/src/tables.rs index 8c6f2722..3c55edd4 100644 --- a/crates/lean_vm/src/tables.rs +++ b/crates/lean_vm/src/tables.rs @@ -1,41 +1,58 @@ -//! Per-instruction tables (§7). Each opcode is one [`Table`] impl that declares, +//! Per-instruction tables (`misc/doc.tex`, “The instruction tables”). Each opcode is one [`Table`] impl that declares, //! in one place, its committed columns, how to fill them from the trace, its bus //! interactions (flushes), the read-count columns that feed the count channel, //! and its degree-2 constraint. Column indices here are *local* (`0..n_committed_columns`); //! `cpu`'s schema offsets them to global witness columns. +//! +//! Columns are `K`-valued (`F64`). Addresses, the pc/fp, operands, counts, +//! opcodes and separators are single `K`-columns; a **machine word** (memory +//! value) is 192-bit (`E = F192`), committed as THREE `K`-lane columns. A +//! constraint is evaluated at an `E`-point, so `eval_constraint` receives +//! `E`-values; a word is reassembled as `c0 + c1·y + c2·y²`, and value +//! relations (`XOR`, `MUL`, the `DEREF` store, +//! the `JUMP` selection) are written as `E`-relations — still degree 2 in the +//! lane columns. use rayon::prelude::*; use crate::cpu::Trace; -use primitives::field::{F128, G, mul_by_x}; use crate::leaf::Coord::{self, Col, Const, GCol}; use crate::witness::Column; +use primitives::field::{F64, F192, G, mul_by_g}; + +/// Reassemble a 192-bit machine word from its three `K`-limbs (as folded +/// `E`-column values). +#[inline] +fn e192(c0: F192, c1: F192, c2: F192) -> F192 { + c0 + F192::Y * (c1 + F192::Y * c2) +} // ---- shared bus vocabulary --------------------------------------------------- -/// `g^k` at compile time (`g = x`, so repeated `mul_by_x` from `g^0 = 1`). -const fn g_pow(k: usize) -> F128 { - let mut acc = F128::ONE; +/// `g^k` at compile time (`g = x`, so repeated `mul_by_g` from `g^0 = 1`). +const fn g_pow(k: usize) -> F64 { + let mut acc = F64::ONE; let mut i = 0; while i < k { - acc = mul_by_x(acc); + acc = mul_by_g(acc); i += 1; } acc } // Domain separators (coordinate 0 of every bus tuple): the g-powers g^0, g^1, g^2. -pub(crate) const SEP_STATE: F128 = g_pow(0); -pub(crate) const SEP_MEM: F128 = g_pow(1); -pub(crate) const SEP_BYTECODE: F128 = g_pow(2); - -// Opcodes (coordinate 3 of a bytecode tuple): the g-powers g^0..g^5. -pub(crate) const OP_XOR: F128 = g_pow(0); -pub(crate) const OP_MUL: F128 = g_pow(1); -pub(crate) const OP_SET: F128 = g_pow(2); -pub(crate) const OP_DEREF: F128 = g_pow(3); -pub(crate) const OP_JUMP: F128 = g_pow(4); -pub(crate) const OP_BLAKE3: F128 = g_pow(5); +pub(crate) const SEP_STATE: F64 = g_pow(0); +pub(crate) const SEP_MEM: F64 = g_pow(1); +pub(crate) const SEP_BYTECODE: F64 = g_pow(2); + +// Opcodes (coordinate 3 of a bytecode tuple). +pub(crate) const OP_XOR: F64 = g_pow(0); +pub(crate) const OP_MUL: F64 = g_pow(1); +pub(crate) const OP_SET: F64 = g_pow(2); +pub(crate) const OP_DEREF: F64 = g_pow(3); +pub(crate) const OP_JUMP: F64 = g_pow(4); +pub(crate) const OP_BLAKE3: F64 = g_pow(5); +pub(crate) const OP_PACK64X2: F64 = g_pow(6); // ---- flush builder ----------------------------------------------------------- @@ -64,7 +81,7 @@ impl FlushBuilder { /// Fall-through state step: the next pc is `g·pc`, fp unchanged. pub(crate) fn state_step(&mut self, pc: usize, fp: usize) { self.pair( - vec![Const(SEP_STATE), GCol(pc), Col(fp)], + vec![Const(SEP_STATE), GCol(pc, 1), Col(fp)], vec![Const(SEP_STATE), Col(pc), Col(fp)], ); } @@ -79,29 +96,99 @@ impl FlushBuilder { /// Bytecode read at `pc`: the program tuple (opcode + seven operand slots), /// with the per-pc execution count advanced by ×g on the push side. - pub(crate) fn bytecode(&mut self, pc: usize, count: usize, opcode: F128, operands: &[Coord]) { - let mut push = vec![Const(SEP_BYTECODE), Col(pc), GCol(count), Const(opcode)]; - let mut pull = vec![Const(SEP_BYTECODE), Col(pc), Col(count), Const(opcode)]; + pub(crate) fn bytecode(&mut self, pc: usize, count: usize, opcode: F64, operands: &[Coord]) { + self.bytecode_coord(pc, count, Const(opcode), operands); + } + + pub(crate) fn bytecode_coord(&mut self, pc: usize, count: usize, opcode: Coord, operands: &[Coord]) { + let mut push = vec![Const(SEP_BYTECODE), Col(pc), GCol(count, 1), opcode.clone()]; + let mut pull = vec![Const(SEP_BYTECODE), Col(pc), Col(count), opcode]; push.extend_from_slice(operands); pull.extend_from_slice(operands); self.pair(push, pull); } - /// Memory access: read `val` at `addr`, advancing the cell's access count by ×g. - pub(crate) fn memory(&mut self, addr: usize, count: usize, val: usize) { + /// Memory access: read the three-limb word at `addr`, advancing the cell's + /// access count by ×g. + pub(crate) fn memory(&mut self, addr: usize, count: usize, val0: usize, val1: usize, val2: usize) { + self.pair( + vec![ + Const(SEP_MEM), + Col(addr), + GCol(count, 1), + Col(val0), + Col(val1), + Col(val2), + ], + vec![Const(SEP_MEM), Col(addr), Col(count), Col(val0), Col(val1), Col(val2)], + ); + } + + /// Memory read of a K-valued word: both higher limbs are literal zero. Used for words the + /// constraints force into K (e.g. the DEREF pointer). Sound because the bus + /// balances only if the stored value's HI lane is likewise 0. + pub(crate) fn memory_k(&mut self, addr: usize, count: usize, val: usize) { self.pair( - vec![Const(SEP_MEM), Col(addr), GCol(count), Col(val)], - vec![Const(SEP_MEM), Col(addr), Col(count), Col(val)], + vec![ + Const(SEP_MEM), + Col(addr), + GCol(count, 1), + Col(val), + Const(F64::ZERO), + Const(F64::ZERO), + ], + vec![ + Const(SEP_MEM), + Col(addr), + Col(count), + Col(val), + Const(F64::ZERO), + Const(F64::ZERO), + ], ); } - /// Memory access at the free successor address `g·col[addr]` — the second of - /// two consecutive words (doc §7.6, `BLAKE3`). The address coordinate is the - /// virtual ×g of the committed base address, so no extra committed column. - pub(crate) fn memory_succ(&mut self, addr: usize, count: usize, val: usize) { + /// Memory access to a canonical 128-bit word `(lo, hi, 0)`. + pub(crate) fn memory_128(&mut self, addr: usize, count: usize, lo: usize, hi: usize) { self.pair( - vec![Const(SEP_MEM), GCol(addr), GCol(count), Col(val)], - vec![Const(SEP_MEM), GCol(addr), Col(count), Col(val)], + vec![ + Const(SEP_MEM), + Col(addr), + GCol(count, 1), + Col(lo), + Col(hi), + Const(F64::ZERO), + ], + vec![ + Const(SEP_MEM), + Col(addr), + Col(count), + Col(lo), + Col(hi), + Const(F64::ZERO), + ], + ); + } + + /// Memory access to the successor of `addr`, carrying `(lo, hi, 0)`. + pub(crate) fn memory_128_succ(&mut self, addr: usize, count: usize, lo: usize, hi: usize) { + self.pair( + vec![ + Const(SEP_MEM), + GCol(addr, 1), + GCol(count, 1), + Col(lo), + Col(hi), + Const(F64::ZERO), + ], + vec![ + Const(SEP_MEM), + GCol(addr, 1), + Col(count), + Col(lo), + Col(hi), + Const(F64::ZERO), + ], ); } } @@ -112,12 +199,12 @@ impl FlushBuilder { /// image (for read values), and `g^0..` for O(1) address/operand lookups. pub struct FillCtx<'a> { pub(crate) trace: &'a Trace, - pub(crate) mem: &'a [F128], - pub(crate) gpow: &'a [F128], + pub(crate) mem: &'a [F192], + pub(crate) gpow: &'a [F64], } impl FillCtx<'_> { - fn g_at(&self, i: u32) -> F128 { + fn g_at(&self, i: u32) -> F64 { self.gpow[i as usize] } } @@ -129,18 +216,18 @@ impl FillCtx<'_> { /// The batched zerocheck carries every committed column of a table, in local /// order, so this is a plain slice. pub struct Cols<'a> { - values: &'a [F128], + values: &'a [F192], } impl<'a> Cols<'a> { - pub(crate) fn new(values: &'a [F128]) -> Self { + pub(crate) fn new(values: &'a [F192]) -> Self { Self { values } } } impl std::ops::Index for Cols<'_> { - type Output = F128; - fn index(&self, local: usize) -> &F128 { + type Output = F192; + fn index(&self, local: usize) -> &F192 { &self.values[local] } } @@ -151,7 +238,7 @@ impl std::ops::Index for Cols<'_> { /// [`count_columns`](Table::count_columns) are local to this table. pub trait Table: Sync { /// Distinct opcode tag (coordinate 3 of the bytecode tuple). - fn opcode_tag(&self) -> F128; + fn opcode_tag(&self) -> F64; /// Number of committed columns (local indices `0..n_committed_columns`). fn n_committed_columns(&self) -> usize; /// Local indices of this table's read-count columns — the `g^{count}` values @@ -168,16 +255,18 @@ pub trait Table: Sync { /// is exactly [`n_constraints`](Table::n_constraints) long: an identity indexed /// past its end panics rather than silently reaching into the next table's /// range. Returns `0` on every valid row (§4.1). - fn eval_constraint(&self, pows: &[F128], cols: &Cols) -> F128; + fn eval_constraint(&self, pows: &[F192], cols: &Cols) -> F192; /// Declare the table's bus interactions. fn flushes(&self, f: &mut FlushBuilder); /// Fill this table's columns (`out[i]` is local column `i`) from the trace. fn fill(&self, ctx: &FillCtx, out: &mut [Column]); } -/// The six tables in fixed order `[XOR, MUL, SET, DEREF, JUMP, BLAKE3]` — the +/// The tables in fixed order `[XOR, MUL, SET, DEREF, JUMP, BLAKE3, PACK64X2]` — the /// order of `row_counts` / `taus` throughout `cpu`. -pub fn tables() -> [&'static dyn Table; 6] { +pub const N_TABLES: usize = 7; + +pub fn tables() -> [&'static dyn Table; N_TABLES] { [ &Arith { is_xor: true }, &Arith { is_xor: false }, @@ -185,31 +274,53 @@ pub fn tables() -> [&'static dyn Table; 6] { &DerefTable, &JumpTable, &Blake3Table, + &Pack64x2Table, ] } /// Index of the BLAKE3 table in [`tables`]. pub(crate) const BLAKE3_TABLE: usize = 5; -/// BLAKE3 virtual-column LOCAL indices in canonical slot order -/// `[a0, a1, b0, b1, c0, c1, cv0, cv1, metadata]` (matches -/// `blake3_flock::VM_SLOTS`). These columns are +/// BLAKE3 value-column LOCAL indices in canonical slot order +/// `[a0..a3, b0..b3, c0..c3, cv0..cv3, md_lo, md_hi]` (matches +/// `blake3_flock::SLOTS`). These columns are /// VIRTUAL (never committed): `q_pkd` already holds those words at fixed packed /// slots, so `cpu` routes their memory-bus evaluation claims straight to `q_pkd` /// (`slot_claims`) — the value the bus flushes IS the flock-proven word. -pub const BLAKE3_VALUE_COLS: [usize; 9] = [ +pub const BLAKE3_VALUE_COLS: [usize; 18] = [ blake3t::VA0, - blake3t::VA1, + blake3t::VA0 + 1, + blake3t::VA0 + 2, + blake3t::VA0 + 3, blake3t::VB0, - blake3t::VB1, + blake3t::VB0 + 1, + blake3t::VB0 + 2, + blake3t::VB0 + 3, blake3t::VC0, - blake3t::VC1, + blake3t::VC0 + 1, + blake3t::VC0 + 2, + blake3t::VC0 + 3, blake3t::VCV0, - blake3t::VCV1, - blake3t::METADATA, + blake3t::VCV0 + 1, + blake3t::VCV0 + 2, + blake3t::VCV0 + 3, + blake3t::MD0, + blake3t::MD1, ]; +// The eighteen value lanes are laid out contiguously (VA0..VA0+17), so they map +// 1:1 onto `blake3_flock::SLOTS`. +const _: () = assert!( + blake3t::VB0 == blake3t::VA0 + 4 + && blake3t::VC0 == blake3t::VA0 + 8 + && blake3t::VCV0 == blake3t::VA0 + 12 + && blake3t::MD0 == blake3t::VA0 + 16 + && blake3t::MD1 == blake3t::VA0 + 17 +); /// Declare consecutive local column indices and the resulting column count. +// Kept from main's table refactor as a tool for future single-lane column sets; +// this branch's tables use explicit LO/HI/TOP constants (192-bit memory words). +#[allow(unused_macros)] macro_rules! columns { ($($column:ident),+ $(,)?) => { columns!(@define 0; $($column),+); @@ -228,17 +339,40 @@ macro_rules! columns { /// `XOR` and `MUL_NATIVE` share their column layout, flushes, and fill; they /// differ only in the opcode tag and the third-operand identity (`vc = va + vb` -/// for `XOR`, `vc = va·vb` for `MUL`). +/// for `XOR`, `vc = va·vb` in `E = K[y]/(y³+y+1)` for `MUL`, degree 2 in the +/// committed K-lane columns). struct Arith { is_xor: bool, } mod arith { - columns!(PC, FP, OA, OB, OC, AA, AB, AC, VA, VB, VC, RA, RB, RC, RBC); + pub const PC: usize = 0; + pub const FP: usize = 1; + pub const OA: usize = 2; + pub const OB: usize = 3; + pub const OC: usize = 4; + pub const AA: usize = 5; + pub const AB: usize = 6; + pub const AC: usize = 7; + // The three read words, each three K-limbs. + pub const VA_LO: usize = 8; + pub const VA_HI: usize = 9; + pub const VA_TOP: usize = 10; + pub const VB_LO: usize = 11; + pub const VB_HI: usize = 12; + pub const VB_TOP: usize = 13; + pub const VC_LO: usize = 14; + pub const VC_HI: usize = 15; + pub const VC_TOP: usize = 16; + pub const RA: usize = 17; + pub const RB: usize = 18; + pub const RC: usize = 19; + pub const RBC: usize = 20; + pub const N: usize = 21; } impl Table for Arith { - fn opcode_tag(&self) -> F128 { + fn opcode_tag(&self) -> F64 { if self.is_xor { OP_XOR } else { OP_MUL } } fn n_committed_columns(&self) -> usize { @@ -251,17 +385,16 @@ impl Table for Arith { fn n_constraints(&self) -> usize { 4 // three addresses + the third-operand identity } - fn eval_constraint(&self, pows: &[F128], cols: &Cols) -> F128 { + fn eval_constraint(&self, pows: &[F192], cols: &Cols) -> F192 { use arith::*; - let third = if self.is_xor { - cols[VA] + cols[VB] - } else { - cols[VA] * cols[VB] - }; + let va = e192(cols[VA_LO], cols[VA_HI], cols[VA_TOP]); + let vb = e192(cols[VB_LO], cols[VB_HI], cols[VB_TOP]); + let vc = e192(cols[VC_LO], cols[VC_HI], cols[VC_TOP]); + let third = if self.is_xor { va + vb } else { va * vb }; pows[0] * (cols[AA] + cols[FP] * cols[OA]) + pows[1] * (cols[AB] + cols[FP] * cols[OB]) + pows[2] * (cols[AC] + cols[FP] * cols[OC]) - + pows[3] * (cols[VC] + third) + + pows[3] * (vc + third) } fn flushes(&self, f: &mut FlushBuilder) { use arith::*; @@ -270,11 +403,11 @@ impl Table for Arith { PC, RBC, self.opcode_tag(), - &[Col(OA), Col(OB), Col(OC), Const(F128::ZERO), Const(F128::ZERO)], + &[Col(OA), Col(OB), Col(OC), Const(F64::ZERO), Const(F64::ZERO)], ); - f.memory(AA, RA, VA); - f.memory(AB, RB, VB); - f.memory(AC, RC, VC); + f.memory(AA, RA, VA_LO, VA_HI, VA_TOP); + f.memory(AB, RB, VB_LO, VB_HI, VB_TOP); + f.memory(AC, RC, VC_LO, VC_HI, VC_TOP); } fn fill(&self, ctx: &FillCtx, out: &mut [Column]) { use arith::*; @@ -287,9 +420,15 @@ impl Table for Arith { out[AA] = rows.par_iter().map(|r| ctx.g_at(r.aa)).collect(); out[AB] = rows.par_iter().map(|r| ctx.g_at(r.ab)).collect(); out[AC] = rows.par_iter().map(|r| ctx.g_at(r.ac)).collect(); - out[VA] = rows.par_iter().map(|r| ctx.mem[r.aa as usize]).collect(); - out[VB] = rows.par_iter().map(|r| ctx.mem[r.ab as usize]).collect(); - out[VC] = rows.par_iter().map(|r| ctx.mem[r.ac as usize]).collect(); + out[VA_LO] = rows.par_iter().map(|r| F64(ctx.mem[r.aa as usize].c0)).collect(); + out[VA_HI] = rows.par_iter().map(|r| F64(ctx.mem[r.aa as usize].c1)).collect(); + out[VA_TOP] = rows.par_iter().map(|r| F64(ctx.mem[r.aa as usize].c2)).collect(); + out[VB_LO] = rows.par_iter().map(|r| F64(ctx.mem[r.ab as usize].c0)).collect(); + out[VB_HI] = rows.par_iter().map(|r| F64(ctx.mem[r.ab as usize].c1)).collect(); + out[VB_TOP] = rows.par_iter().map(|r| F64(ctx.mem[r.ab as usize].c2)).collect(); + out[VC_LO] = rows.par_iter().map(|r| F64(ctx.mem[r.ac as usize].c0)).collect(); + out[VC_HI] = rows.par_iter().map(|r| F64(ctx.mem[r.ac as usize].c1)).collect(); + out[VC_TOP] = rows.par_iter().map(|r| F64(ctx.mem[r.ac as usize].c2)).collect(); out[RA] = rows.par_iter().map(|r| r.ra).collect(); out[RB] = rows.par_iter().map(|r| r.rb).collect(); out[RC] = rows.par_iter().map(|r| r.rc).collect(); @@ -302,11 +441,21 @@ impl Table for Arith { struct SetTable; mod set { - columns!(PC, FP, O, K, A, R, RBC); + pub const PC: usize = 0; + pub const FP: usize = 1; + pub const O: usize = 2; + // The stored immediate's three K-limbs ride the bytecode's spare slots. + pub const K_LO: usize = 3; + pub const K_HI: usize = 4; + pub const K_TOP: usize = 5; + pub const A: usize = 6; + pub const R: usize = 7; + pub const RBC: usize = 8; + pub const N: usize = 9; } impl Table for SetTable { - fn opcode_tag(&self) -> F128 { + fn opcode_tag(&self) -> F64 { OP_SET } fn n_committed_columns(&self) -> usize { @@ -319,7 +468,7 @@ impl Table for SetTable { fn n_constraints(&self) -> usize { 1 // the single address binding } - fn eval_constraint(&self, pows: &[F128], cols: &Cols) -> F128 { + fn eval_constraint(&self, pows: &[F192], cols: &Cols) -> F192 { use set::*; // The address a = fp·o. pows[0] * (cols[A] + cols[FP] * cols[O]) @@ -327,13 +476,15 @@ impl Table for SetTable { fn flushes(&self, f: &mut FlushBuilder) { use set::*; f.state_step(PC, FP); + // The immediate's three limbs occupy bytecode operand slots o2..o4 + // (matching layout::operands for SET). f.bytecode( PC, RBC, OP_SET, - &[Col(O), Col(K), Const(F128::ZERO), Const(F128::ZERO), Const(F128::ZERO)], + &[Col(O), Col(K_LO), Col(K_HI), Col(K_TOP), Const(F64::ZERO)], ); - f.memory(A, R, K); // the stored constant K is the cell's value + f.memory(A, R, K_LO, K_HI, K_TOP); // the stored constant K is the cell's value } fn fill(&self, ctx: &FillCtx, out: &mut [Column]) { use set::*; @@ -341,7 +492,9 @@ impl Table for SetTable { out[PC] = rows.par_iter().map(|r| ctx.g_at(r.pc)).collect(); out[FP] = rows.par_iter().map(|r| ctx.g_at(r.fp)).collect(); out[O] = rows.par_iter().map(|r| ctx.g_at(r.o)).collect(); - out[K] = rows.par_iter().map(|r| r.k).collect(); + out[K_LO] = rows.par_iter().map(|r| F64(r.k.c0)).collect(); + out[K_HI] = rows.par_iter().map(|r| F64(r.k.c1)).collect(); + out[K_TOP] = rows.par_iter().map(|r| F64(r.k.c2)).collect(); out[A] = rows.par_iter().map(|r| ctx.g_at(r.a)).collect(); out[R] = rows.par_iter().map(|r| r.r).collect(); out[RBC] = rows.par_iter().map(|r| r.bytecode_read).collect(); @@ -353,11 +506,37 @@ impl Table for SetTable { struct DerefTable; mod deref { - columns!(PC, FP, OAL, OBE, OGA, FPC, FFP, A1, A2, A3, P, V2, V3, R1, R2, R3, RBC); + pub const PC: usize = 0; + pub const FP: usize = 1; + pub const OAL: usize = 2; + pub const OBE: usize = 3; + pub const OGA: usize = 4; + pub const FPC: usize = 5; + pub const FFP: usize = 6; + pub const A1: usize = 7; + pub const A2: usize = 8; + pub const A3: usize = 9; + // The pointer word — a SINGLE K-lane. The address constraint a2 = p·obe + // (with a2 a single-lane K column) forces `p` into K, so its extension + // limbs are provably zero: they are NOT committed, and the memory read + // carries literal zeros there. + pub const P: usize = 10; + // The store target and the local cell, each a full 192-bit word. + pub const V2_LO: usize = 11; + pub const V2_HI: usize = 12; + pub const V2_TOP: usize = 13; + pub const V3_LO: usize = 14; + pub const V3_HI: usize = 15; + pub const V3_TOP: usize = 16; + pub const R1: usize = 17; + pub const R2: usize = 18; + pub const R3: usize = 19; + pub const RBC: usize = 20; + pub const N: usize = 21; } impl Table for DerefTable { - fn opcode_tag(&self) -> F128 { + fn opcode_tag(&self) -> F64 { OP_DEREF } fn n_committed_columns(&self) -> usize { @@ -370,26 +549,31 @@ impl Table for DerefTable { fn n_constraints(&self) -> usize { 4 // three addresses + the flag-selected store } - fn eval_constraint(&self, pows: &[F128], cols: &Cols) -> F128 { + fn eval_constraint(&self, pows: &[F192], cols: &Cols) -> F192 { use deref::*; - // Three addresses (a2 = p·obe is pointer-relative) plus the flag-selected + // The pointer is K-valued; the target and local words are full F192 values. + let p = cols[P]; // single K-lane pointer; extension limbs are zero + let v2 = e192(cols[V2_LO], cols[V2_HI], cols[V2_TOP]); + let v3 = e192(cols[V3_LO], cols[V3_HI], cols[V3_TOP]); + // Three addresses (a2 = p·obe is pointer-relative — with a2 a single K + // column, this forces the pointer word `p` into K) plus the flag-selected // store `v2 = src`, where `src = (1+f_pc+f_fp)·v3 + f_pc·(g²·pc) + f_fp·fp` // over the two boolean store-mode flags. The `pc` source is the virtual // return target g²·pc (a free ×g² of the committed pc), so no column. let src = - (F128::ONE + cols[FPC] + cols[FFP]) * cols[V3] + cols[FPC] * (G * G * cols[PC]) + cols[FFP] * cols[FP]; + (F192::ONE + cols[FPC] + cols[FFP]) * v3 + cols[FPC] * cols[PC].mul_base(G * G) + cols[FFP] * cols[FP]; pows[0] * (cols[A1] + cols[FP] * cols[OAL]) - + pows[1] * (cols[A2] + cols[P] * cols[OBE]) + + pows[1] * (cols[A2] + p * cols[OBE]) + pows[2] * (cols[A3] + cols[FP] * cols[OGA]) - + pows[3] * (cols[V2] + src) + + pows[3] * (v2 + src) } fn flushes(&self, f: &mut FlushBuilder) { use deref::*; f.state_step(PC, FP); f.bytecode(PC, RBC, OP_DEREF, &[Col(OAL), Col(OBE), Col(OGA), Col(FPC), Col(FFP)]); - f.memory(A1, R1, P); - f.memory(A2, R2, V2); - f.memory(A3, R3, V3); + f.memory_k(A1, R1, P); + f.memory(A2, R2, V2_LO, V2_HI, V2_TOP); + f.memory(A3, R3, V3_LO, V3_HI, V3_TOP); } fn fill(&self, ctx: &FillCtx, out: &mut [Column]) { use deref::*; @@ -404,9 +588,17 @@ impl Table for DerefTable { out[A1] = rows.par_iter().map(|r| ctx.g_at(r.a1)).collect(); out[A2] = rows.par_iter().map(|r| ctx.gpow[r.a2]).collect(); // a2 is a full memory index out[A3] = rows.par_iter().map(|r| ctx.g_at(r.a3)).collect(); - out[P] = rows.par_iter().map(|r| r.p).collect(); - out[V2] = rows.par_iter().map(|r| r.v2).collect(); - out[V3] = rows.par_iter().map(|r| r.v3).collect(); + debug_assert!( + rows.iter().all(|r| r.p.c1 == 0 && r.p.c2 == 0), + "deref pointer must be K-valued" + ); + out[P] = rows.par_iter().map(|r| F64(r.p.c0)).collect(); + out[V2_LO] = rows.par_iter().map(|r| F64(r.v2.c0)).collect(); + out[V2_HI] = rows.par_iter().map(|r| F64(r.v2.c1)).collect(); + out[V2_TOP] = rows.par_iter().map(|r| F64(r.v2.c2)).collect(); + out[V3_LO] = rows.par_iter().map(|r| F64(r.v3.c0)).collect(); + out[V3_HI] = rows.par_iter().map(|r| F64(r.v3.c1)).collect(); + out[V3_TOP] = rows.par_iter().map(|r| F64(r.v3.c2)).collect(); out[R1] = rows.par_iter().map(|r| r.r1).collect(); out[R2] = rows.par_iter().map(|r| r.r2).collect(); out[R3] = rows.par_iter().map(|r| r.r3).collect(); @@ -419,13 +611,43 @@ impl Table for DerefTable { struct JumpTable; mod jump { - // Local witness columns (committed, never flushed): the inverse hint `w` and - // the taken indicator `b = [c ≠ 0]` it certifies (doc §7.5). - columns!(PC, FP, NPC, NFP, OC, OD, OF, AC, AD, AF, C, D, F, RC, RD, RF, RBC, W, B); + pub const PC: usize = 0; + pub const FP: usize = 1; + pub const NPC: usize = 2; // next pc — a K address (single lane) + pub const NFP: usize = 3; // next fp — a K address (single lane) + pub const OC: usize = 4; + pub const OD: usize = 5; + pub const OF: usize = 6; + pub const AC: usize = 7; + pub const AD: usize = 8; + pub const AF: usize = 9; + // The condition is an arbitrary F192 word. Destination/frame words are + // K-valued addresses read through the full three-limb memory bus. + pub const C_LO: usize = 10; + pub const C_HI: usize = 11; + pub const C_TOP: usize = 12; + pub const D_LO: usize = 13; + pub const D_HI: usize = 14; + pub const D_TOP: usize = 15; + pub const F_LO: usize = 16; + pub const F_HI: usize = 17; + pub const F_TOP: usize = 18; + pub const RC: usize = 19; + pub const RD: usize = 20; + pub const RF: usize = 21; + pub const RBC: usize = 22; + // Local witness columns (committed, never flushed): the inverse hint `w` + // (192-bit: c⁻¹ in E) and the taken indicator `b = [c ≠ 0]` it certifies + // (the `JUMP` table in `misc/doc.tex`). `b` is a single K-lane (0/1). + pub const W_LO: usize = 23; + pub const W_HI: usize = 24; + pub const W_TOP: usize = 25; + pub const B: usize = 26; + pub const N: usize = 27; } impl Table for JumpTable { - fn opcode_tag(&self) -> F128 { + fn opcode_tag(&self) -> F64 { OP_JUMP } fn n_committed_columns(&self) -> usize { @@ -438,20 +660,24 @@ impl Table for JumpTable { fn n_constraints(&self) -> usize { 7 // three addresses + two indicator identities + the pc/fp selections } - fn eval_constraint(&self, pows: &[F128], cols: &Cols) -> F128 { + fn eval_constraint(&self, pows: &[F192], cols: &Cols) -> F192 { use jump::*; - let one = F128::ONE; - let fall_through = G * cols[PC]; // next pc when the branch is not taken + let one = F192::ONE; + let c = e192(cols[C_LO], cols[C_HI], cols[C_TOP]); + let d = e192(cols[D_LO], cols[D_HI], cols[D_TOP]); + let ff = e192(cols[F_LO], cols[F_HI], cols[F_TOP]); + let w = e192(cols[W_LO], cols[W_HI], cols[W_TOP]); + let fall_through = cols[PC].mul_base(G); let addrs = pows[0] * (cols[AC] + cols[FP] * cols[OC]) + pows[1] * (cols[AD] + cols[FP] * cols[OD]) + pows[2] * (cols[AF] + cols[FP] * cols[OF]); - // `b = cond·w` and `cond·(b+1) = 0` together force `b = [cond ≠ 0]` (doc §7.5): + // `b = cond·w` and `cond·(b+1) = 0` together force `b = [cond ≠ 0]`: // when `cond ≠ 0` the second gives `b = 1` (and the first `w = cond⁻¹`); // when `cond = 0` the first gives `b = 0`. - let ind_def = pows[3] * (cols[B] + cols[C] * cols[W]); - let ind_nz = pows[4] * (cols[C] * (cols[B] + one)); - let sel_pc = pows[5] * (cols[NPC] + cols[B] * cols[D] + (cols[B] + one) * fall_through); - let sel_fp = pows[6] * (cols[NFP] + cols[B] * cols[F] + (cols[B] + one) * cols[FP]); + let ind_def = pows[3] * (cols[B] + c * w); + let ind_nz = pows[4] * (c * (cols[B] + one)); + let sel_pc = pows[5] * (cols[NPC] + cols[B] * d + (cols[B] + one) * fall_through); + let sel_fp = pows[6] * (cols[NFP] + cols[B] * ff + (cols[B] + one) * cols[FP]); addrs + ind_def + ind_nz + sel_pc + sel_fp } fn flushes(&self, f: &mut FlushBuilder) { @@ -461,11 +687,11 @@ impl Table for JumpTable { PC, RBC, OP_JUMP, - &[Col(OC), Col(OD), Col(OF), Const(F128::ZERO), Const(F128::ZERO)], + &[Col(OC), Col(OD), Col(OF), Const(F64::ZERO), Const(F64::ZERO)], ); - f.memory(AC, RC, C); - f.memory(AD, RD, D); - f.memory(AF, RF, F); + f.memory(AC, RC, C_LO, C_HI, C_TOP); + f.memory(AD, RD, D_LO, D_HI, D_TOP); + f.memory(AF, RF, F_LO, F_HI, F_TOP); } fn fill(&self, ctx: &FillCtx, out: &mut [Column]) { use jump::*; @@ -480,10 +706,18 @@ impl Table for JumpTable { out[AC] = rows.par_iter().map(|r| ctx.g_at(r.ac)).collect(); out[AD] = rows.par_iter().map(|r| ctx.g_at(r.ad)).collect(); out[AF] = rows.par_iter().map(|r| ctx.g_at(r.af)).collect(); - out[C] = rows.par_iter().map(|r| r.c).collect(); - out[D] = rows.par_iter().map(|r| r.d).collect(); - out[F] = rows.par_iter().map(|r| r.f).collect(); - out[W] = rows.par_iter().map(|r| r.w).collect(); + out[C_LO] = rows.par_iter().map(|r| F64(r.c.c0)).collect(); + out[C_HI] = rows.par_iter().map(|r| F64(r.c.c1)).collect(); + out[C_TOP] = rows.par_iter().map(|r| F64(r.c.c2)).collect(); + out[D_LO] = rows.par_iter().map(|r| F64(r.d.c0)).collect(); + out[D_HI] = rows.par_iter().map(|r| F64(r.d.c1)).collect(); + out[D_TOP] = rows.par_iter().map(|r| F64(r.d.c2)).collect(); + out[F_LO] = rows.par_iter().map(|r| F64(r.f.c0)).collect(); + out[F_HI] = rows.par_iter().map(|r| F64(r.f.c1)).collect(); + out[F_TOP] = rows.par_iter().map(|r| F64(r.f.c2)).collect(); + out[W_LO] = rows.par_iter().map(|r| F64(r.w.c0)).collect(); + out[W_HI] = rows.par_iter().map(|r| F64(r.w.c1)).collect(); + out[W_TOP] = rows.par_iter().map(|r| F64(r.w.c2)).collect(); out[B] = rows.par_iter().map(|r| r.b).collect(); out[RC] = rows.par_iter().map(|r| r.rc).collect(); out[RD] = rows.par_iter().map(|r| r.rd).collect(); @@ -492,36 +726,151 @@ impl Table for JumpTable { } } +// ---- PACK64X2 ---------------------------------------------------------------- + +/// Pack two K-valued memory cells into one canonical 128-bit cell. There are +/// deliberately no source extension-limb columns: `memory_k` puts literal +/// zeros in those bus coordinates, so the global memory permutation can +/// balance only when the actual source words are in K. Likewise `memory_128` +/// writes the destination as `(va, vb, 0)` directly through the bus. +struct Pack64x2Table; + +mod pack64 { + pub const PC: usize = 0; + pub const FP: usize = 1; + pub const OA: usize = 2; + pub const OB: usize = 3; + pub const OC: usize = 4; + pub const AA: usize = 5; + pub const AB: usize = 6; + pub const AC: usize = 7; + pub const VA: usize = 8; + pub const VB: usize = 9; + pub const RA: usize = 10; + pub const RB: usize = 11; + pub const RC: usize = 12; + pub const RBC: usize = 13; + pub const N: usize = 14; +} + +impl Table for Pack64x2Table { + fn opcode_tag(&self) -> F64 { + OP_PACK64X2 + } + + fn n_committed_columns(&self) -> usize { + pack64::N + } + + fn count_columns(&self) -> &'static [usize] { + use pack64::*; + &[RA, RB, RC, RBC] + } + + fn n_constraints(&self) -> usize { + 3 // the three address bindings + } + + fn eval_constraint(&self, pows: &[F192], cols: &Cols) -> F192 { + use pack64::*; + pows[0] * (cols[AA] + cols[FP] * cols[OA]) + + pows[1] * (cols[AB] + cols[FP] * cols[OB]) + + pows[2] * (cols[AC] + cols[FP] * cols[OC]) + } + + fn flushes(&self, f: &mut FlushBuilder) { + use pack64::*; + f.state_step(PC, FP); + f.bytecode( + PC, + RBC, + OP_PACK64X2, + &[Col(OA), Col(OB), Col(OC), Const(F64::ZERO), Const(F64::ZERO)], + ); + f.memory_k(AA, RA, VA); + f.memory_k(AB, RB, VB); + f.memory_128(AC, RC, VA, VB); + } + + fn fill(&self, ctx: &FillCtx, out: &mut [Column]) { + use pack64::*; + let rows = &ctx.trace.pack64x2; + out[PC] = rows.par_iter().map(|r| ctx.g_at(r.pc)).collect(); + out[FP] = rows.par_iter().map(|r| ctx.g_at(r.fp)).collect(); + out[OA] = rows.par_iter().map(|r| ctx.g_at(r.aa - r.fp)).collect(); + out[OB] = rows.par_iter().map(|r| ctx.g_at(r.ab - r.fp)).collect(); + out[OC] = rows.par_iter().map(|r| ctx.g_at(r.ac - r.fp)).collect(); + out[AA] = rows.par_iter().map(|r| ctx.g_at(r.aa)).collect(); + out[AB] = rows.par_iter().map(|r| ctx.g_at(r.ab)).collect(); + out[AC] = rows.par_iter().map(|r| ctx.g_at(r.ac)).collect(); + out[VA] = rows.par_iter().map(|r| F64(ctx.mem[r.aa as usize].c0)).collect(); + out[VB] = rows.par_iter().map(|r| F64(ctx.mem[r.ab as usize].c0)).collect(); + out[RA] = rows.par_iter().map(|r| r.ra).collect(); + out[RB] = rows.par_iter().map(|r| r.rb).collect(); + out[RC] = rows.par_iter().map(|r| r.rc).collect(); + out[RBC] = rows.par_iter().map(|r| r.bytecode_read).collect(); + } +} + // ---- BLAKE3 ------------------------------------------------------------------ -/// `BLAKE3` (doc §7.6): the four 16-byte input words are addressed -/// *independently* at `aa0, aa1, ab0, ab1` (`= fp·g^{ins[i]}`) — no forced -/// contiguity, so a caller hashing e.g. `(tweak, pp)` need not copy them into -/// adjacent cells. The chaining value and 32-byte output each occupy two -/// consecutive words, based at `acv` and `ac`. Six address bindings -/// `a_X = fp·o_X` are constrained; the -/// compression relating output to input words carries no table constraint -/// here: it is proven by flock's R1CS validity via `q_pkd` (§blake3_flock). +/// `BLAKE3` (“BLAKE3” in `misc/doc.tex`): one standard compression. The four 128-bit message +/// chunks are addressed *independently* at `aa0, aa1, ab0, ab1` +/// (`= fp·g^{ins[i]}`), each a single cell — no forced contiguity between +/// chunks, so a caller hashing e.g. `(tweak, pp)` need not copy them into +/// adjacent cells. The chaining value and the 32-byte output each occupy two +/// consecutive cells, based at `acv` and `ac`, so the row reads eight cells in +/// all. Six address bindings `a_X = fp·o_X` are constrained; the compression +/// relating output words to input words carries no table constraint here: it is +/// proven by flock's R1CS validity via `q_pkd` (§blake3_flock). /// -/// The nine value columns are listed in `n_committed_columns` (they need a local -/// index for the flushes and are filled from the trace for the bus), but `cpu` -/// treats them as VIRTUAL — not committed — and routes their bus claims to -/// `q_pkd`, which already holds those words (see [`BLAKE3_VALUE_COLS`]). +/// A 128-bit chunk is two flock 64-bit words (lo, hi lanes), so the sixteen +/// memory-borne flock words are sixteen value LANE columns over eight cells, +/// plus the metadata immediate's two lanes. They are listed in +/// `n_committed_columns` (they need a local index for the flushes and are filled +/// from the trace for the bus), but `cpu` treats them as VIRTUAL — not committed — +/// and routes their bus claims to `q_pkd`, which already holds those words (see +/// [`BLAKE3_VALUE_COLS`]). struct Blake3Table; -mod blake3t { - // Operands: four input offsets + cv base + output base; addresses mirror that layout, - // with word 1 of the output at the free successor g·AC. Values and read - // counts then follow in canonical a0/a1/b0/b1/c0/c1 order. - columns!( - PC, FP, OA0, OA1, OB0, OB1, OCV, OC, AA0, AA1, AB0, AB1, ACV, AC, - VA0, VA1, VB0, VB1, VC0, VC1, VCV0, VCV1, METADATA, - RA0, RA1, RB0, RB1, RCV0, RCV1, RC0, RC1, RBC, - ); +pub(crate) mod blake3t { + pub const PC: usize = 0; + pub const FP: usize = 1; + pub const OA0: usize = 2; // operand g-powers (offsets) of the four message cells … + pub const OA1: usize = 3; + pub const OB0: usize = 4; + pub const OB1: usize = 5; + pub const OCV: usize = 6; // … the chaining-value base … + pub const OC: usize = 7; // … and the output base + pub const AA0: usize = 8; // the four independent message cell addresses … + pub const AA1: usize = 9; + pub const AB0: usize = 10; + pub const AB1: usize = 11; + pub const ACV: usize = 12; // … the cv base (the second cell is g·ACV) … + pub const AC: usize = 13; // … and the output base (the second cell is g·AC) + // The eighteen flock words as value lanes: a's cells (AA0, AA1), b's cells + // (AB0, AB1), c's cells (AC, g·AC), cv's cells (ACV, g·ACV), two lanes + // (lo, hi) each, then the bytecode metadata immediate's two lanes. + pub const VA0: usize = 14; // AA0.lo, AA0.hi, AA1.lo, AA1.hi + pub const VB0: usize = 18; // AB0.lo, AB0.hi, AB1.lo, AB1.hi + pub const VC0: usize = 22; // AC.lo, AC.hi, (g·AC).lo, (g·AC).hi + pub const VCV0: usize = 26; // ACV.lo, ACV.hi, (g·ACV).lo, (g·ACV).hi + pub const MD0: usize = 30; // metadata: the counter lane … + pub const MD1: usize = 31; // … and the block_len‖flags lane + pub const RA0: usize = 32; // per-cell read counts (two a cells) … + pub const RA1: usize = 33; + pub const RB0: usize = 34; // … two b cells … + pub const RB1: usize = 35; + pub const RCV0: usize = 36; // … two cv cells … + pub const RCV1: usize = 37; + pub const RC0: usize = 38; // … two c cells. + pub const RC1: usize = 39; + pub const RBC: usize = 40; + pub const N: usize = 41; } impl Table for Blake3Table { - fn opcode_tag(&self) -> F128 { + fn opcode_tag(&self) -> F64 { OP_BLAKE3 } fn n_committed_columns(&self) -> usize { @@ -534,7 +883,7 @@ impl Table for Blake3Table { fn n_constraints(&self) -> usize { 6 // the six address bindings } - fn eval_constraint(&self, pows: &[F128], cols: &Cols) -> F128 { + fn eval_constraint(&self, pows: &[F192], cols: &Cols) -> F192 { use blake3t::*; // The six address bindings a_X = fp·o_X (degree 2). The compression // carries no table constraint here: flock's R1CS validity proves it @@ -554,18 +903,30 @@ impl Table for Blake3Table { PC, RBC, OP_BLAKE3, - &[Col(OA0), Col(OA1), Col(OB0), Col(OB1), Col(OCV), Col(OC), Col(METADATA)], + &[ + Col(OA0), + Col(OA1), + Col(OB0), + Col(OB1), + Col(OCV), + Col(OC), + Col(MD0), + Col(MD1), + ], ); - // Four independent input reads; the output occupies two consecutive words - // (base and the free successor g·AC). - f.memory(AA0, RA0, VA0); - f.memory(AA1, RA1, VA1); - f.memory(AB0, RB0, VB0); - f.memory(AB1, RB1, VB1); - f.memory(ACV, RCV0, VCV0); - f.memory_succ(ACV, RCV1, VCV1); - f.memory(AC, RC0, VC0); - f.memory_succ(AC, RC1, VC1); + // Eight cell reads: four independent 128-bit message cells, the chaining + // value's two consecutive cells (ACV, g·ACV), then the output's two + // consecutive cells (AC, g·AC). Each carries its chunk's two lanes with a + // literal-zero top limb (`memory_128`), so the canonical embedding is + // proof-enforced and the zero limbs are never committed. + f.memory_128(AA0, RA0, VA0, VA0 + 1); + f.memory_128(AA1, RA1, VA0 + 2, VA0 + 3); + f.memory_128(AB0, RB0, VB0, VB0 + 1); + f.memory_128(AB1, RB1, VB0 + 2, VB0 + 3); + f.memory_128(ACV, RCV0, VCV0, VCV0 + 1); + f.memory_128_succ(ACV, RCV1, VCV0 + 2, VCV0 + 3); + f.memory_128(AC, RC0, VC0, VC0 + 1); + f.memory_128_succ(AC, RC1, VC0 + 2, VC0 + 3); } fn fill(&self, ctx: &FillCtx, out: &mut [Column]) { use blake3t::*; @@ -584,23 +945,22 @@ impl Table for Blake3Table { out[AB1] = rows.par_iter().map(|r| ctx.g_at(r.ab1)).collect(); out[ACV] = rows.par_iter().map(|r| ctx.g_at(r.acv)).collect(); out[AC] = rows.par_iter().map(|r| ctx.g_at(r.ac)).collect(); - out[VA0] = rows.par_iter().map(|r| r.va0).collect(); - out[VA1] = rows.par_iter().map(|r| r.va1).collect(); - out[VB0] = rows.par_iter().map(|r| r.vb0).collect(); - out[VB1] = rows.par_iter().map(|r| r.vb1).collect(); - out[VC0] = rows.par_iter().map(|r| r.vc0).collect(); - out[VC1] = rows.par_iter().map(|r| r.vc1).collect(); - out[VCV0] = rows.par_iter().map(|r| r.vcv0).collect(); - out[VCV1] = rows.par_iter().map(|r| r.vcv1).collect(); - out[METADATA] = rows.par_iter().map(|r| r.metadata).collect(); - out[RA0] = rows.par_iter().map(|r| r.ra0).collect(); - out[RA1] = rows.par_iter().map(|r| r.ra1).collect(); - out[RB0] = rows.par_iter().map(|r| r.rb0).collect(); - out[RB1] = rows.par_iter().map(|r| r.rb1).collect(); - out[RCV0] = rows.par_iter().map(|r| r.rcv0).collect(); - out[RCV1] = rows.par_iter().map(|r| r.rcv1).collect(); - out[RC0] = rows.par_iter().map(|r| r.rc0).collect(); - out[RC1] = rows.par_iter().map(|r| r.rc1).collect(); + for k in 0..4 { + out[VA0 + k] = rows.par_iter().map(|r| r.va[k]).collect(); + out[VB0 + k] = rows.par_iter().map(|r| r.vb[k]).collect(); + out[VC0 + k] = rows.par_iter().map(|r| r.vc[k]).collect(); + out[VCV0 + k] = rows.par_iter().map(|r| r.vcv[k]).collect(); + } + out[MD0] = rows.par_iter().map(|r| F64(r.metadata.c0)).collect(); + out[MD1] = rows.par_iter().map(|r| F64(r.metadata.c1)).collect(); + out[RA0] = rows.par_iter().map(|r| r.ra[0]).collect(); + out[RA1] = rows.par_iter().map(|r| r.ra[1]).collect(); + out[RB0] = rows.par_iter().map(|r| r.rb[0]).collect(); + out[RB1] = rows.par_iter().map(|r| r.rb[1]).collect(); + out[RCV0] = rows.par_iter().map(|r| r.rcv[0]).collect(); + out[RCV1] = rows.par_iter().map(|r| r.rcv[1]).collect(); + out[RC0] = rows.par_iter().map(|r| r.rc[0]).collect(); + out[RC1] = rows.par_iter().map(|r| r.rc[1]).collect(); out[RBC] = rows.par_iter().map(|r| r.bytecode_read).collect(); } } diff --git a/crates/lean_vm/src/transcript.rs b/crates/lean_vm/src/transcript.rs index cb226edd..56bea98a 100644 --- a/crates/lean_vm/src/transcript.rs +++ b/crates/lean_vm/src/transcript.rs @@ -1,7 +1,14 @@ -//! The shared Fiat–Shamir transcript, concretized with the Ligerito opening -//! type (see [`fiat_shamir`] for the sponge and the wrapper states): flock's -//! protocol functions take these same `ps`/`vs` states, so the whole stack -//! shares ONE transcript. -pub use ::pcs::{Proof, ProverState, VerifierState}; +//! The shared Fiat–Shamir transcript, concretized with the VM's stacked +//! Ligerito opening type (see [`fiat_shamir`] for the sponge and the wrapper +//! states): flock's protocol functions take these same `ps`/`vs` states, so +//! the whole stack shares ONE transcript. pub use fiat_shamir::sponge::{Sponge, TraceOp, trace_start, trace_take}; pub use fiat_shamir::transcript::Error; + +/// The one hash-bearing artifact on the `openings` channel: the batched +/// stacked opening (its ring-switch messages + ONE Ligerito proof). +pub type Opening = ::pcs::stack_open::BatchOpeningProof; + +pub type Proof = fiat_shamir::transcript::Proof; +pub type ProverState = fiat_shamir::transcript::ProverState; +pub type VerifierState<'a> = fiat_shamir::transcript::VerifierState<'a, Opening>; diff --git a/crates/lean_vm/src/vmhash.rs b/crates/lean_vm/src/vmhash.rs index 7999c679..1e85ae4d 100644 --- a/crates/lean_vm/src/vmhash.rs +++ b/crates/lean_vm/src/vmhash.rs @@ -5,43 +5,66 @@ //! guest can therefore replay the standard chunk/tree mode instead of using a //! custom Merkle--Damgard construction. -use primitives::field::F128; +use primitives::field::F64; -/// The historical one-block helper: standard BLAKE3 of exactly 64 bytes. -/// This remains useful for Merkle parent nodes and existing callers. +/// The historical one-block helper: standard BLAKE3 of exactly 64 bytes (two +/// 256-bit halves laid out little-endian — the `Blake3` opcode's default +/// metadata). This remains useful for Merkle parent nodes and existing callers. +/// Lives in [`fiat_shamir::sponge`] (the shared Fiat–Shamir sponge is built on it). pub use fiat_shamir::sponge::compress; -/// Standard BLAKE3 of the little-endian byte encoding of a field-element slice. -pub fn hash_slice(data: &[F128]) -> [F128; 2] { +/// Standard BLAKE3 of the little-endian byte encoding of a K-word slice, +/// returned as four field words. A guest replays it with the `blake3` opcode's +/// chunk-chaining metadata: one compression per 64-byte block, CHUNK_START on +/// the first, CHUNK_END | ROOT on the last (zero-padded partial tail). +pub fn hash_slice(data: &[F64]) -> [F64; 4] { let mut hasher = blake3::Hasher::new(); for word in data { - hasher.update(&word.to_le_bytes()); + hasher.update(&word.0.to_le_bytes()); } let digest = hasher.finalize(); - [ - F128::from_le_bytes(digest.as_bytes()[..16].try_into().unwrap()), - F128::from_le_bytes(digest.as_bytes()[16..].try_into().unwrap()), - ] + let w = |o: usize| u64::from_le_bytes(digest.as_bytes()[o..o + 8].try_into().unwrap()); + [F64(w(0)), F64(w(8)), F64(w(16)), F64(w(24))] } #[cfg(test)] mod tests { use super::*; - fn e(k: u64) -> F128 { - F128::new(k, k.wrapping_mul(0x9e37_79b9) ^ 0x51) + fn e(k: u64) -> F64 { + F64(k.wrapping_mul(0x9e37_79b9_7f4a_7c15) ^ 0x51) } + /// `hash_slice` is exactly standard BLAKE3 of the words' little-endian + /// bytes, and the PCS Merkle leaf hash (`::pcs::merkle::hash_leaf`) equals + /// it on the same field words — the invariant that lets a recursive + /// verifier reuse ONE routine for the transcript/leaf hashing and the PCS + /// tree. Covers empty, single-word, odd, multi-block, and multi-chunk + /// slices. #[test] fn hash_slice_is_standard_blake3() { - for n in [0usize, 1, 2, 3, 64, 65] { - let words: Vec = (0..n).map(|i| e(i as u64 + 1)).collect(); - let bytes: Vec = words.iter().flat_map(|w| w.to_le_bytes()).collect(); + for n in [0usize, 1, 2, 3, 5, 8, 64, 65] { + let words: Vec = (0..n).map(|i| e(i as u64 + 1)).collect(); + let bytes: Vec = words.iter().flat_map(|w| w.0.to_le_bytes()).collect(); let got = hash_slice(&words); let expected = blake3::hash(&bytes); - assert_eq!(got[0].to_le_bytes(), expected.as_bytes()[..16]); - assert_eq!(got[1].to_le_bytes(), expected.as_bytes()[16..]); - assert_eq!(::pcs::merkle::hash_leaf(&bytes), *expected.as_bytes()); + let mut got_bytes = [0u8; 32]; + for (k, w) in got.iter().enumerate() { + got_bytes[8 * k..8 * k + 8].copy_from_slice(&w.0.to_le_bytes()); + } + assert_eq!(got_bytes, *expected.as_bytes(), "n={n}"); + assert_eq!(::pcs::merkle::hash_leaf(&bytes), *expected.as_bytes(), "n={n}"); } } + + /// Standard BLAKE3 binds the length, so a slice and the same slice with an + /// extra trailing zero word hash differently — no padding ambiguity. + #[test] + fn hash_slice_binds_length() { + assert_ne!(hash_slice(&[e(7)]), hash_slice(&[e(7), F64::ZERO])); + let four: Vec = (1..=4).map(e).collect(); + let mut padded = four.clone(); + padded.extend([F64::ZERO; 4]); + assert_ne!(hash_slice(&four), hash_slice(&padded)); + } } diff --git a/crates/lean_vm/src/witness.rs b/crates/lean_vm/src/witness.rs index e8862f98..5eb88bcd 100644 --- a/crates/lean_vm/src/witness.rs +++ b/crates/lean_vm/src/witness.rs @@ -1,12 +1,13 @@ -//! Field-valued columns stacked into one committed witness (§3.1): columns laid -//! end to end, largest first at aligned offsets, into one multilinear `q`. An -//! evaluation claim on column `i` at `ζ` becomes the claim `q̂(ζ, sel_i) = c` on -//! the stack, where `sel_i` is the high-bit selector of the column's offset. +//! `K`-valued columns stacked into one committed witness (§3.1): columns laid +//! end to end, largest first at aligned offsets, into one multilinear `q` over +//! `F64`. An evaluation claim on column `i` at `ζ ∈ E` becomes the claim +//! `q̂(ζ, sel_i) = c` on the stack, where `sel_i` is the high-bit selector of +//! the column's offset. -use primitives::field::F128; +use primitives::field::F64; -/// A committed column: `2^κ` field elements. -pub type Column = Vec; +/// A committed column: `2^κ` `K`-elements. +pub type Column = Vec; /// Where a column sits in the stacked witness. A [`Placement::VIRTUAL`] column is /// NOT committed: it carries data for the bus, but its evaluation claims settle @@ -37,7 +38,7 @@ impl Placement { #[cfg(test)] pub(crate) struct Stacked { pub m: usize, - pub q: Vec, + pub q: Vec, pub placements: Vec, } @@ -56,8 +57,8 @@ pub fn placements_of(kappas: &[Option]) -> (Vec, usize) { placements[i] = Placement { n_vars: k, offset: off }; off += 1 << k; } - // Floor at the PCS minimum (Ligerito's level ladder needs room); tiny witnesses - // zero-pad up. Both sides derive this identically from the kappas. + // Floor at the PCS minimum (Ligerito's level ladder needs room); tiny + // witnesses zero-pad up. Both sides derive this identically from the kappas. let m = crate::log2_ceil_usize(off.max(1)).max(crate::pcs::MIN_MU); (placements, m) } @@ -67,11 +68,11 @@ pub fn placements_of(kappas: &[Option]) -> (Vec, usize) { /// (e.g. `q_pkd`, ~1 GB at scale) copy in parallel — the `2^m` stack is /// memory-bandwidth bound, so a single-threaded `memcpy` leaves most of the /// machine idle. -pub fn stack_q(cols: &[Column], placements: &[Placement], m: usize) -> Vec { +pub fn stack_q(cols: &[Column], placements: &[Placement], m: usize) -> Vec { use rayon::prelude::*; // `alloc_zeroed`-backed for the all-zero pad tail; only the copied ranges are - // touched. (F128 is all-zero bytes at ZERO, so the pad needs no explicit write.) - let mut q = vec![F128::ZERO; 1 << m]; + // touched. (F64 is all-zero bytes at ZERO, so the pad needs no explicit write.) + let mut q = vec![F64::ZERO; 1 << m]; // Copy chunk width: big enough that per-chunk `copy_from_slice` amortizes rayon // dispatch, small enough to spread the largest column across cores. const COPY_CHUNK: usize = 1 << 16; diff --git a/crates/pcs/Cargo.toml b/crates/pcs/Cargo.toml index f9c6a71f..cf66740c 100644 --- a/crates/pcs/Cargo.toml +++ b/crates/pcs/Cargo.toml @@ -13,3 +13,6 @@ blake3.workspace = true rayon.workspace = true serde.workspace = true tracing.workspace = true + +[dev-dependencies] +bincode.workspace = true diff --git a/crates/pcs/src/commit.rs b/crates/pcs/src/commit.rs deleted file mode 100644 index e5db6b2b..00000000 --- a/crates/pcs/src/commit.rs +++ /dev/null @@ -1,396 +0,0 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. -//! PCS commit phase: pack → RS encode (additive NTT) → Merkle root. -//! -//! Uses [`AdditiveNttF128`], the binius-style LCH NTT with neighbors-last -//! pairing. The commit produces a non-systematic RS codeword (treating the -//! packed witness as novel-basis coefficients, zero-padded to the larger -//! domain, then forward-NTT'd). -//! -//! ## Layout -//! -//! With parameters `(m, log_inv_rate)`: -//! - `log_msg_len = m − LOG_PACKING` (= log2 of packed witness length) -//! - `k_code = log_msg_len + log_inv_rate` (= log2 of codeword length) -//! -//! The codeword is a flat sequence of `2^k_code` F_{2^128} elements. Each -//! Merkle leaf is **one** F_{2^128} element = 16 bytes. - -use primitives::{field::F128, pretty_integer}; -use crate::merkle::{self, Hash}; -use crate::ntt::AdditiveNttF128; -use crate::pack::LOG_PACKING; -use serde::{Deserialize, Serialize}; - -/// PCS configuration. Polynomial-basis subspace `{1, x, x², …}` for the NTT. -/// -/// Interleaved RS: the packed witness is split into `2^log_batch_size` -/// independent sub-NTTs of size `2^log_dim` each. Each Merkle leaf holds one -/// codeword position across all `2^log_batch_size` lanes -/// (`2^log_batch_size · 16` bytes per leaf). This trades leaf-call SHA-256 -/// overhead (was 16 B leaves, now 512 B leaves at default `log_batch_size=5`) -/// for much fewer Merkle nodes and better scaling to large `m`. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct PcsParams { - pub m: usize, - pub log_inv_rate: usize, - /// Number of parallel sub-NTTs = `2^log_batch_size`. Default 5 (= 32 lanes). - pub log_batch_size: usize, -} - -impl PcsParams { - /// Total log message length (= log2 packed witness length). - pub fn log_msg_len(&self) -> usize { - self.m - LOG_PACKING - } - /// Per-sub-NTT log dimension (= number of "position" coords). - pub fn log_dim(&self) -> usize { - self.log_msg_len() - self.log_batch_size - } - /// Codeword size (log) per sub-NTT. - pub fn k_code(&self) -> usize { - self.log_dim() + self.log_inv_rate - } - /// Number of Merkle leaves (= per-sub-NTT codeword length). - pub fn n_positions(&self) -> usize { - 1usize << self.k_code() - } - /// `num_ntts` = `2^log_batch_size`. - pub fn num_ntts(&self) -> usize { - 1usize << self.log_batch_size - } - /// Total codeword length in F_{2^128} elements - /// (= `n_positions() * num_ntts()`). - pub fn codeword_len_f128(&self) -> usize { - self.n_positions() * self.num_ntts() - } - /// `log_2` of the F_{2^128} count per **initial** Merkle leaf - /// (= `log_batch_size`; just the row-batch lanes per position). - pub fn log_leaf_f128_count(&self) -> usize { - self.log_batch_size - } - /// Number of initial-tree Merkle leaves - /// (= `codeword_len_f128() / 2^log_batch_size = 2^k_code`). - pub fn n_leaves(&self) -> usize { - self.codeword_len_f128() >> self.log_leaf_f128_count() - } - fn validate(&self) { - assert!( - self.m >= LOG_PACKING + self.log_batch_size, - "m={} too small (need m ≥ LOG_PACKING + log_batch_size = {})", - self.m, - LOG_PACKING + self.log_batch_size, - ); - assert!( - self.log_inv_rate >= 1, - "log_inv_rate must be ≥ 1 for a non-trivial RS code", - ); - } -} - -/// Public commitment (Merkle root + params). -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Commitment { - pub root: Hash, - pub params: PcsParams, -} - -/// Prover-side state retained after commit for use in the opening phase. -/// -/// **The packed witness is NOT stored here.** The caller is responsible for -/// retaining its own copy of the packed witness across commit + open. This -/// avoids ~4 GB of duplication at large `m`, dropping peak commit memory by -/// a factor of ~1.5 (e.g. at m=35: 13 GB → 9 GB). -pub struct ProverData { - pub codeword: Vec, - pub merkle_tree: Vec, -} - -// Recycle the codeword buffer (the prover's largest single allocation — -// 128 MB at m = 29) through the scratch pool instead of unmapping it. -impl Drop for ProverData { - fn drop(&mut self) { - primitives::scratch::give_f128(std::mem::take(&mut self.codeword)); - } -} - -/// Commit to a witness in **F_{2^128}-packed** form (polynomial basis: bit -/// `r` of `z_packed[i]` = logical bit `i·128 + r`). -/// -/// Uses **interleaved RS encoding**: `num_ntts = 2^log_batch_size` independent -/// sub-NTTs share the same domain and twiddles, processed via the SoA -/// interleaved transform. The codeword is stored position-major SoA -/// (`codeword[pos · num_ntts + lane]`); each Merkle leaf is one position = -/// `num_ntts` F_{2^128} = `num_ntts · 16` bytes. -/// -/// **Takes the witness by reference**. The returned [`ProverData`] does NOT -/// retain a copy of the packed witness — the caller is responsible for -/// keeping its own copy across commit + open. This frees ~4 GB during the -/// NTT/Merkle phase at large `m`. -/// -/// `z_packed.len()` must equal `2^(m - LOG_PACKING) = 2^(m - 7)`. -pub fn commit(z_packed: &[F128], params: &PcsParams) -> (Commitment, ProverData) { - params.validate(); - assert_eq!(z_packed.len(), 1usize << params.log_msg_len()); - - let num_ntts = params.num_ntts(); - let n_positions = params.n_positions(); - let codeword_len = n_positions * num_ntts; - - // ---- Codeword buffer (SoA): codeword[pos * num_ntts + lane]. - // - // At large m the codeword buffer is huge (128 MB at m=29, 512 MB at m=31). - // `vec![F128::ZERO; n]` would eagerly zero the whole buffer only for every - // slot to be overwritten anyway, so take a possibly-stale buffer from the - // scratch pool instead: `commit_into` (via `replicate_message_fill`) - // writes every slot exactly once. - let codeword = primitives::scratch::take_f128(codeword_len); - commit_into(z_packed, params, codeword) -} - -/// Like [`commit`], but reuses a caller-provided codeword buffer instead of -/// allocating its own. The buffer must have length `codeword_len`; its -/// CONTENTS may be arbitrary (uninit/stale) — every slot is written here: -/// `z_packed` is replicated into all `2^log_inv_rate` sub-blocks (the exact -/// state after the first `log_inv_rate` NTT layers on `[z, 0, …, 0]`), in -/// parallel. Buffers from the scratch pool -/// are already resident, so no write faults. -pub(crate) fn commit_into( - z_packed: &[F128], - params: &PcsParams, - mut codeword: Vec, -) -> (Commitment, ProverData) { - params.validate(); - assert_eq!(z_packed.len(), 1usize << params.log_msg_len()); - let codeword_len = params.n_positions() * params.num_ntts(); - assert_eq!( - codeword.len(), - codeword_len, - "commit_into: prebuilt codeword buffer has wrong length" - ); - - // RS encoding of [z, 0, …, 0] starts with `log_inv_rate` butterfly layers - // whose bottom inputs are all zero — each is a pure copy, so after those - // layers the buffer holds 2^log_inv_rate replicas of z. Write that state - // directly (replicating z costs the same writes as the zero-fill it - // replaces) and start the NTT at layer `log_inv_rate`, skipping those - // layers' full-buffer reads and multiplies. - replicate_message_fill(&mut codeword, z_packed); - - finalize_commit(codeword, params) -} - -/// Fill `codeword` with `2^r` replicas of `msg` (`r = log2(codeword.len() / -/// msg.len())`) — the exact state after the first `r` forward-NTT layers on -/// the zero-padded coefficient vector `[msg, 0, …, 0]`. Pair with -/// `forward_transform_interleaved_from_layer(…, r)`. Every slot of `codeword` -/// is written (input contents may be stale/uninit). -pub(crate) fn replicate_message_fill(codeword: &mut [F128], msg: &[F128]) { - use rayon::prelude::*; - let msg_len = msg.len(); - debug_assert!(codeword.len().is_multiple_of(msg_len)); - const COPY_CHUNK: usize = 1 << 16; - if msg_len >= COPY_CHUNK { - // Both are powers of two, so chunks never straddle a replica boundary. - codeword - .par_chunks_mut(COPY_CHUNK) - .enumerate() - .for_each(|(i, dst)| { - let src_off = (i * COPY_CHUNK) % msg_len; - dst.copy_from_slice(&msg[src_off..src_off + dst.len()]); - }); - } else { - for rep in codeword.chunks_mut(msg_len) { - rep.copy_from_slice(msg); - } - } -} - -/// Shared tail of [`commit`] / [`commit_into`]: interleaved forward additive -/// NTT (RS-encode every lane) then the initial Merkle tree over codeword rows. -fn finalize_commit(mut codeword: Vec, params: &PcsParams) -> (Commitment, ProverData) { - let timing = std::env::var_os("FLOCK_COMMIT_TIMING").is_some(); - let t_ntt = std::time::Instant::now(); - // ---- Interleaved forward additive NTT: 2^log_batch_size independent - // sub-NTTs with shared twiddles. Each sub-NTT operates on its lane of the - // SoA buffer. The first `log_inv_rate` layers were pre-applied by the - // caller's replicate-fill (commit_into), so start past them. - tracing::info_span!( - "NTT", - kind = "extension encode", - log_domain = %pretty_integer(params.k_code()), - lanes = %pretty_integer(params.num_ntts()) - ) - .in_scope(|| { - let ntt = AdditiveNttF128::standard(params.k_code()); - ntt.forward_transform_interleaved_from_layer( - &mut codeword, - params.num_ntts(), - params.log_inv_rate, - ); - }); - if timing { - eprintln!( - "[commit-timing] ntt: {:.2} ms", - t_ntt.elapsed().as_secs_f64() * 1e3 - ); - } - let t_merkle = std::time::Instant::now(); - - // ---- Merkle commitment: one leaf per codeword position = num_ntts F128. - // Zero-copy: cast the codeword Vec directly to &[u8]. F128 is - // repr(C, align(16)) with two u64s laid out little-endian — same bytes - // as the explicit lo.to_le_bytes() + hi.to_le_bytes() serialization. - let codeword_bytes: &[u8] = unsafe { - core::slice::from_raw_parts( - codeword.as_ptr() as *const u8, - codeword.len() * core::mem::size_of::(), - ) - }; - // Initial tree: one leaf per codeword position, each containing the - // row-batch lanes (num_ntts F_{2^128} values = 2^log_batch_size). The - // Ligerito multilevel prover reuses this initial tree as its L0 tree. - let merkle_tree = merkle::merkle_tree(codeword_bytes, params.n_leaves()); - let root = *merkle_tree.last().expect("merkle tree non-empty"); - if timing { - eprintln!( - "[commit-timing] merkle: {:.2} ms", - t_merkle.elapsed().as_secs_f64() * 1e3 - ); - } - - ( - Commitment { - root, - params: params.clone(), - }, - ProverData { - codeword, - merkle_tree, - }, - ) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_rng::Rng; - - fn default_params(m: usize) -> PcsParams { - PcsParams { - m, - log_inv_rate: 1, - log_batch_size: 1, - } - } - - /// The replicate-fill + start-at-layer-`log_inv_rate` fast path must be - /// byte-identical to the definitional encoding: zero-padded coefficients - /// through the FULL forward NTT. Covers rate 1/2 and 1/4 and both - /// interleaving widths. - #[test] - fn commit_matches_full_ntt_oracle() { - use crate::ntt::AdditiveNttF128; - let mut rng = Rng::new(0xFEED); - for (m, log_inv_rate, log_batch_size) in [(10, 1, 1), (12, 1, 2), (12, 2, 1), (14, 2, 3)] { - let params = PcsParams { - m, - log_inv_rate, - log_batch_size, - }; - let z = rng.bits(1 << m); - let z_packed = super::super::pack::pack_witness(&z, m); - - let (commitment, pd) = commit(&z_packed, ¶ms); - - // Oracle: explicit [z, 0, …, 0] coefficients, full NTT from layer 0. - let mut oracle = vec![F128::ZERO; params.codeword_len_f128()]; - oracle[..z_packed.len()].copy_from_slice(&z_packed); - let ntt = AdditiveNttF128::standard(params.k_code()); - ntt.forward_transform_interleaved(&mut oracle, params.num_ntts()); - - assert_eq!( - pd.codeword, oracle, - "codeword mismatch at m={m} r={log_inv_rate}" - ); - let oracle_bytes: &[u8] = unsafe { - core::slice::from_raw_parts(oracle.as_ptr() as *const u8, oracle.len() * 16) - }; - let oracle_root = *crate::merkle::merkle_tree(oracle_bytes, params.n_leaves()) - .last() - .unwrap(); - assert_eq!( - commitment.root, oracle_root, - "root mismatch at m={m} r={log_inv_rate}" - ); - } - } - - #[test] - fn commit_runs_and_produces_root() { - let mut rng = Rng::new(42); - for m in [8usize, 10, 12] { - let z = rng.bits(1 << m); - let z_packed = super::super::pack::pack_witness(&z, m); - let params = default_params(m); - let (commitment, prover_data) = commit(&z_packed, ¶ms); - assert_eq!(prover_data.codeword.len(), params.codeword_len_f128()); - assert_eq!( - prover_data.merkle_tree.last().copied().unwrap(), - commitment.root - ); - assert_eq!(z_packed.len(), 1 << params.log_msg_len()); - } - } - - #[test] - fn commit_is_deterministic() { - let mut rng = Rng::new(7); - let m = 10; - let z = rng.bits(1 << m); - let z_packed = super::super::pack::pack_witness(&z, m); - let params = default_params(m); - let (c1, _) = commit(&z_packed, ¶ms); - let (c2, _) = commit(&z_packed, ¶ms); - assert_eq!(c1.root, c2.root); - } - - #[test] - fn commit_root_sensitive_to_witness() { - let mut rng = Rng::new(99); - let m = 10; - let mut z = rng.bits(1 << m); - let params = default_params(m); - let (c1, _) = commit(&super::super::pack::pack_witness(&z, m), ¶ms); - z[7] ^= true; - let (c2, _) = commit(&super::super::pack::pack_witness(&z, m), ¶ms); - assert_ne!(c1.root, c2.root); - } - - #[test] - fn rs_encoding_is_linear() { - let mut rng = Rng::new(123); - let m = 9; - let params = default_params(m); - let z1 = rng.bits(1 << m); - let z2 = rng.bits(1 << m); - let z_xor: Vec = z1.iter().zip(&z2).map(|(a, b)| a ^ b).collect(); - let pack = |z: &[bool]| super::super::pack::pack_witness(z, m); - let (_, pd1) = commit(&pack(&z1), ¶ms); - let (_, pd2) = commit(&pack(&z2), ¶ms); - let (_, pd_x) = commit(&pack(&z_xor), ¶ms); - for (i, (&c1, &c2)) in pd1.codeword.iter().zip(&pd2.codeword).enumerate() { - assert_eq!(c1 + c2, pd_x.codeword[i], "linearity fails at i={i}"); - } - } - - #[test] - fn codeword_doubles_message_length() { - let mut rng = Rng::new(2); - let m = 10; - let params = default_params(m); - let z = rng.bits(1 << m); - let z_packed = super::super::pack::pack_witness(&z, m); - let (_, pd) = commit(&z_packed, ¶ms); - assert_eq!(pd.codeword.len(), 2 * z_packed.len()); - } -} diff --git a/crates/pcs/src/lib.rs b/crates/pcs/src/lib.rs index c84bd658..1453c32a 100644 --- a/crates/pcs/src/lib.rs +++ b/crates/pcs/src/lib.rs @@ -1,492 +1,24 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. -//! Polynomial commitment scheme for the bit-MLE witness `ẑ` over GF(2). +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +//! Tower-field polynomial commitment infrastructure. //! -//! Construction: Binius-style packing with a Ligerito opening. -//! -//! - **Commit**: pack the 2^m Boolean witness into 2^(m−7) F_{2^128} elements -//! (one bit per polynomial-basis coordinate of F_{2^128}), batch RS-encode -//! via additive NTT, Merkle-commit the codeword. -//! - **Open** ([`open_batch_mixed_ligerito_stacked`]): γ-combine the -//! ring-switched `q_pkd` claims ([`ring_switch`]) with the caller's stacked -//! point claims ([`StackClaim`]) into ONE basis vector + target, discharged -//! by a single multilevel Ligerito ([`ligerito`]). -//! - **Verify** ([`verify_opening_batch_mixed_ligerito_stacked`]): replay the -//! ring-switch binding, recombine the targets, and run the succinct -//! Ligerito verifier, evaluating each claim's eq-weight at the final point. -//! -//! See [DP24](https://eprint.iacr.org/2024/504) (ring-switching) and the -//! Ligerito paper. +//! Boolean witnesses are packed into `K = GF(2^64)` and Ligerito opens them +//! over its cubic extension `E = GF(2^192)`. -pub mod commit; pub mod ligerito; +mod ligerito_config; pub mod merkle; pub mod ntt; pub mod pack; pub mod ring_switch; +pub mod stack_open; +pub mod tensor_algebra; #[cfg(test)] pub(crate) mod test_rng; -pub use commit::{Commitment, PcsParams, ProverData, commit}; pub use pack::{LOG_PACKING, PaddingSpec, pack_witness}; -/// The transcript states, concretized with this crate's opening type: the one -/// hash-bearing artifact on the `openings` channel is a [`ligerito::LigeritoProof`]. +/// Transcript aliases used by Flock's reduction-only tests. pub type Proof = fiat_shamir::transcript::Proof; pub type ProverState = fiat_shamir::transcript::ProverState; pub type VerifierState<'a> = fiat_shamir::transcript::VerifierState<'a, ligerito::LigeritoProof>; - -use primitives::field::F128; - -// (No composite opening structs: the ring-switch `s_hat_v` slices ride the -// shared transcript stream, so an opening is just the hash-bearing -// [`ligerito::LigeritoProof`].) - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum VerifyError { - RingSwitch(ring_switch::VerifyError), - /// The transmitted opening's shape is inconsistent with the claims - /// (attacker-controlled proof data; rejected before any crypto work). - InvalidProofShape, - /// The Ligerito verifier rejected the opening. - Ligerito, -} - -/// What ring_switch + claim-combination produces, fed to the Ligerito opener. -struct CombinedClaim { - b_combined: Vec, - target_combined: F128, -} - -/// Run the batched ring-switch over the `q_pkd` claims, then build -/// `b_combined` (the γ-weighted combination of the per-claim `rs_eq_ind` -/// weights) and `target_combined`. -fn compute_combined_basis_and_target( - packed_witness: &[F128], - x_outers: &[&[F128]], - precomputed_s_hat_v: &[Option<&[F128]>], - padding: &PaddingSpec, - ps: &mut ProverState, -) -> CombinedClaim { - let n_rs = x_outers.len(); - assert!(n_rs > 0, "need at least one ring-switched claim"); - assert!( - precomputed_s_hat_v.is_empty() || precomputed_s_hat_v.len() == n_rs, - "precomputed_s_hat_v: must be empty or length {n_rs}, got {}", - precomputed_s_hat_v.len(), - ); - - // 1. Ring-switching for all x_outers. - let (rs_results, gammas_rs) = ring_switch::prove_batched_padded_with_precomputed( - packed_witness, - x_outers, - precomputed_s_hat_v, - padding, - ps, - ); - - use rayon::prelude::*; - - let l = rs_results[0].rs_eq_ind.dense_len(); - debug_assert!(rs_results.iter().all(|o| o.rs_eq_ind.dense_len() == l)); - - let mut target_combined = F128::ZERO; - for (output, g) in rs_results.iter().zip(gammas_rs.iter()) { - target_combined += *g * output.sumcheck_claim; - } - - let rs_baked: Vec<&[F128]> = rs_results - .iter() - .filter_map(|o| match &o.rs_eq_ind { - ring_switch::RsEqInd::Dense(v) => Some(v.as_slice()), - _ => None, - }) - .collect(); - // Deferred-dense claims (fused fast path): the per-claim `γ_k·B_k` buffer - // was never materialized — fold each slot on the fly below and accumulate - // straight into `b_combined`, saving a 2^(m-7) materialize + readback per - // claim. Carries (eq_lo, eq_hi, γ-baked byte table, log₂ block) per - // deferred claim. - type DeferredFold<'a> = (&'a [F128], &'a [F128], &'a [F128], usize); - let rs_deferred: Vec = rs_results - .iter() - .filter_map(|o| match &o.rs_eq_ind { - ring_switch::RsEqInd::DeferredDense { - eq_lo, - eq_hi, - table, - } => Some(( - eq_lo.as_slice(), - eq_hi.as_slice(), - table.as_slice(), - eq_lo.len().trailing_zeros() as usize, - )), - _ => None, - }) - .collect(); - // ---- Build b_combined (γ-weighted sum of all rs_eq_ind weights). - let mut b_combined: Vec = primitives::scratch::take_f128(l); - - // Fast path (the standard open: claims ab, c): every RS claim is a fused - // DeferredDense fold. Fold all claims block-by-block straight into - // b_combined — each claim's `e_hi` hoisted once per block, exactly as in - // `fold_b128_elems_split`. The per-claim `γ_k·B_k` buffer is never - // materialized (saves ~2·L writes + 2·L reads of the 2^(m-7) basis). - let use_fast = !rs_deferred.is_empty() && rs_deferred.len() == rs_results.len(); - - if use_fast { - let b = rs_deferred[0].0.len(); // eq_lo.len(); shared across claims (same split) - debug_assert!(b >= 2 && b.is_multiple_of(2)); - debug_assert!(rs_deferred.iter().all(|d| d.0.len() == b)); - b_combined.par_chunks_mut(b).enumerate().for_each(|(hi, out_block)| { - // Accumulate each claim's block: first claim writes, rest add. - // `e_hi` is read once per claim per block, then swept over eq_lo. - for (ci, (eq_lo, eq_hi, table, _)) in rs_deferred.iter().enumerate() { - let e_hi = eq_hi[hi]; - if ci == 0 { - for (slot, &lo) in out_block.iter_mut().zip(eq_lo.iter()) { - *slot = ring_switch::fold_one_slot(lo * e_hi, table); - } - } else { - for (slot, &lo) in out_block.iter_mut().zip(eq_lo.iter()) { - *slot += ring_switch::fold_one_slot(lo * e_hi, table); - } - } - } - }); - } else { - // General path (sparse / dense RS claims): materialize any - // deferred-dense claims (parallel block fold), then the per-element - // combine over all dense buffers. - let materialized: Vec> = rs_results - .iter() - .filter_map(|o| match &o.rs_eq_ind { - ring_switch::RsEqInd::DeferredDense { - eq_lo, - eq_hi, - table, - } => Some(ring_switch::fold_b128_from_table(eq_lo, eq_hi, table)), - _ => None, - }) - .collect(); - let mut rs_dense_all: Vec<&[F128]> = rs_baked.clone(); - rs_dense_all.extend(materialized.iter().map(|v| v.as_slice())); - b_combined.par_iter_mut().enumerate().for_each(|(i, slot)| { - let mut acc = F128::ZERO; - for v in rs_dense_all.iter() { - acc += v[i]; - } - *slot = acc; - }); - for v in materialized { - primitives::scratch::give_f128(v); - } - } - for output in rs_results.iter() { - if let ring_switch::RsEqInd::Sparse { entries, .. } = &output.rs_eq_ind { - for &(idx, val) in entries { - b_combined[idx] += val; - } - } - } - - // The per-claim rs_eq_ind (L F128s) dies here — recycle it. (The s_hat_v - // slices were already streamed inside `prove_batched_*`.) - for o in rs_results { - if let ring_switch::RsEqInd::Dense(v) = o.rs_eq_ind { - primitives::scratch::give_f128(v); - } - } - CombinedClaim { - b_combined, - target_combined, - } -} - -// ===== leanVM-b stacked opener (grafted) ===== -/// A point claim folded into the stacked mixed opening ([`open_batch_mixed_ligerito_stacked`]). -/// Either a **block-sparse** slot claim — weight `eq(low_point,·)` supported on the -/// aligned sub-block `[offset, offset + 2^low_point.len())`, so the opener builds -/// `eq` over just the slot instead of the whole `2^m` stack — or a **general** -/// full-stack point (`eq(point,·)` over all `2^m`). leanVM's point claims are all -/// `Slot`s (their `eq` is zero outside the slot); `Point` keeps the opener usable -/// for arbitrary claims. -pub enum StackClaim<'a> { - /// `eq(low_point,·)` on `[offset, offset + 2^low_point.len())`. `offset` must - /// be a multiple of `2^low_point.len()` (an aligned slot). - Slot { offset: usize, low_point: &'a [F128], value: F128 }, - /// A **boolean-selector** claim on a packed column, equivalent to a `Slot` - /// with `low_point = slot_bits(slot, stride_log) ++ point` but folded sparsely: - /// the low `stride_log` block coords are frozen to `slot`'s bits (so the weight - /// is nonzero only at `offset + slot + j·2^stride_log`), and `point` is the - /// high part. Costs `2^point.len()` instead of `2^(stride_log + point.len())`. - /// `offset` must be a multiple of `2^(stride_log + point.len())`. - StridedSlot { offset: usize, slot: usize, stride_log: usize, point: &'a [F128], value: F128 }, - /// `eq(point,·)` over the whole `2^m` stack. - Point { point: &'a [F128], value: F128 }, -} - -impl StackClaim<'_> { - #[inline] - fn value(&self) -> F128 { - match self { - StackClaim::Slot { value, .. } - | StackClaim::StridedSlot { value, .. } - | StackClaim::Point { value, .. } => *value, - } - } -} - -/// Fold the γ-weighted point claims into the lifted stack weight `b_stack` and -/// running `target` (pure — the caller has already observed the claim values and -/// sampled `gammas_pd` in transcript order). Factored out of -/// [`open_batch_mixed_ligerito_stacked`]; produces the -/// `⟨stack, b_stack⟩ = target` inner-product claim. -fn fold_stacked_point_claims(b_stack: &mut [F128], target: &mut F128, stack_pd: &[StackClaim], gammas_pd: &[F128]) { - use rayon::prelude::*; - // `build_eq` and `build_eq_parallel` produce the identical table and serial - // and parallel scatter give the identical result, so the proof is - // byte-for-byte unchanged. A `Slot` builds `eq` over ONLY its aligned - // sub-block (leanVM's claims — `eq` is zero elsewhere), a `Point` over the - // whole stack. Both scatter with `+=`, so overlapping slots (e.g. several - // claims on the q_pkd column) accumulate correctly. Small slots use the - // serial path: with hundreds of tiny point claims, rayon dispatch would - // cost more than the fold itself. - const PAR_FOLD_THRESHOLD: usize = 1 << 14; - for (claim, g) in stack_pd.iter().zip(gammas_pd.iter()) { - let g = *g; - match claim { - StackClaim::Slot { offset, low_point, value } => { - let len = 1usize << low_point.len(); - let dst = &mut b_stack[*offset..*offset + len]; - if len < PAR_FOLD_THRESHOLD { - let eq = primitives::multilinear::build_eq(low_point); - for (bi, ei) in dst.iter_mut().zip(eq.iter()) { - *bi += g * *ei; - } - } else { - let eq = ring_switch::build_eq_parallel(low_point); - dst.par_iter_mut().zip(eq.par_iter()).for_each(|(bi, ei)| *bi += g * *ei); - } - *target += g * *value; - } - StackClaim::StridedSlot { offset, slot, stride_log, point, value } => { - // Sparse: eq over the instance `point` (2^point.len()), - // scattered at stride 2^stride_log into the slot's positions. - // Identical b_stack contribution to the dense Slot with - // low_point = slot_bits ++ point, at ~2^stride_log× less work. - let stride = 1usize << stride_log; - let base = *offset + *slot; - let eq = if point.len() < 14 { - primitives::multilinear::build_eq(point) - } else { - ring_switch::build_eq_parallel(point) - }; - for (j, &ej) in eq.iter().enumerate() { - b_stack[base + j * stride] += g * ej; - } - *target += g * *value; - } - StackClaim::Point { point, value } => { - let eq = ring_switch::build_eq_parallel(point); - b_stack - .par_iter_mut() - .zip(eq.par_iter()) - .for_each(|(bi, ei)| *bi += g * *ei); - *target += g * *value; - } - } - } -} - -/// The claim's weight `eq(full claim point, x)` at an arbitrary point `x` of the -/// full stack cube — a `Slot`'s full point is `[low_point, selector_bits]`, a -/// `StridedSlot`'s is `[slot_bits, point, selector_bits]`; neither is -/// materialized. Used by the Ligerito verifier's residual evaluator (at the -/// residual points). -fn stack_claim_eq_at(claim: &StackClaim, x: &[F128]) -> F128 { - match claim { - StackClaim::Slot { offset, low_point, .. } => { - let n = low_point.len(); - let mut e = primitives::multilinear::eq_eval(low_point, &x[..n]); - let sel = offset >> n; - for (k, &xi) in x[n..].iter().enumerate() { - e *= if (sel >> k) & 1 == 1 { xi } else { F128::ONE + xi }; - } - e - } - StackClaim::StridedSlot { offset, slot, stride_log, point, .. } => { - let mut e = F128::ONE; - for (k, &xi) in x[..*stride_log].iter().enumerate() { - e *= if (slot >> k) & 1 == 1 { xi } else { F128::ONE + xi }; - } - let block_vars = stride_log + point.len(); - e *= primitives::multilinear::eq_eval(point, &x[*stride_log..block_vars]); - let sel = offset >> block_vars; - for (k, &xi) in x[block_vars..].iter().enumerate() { - e *= if (sel >> k) & 1 == 1 { xi } else { F128::ONE + xi }; - } - e - } - StackClaim::Point { point, .. } => primitives::multilinear::eq_eval(point, x), - } -} - - -/// Open ring-switched claims and full-stack point claims in ONE Ligerito -/// opening: ring-switch combine + lifted `b_stack` build, γ-folded into a -/// single `⟨stack, b_stack⟩ = target` inner-product claim discharged by the -/// Ligerito multilevel prover, reusing the caller's commit as L0. -/// `lig_config.initial_k` / `log_inv_rates[0]` must match the commit's params. -#[allow(clippy::too_many_arguments)] -pub fn open_batch_mixed_ligerito_stacked( - qpkd: &[F128], - x_outers: &[&[F128]], - precomputed_s_hat_v: &[Option<&[F128]>], - padding: &PaddingSpec, - stack: &[F128], - stack_offset: usize, - stack_data: &ProverData, - stack_commitment: &Commitment, - stack_pd: &[StackClaim], - lig_config: &ligerito::LigeritoConfig, - ps: &mut ProverState, -) -> ligerito::LigeritoProof { - assert_eq!( - lig_config.initial_k, stack_commitment.params.log_batch_size, - "ligerito initial_k must match PcsParams.log_batch_size for L0 reuse", - ); - assert_eq!( - lig_config.log_inv_rates[0], stack_commitment.params.log_inv_rate, - "ligerito log_inv_rates[0] must match PcsParams.log_inv_rate for L0 reuse", - ); - - let combined = compute_combined_basis_and_target(qpkd, x_outers, precomputed_s_hat_v, padding, ps); - // SAFETY: F128 is a pair of integer lanes and its all-zero byte pattern is - // exactly F128::ZERO. A zeroed allocation leaves untouched padding on - // demand-zero pages instead of eagerly writing the entire stack. - let mut b_stack: Vec = unsafe { primitives::alloc_zeroed_vec(stack.len()) }; - b_stack[stack_offset..stack_offset + combined.b_combined.len()].copy_from_slice(&combined.b_combined); - let mut target = combined.target_combined; - - for claim in stack_pd { - ps.observe_scalar(claim.value()); - } - let gammas_pd: Vec = (0..stack_pd.len()).map(|_| ps.sample()).collect(); - fold_stacked_point_claims(&mut b_stack, &mut target, stack_pd, &gammas_pd); - - - ligerito::multilevel_prover_with_basis( - lig_config, - stack.to_vec(), - b_stack, - target, - &stack_data.codeword, - &stack_data.merkle_tree, - ps, - ) -} - -/// What the stacked opening verifier hands back on accept: the ring-switch -/// batching challenges and the Ligerito fold/query data — everything a -/// recursion harness needs, named and typed. -#[derive(Clone, Debug)] -pub struct StackedOpeningSummary { - /// The `r''` shared by every ring-switch claim of the batch. - pub r_dprime: Vec, - pub lig: ligerito::LigVerifierSummary, -} - -/// Verifier mirror of [`open_batch_mixed_ligerito_stacked`]: replay the -/// ring-switch reduction + γ-folds in the prover's transcript order, then -/// drive the SUCCINCT Ligerito verifier with a residual evaluator for the -/// lifted weight: at each residual point `x = ris ++ y_bits`, -/// `b(x) = eq(sel, x_hi)·Σ γ_rs·rs_eq(x_lo) + Σ γ_pd·eq(claim, x)`. -pub fn verify_opening_batch_mixed_ligerito_stacked( - stack_commitment: &Commitment, - stack_offset: usize, - qpkd_vars: usize, - claims: &[F128], - z_skips: &[F128], - x_outers: &[&[F128]], - stack_pd: &[StackClaim], - proof: &ligerito::LigeritoProof, - lig_config: &ligerito::LigeritoConfig, - vs: &mut VerifierState<'_>, -) -> Result { - let n_rs = claims.len(); - // These are caller (leanVM) invariants. - assert_eq!(z_skips.len(), n_rs); - assert_eq!(x_outers.len(), n_rs); - // (The s_hat_v slices ride the stream; `verify_bind` reads exactly - // 2^LOG_PACKING words per claim, so there is no shape to validate here.) - - // Bind + check every claim, then sample ONE shared r'' (sound: every - // slice is absorbed before the challenge), then form the batched claims. - let mut rs_proofs = Vec::with_capacity(n_rs); - for i in 0..n_rs { - rs_proofs.push( - ring_switch::verify_bind(claims[i], z_skips[i], x_outers[i], vs) - .map_err(VerifyError::RingSwitch)?, - ); - } - // Mirror the prover: with n_rs = 0 the ring-switch batch never runs on the - // prover side, so no r'' is sampled there — skip it here too or the two - // transcripts diverge (the prover samples r'' inside `prove_batched_*`). - let r_dprime = if n_rs > 0 { vs.sample_vec(LOG_PACKING) } else { Vec::new() }; - let lin_coeffs = - if n_rs > 0 { ring_switch::linearized_eq_coeffs_eq(&r_dprime) } else { [F128::ZERO; 128] }; - let rs_claims: Vec = (0..n_rs) - .map(|i| ring_switch::transposed_claim_linearized(&rs_proofs[i], &lin_coeffs)) - .collect(); - let gammas_rs: Vec = (0..n_rs).map(|_| vs.sample()).collect(); - let mut target_combined = F128::ZERO; - for (claim, g) in rs_claims.iter().zip(gammas_rs.iter()) { - target_combined += *g * *claim; - } - - for claim in stack_pd { - vs.observe_scalar(claim.value()); - } - let gammas_pd: Vec = (0..stack_pd.len()).map(|_| vs.sample()).collect(); - for (claim, g) in stack_pd.iter().zip(gammas_pd.iter()) { - target_combined += *g * claim.value(); - } - - // The lifted weight at ONE point. The opening now runs its sumcheck to - // completion, so a single evaluation closes it; there is no residual cube to - // sweep and no need for the weight to factor across a split. - let log_n = stack_commitment.params.m - LOG_PACKING; - let sel = stack_offset >> qpkd_vars; - let eval_b_at = |x: &[F128]| -> F128 { - let (x_lo, x_hi) = x.split_at(qpkd_vars); - let mut sel_eq = F128::ONE; - for (k, &xi) in x_hi.iter().enumerate() { - sel_eq *= if (sel >> k) & 1 == 1 { xi } else { F128::ONE + xi }; - } - let mut rs_part = F128::ZERO; - for (g, x_outer) in gammas_rs.iter().zip(x_outers.iter()) { - rs_part += *g * ring_switch::eval_rs_eq_from_coeffs(&x_outer[1..], x_lo, &lin_coeffs); - } - let mut acc = rs_part * sel_eq; - for (claim, g) in stack_pd.iter().zip(gammas_pd.iter()) { - acc += *g * stack_claim_eq_at(claim, x); - } - acc - }; - - let lig = ligerito::multilevel_verifier_with_basis_succinct( - lig_config, - proof, - log_n, - target_combined, - &stack_commitment.root, - eval_b_at, - vs, - ) - .ok_or(VerifyError::Ligerito)?; - Ok(StackedOpeningSummary { - r_dprime, - lig, - }) -} diff --git a/crates/pcs/src/ligerito.rs b/crates/pcs/src/ligerito.rs index dab20afc..dc1ecbd7 100644 --- a/crates/pcs/src/ligerito.rs +++ b/crates/pcs/src/ligerito.rs @@ -1,1120 +1,895 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. // Copyright (c) 2026 Bain Capital Crypto, LP and Ron Rothblum // Modifications copyright 2026 Succinct Labs, Benedikt Bunz, William Wang // SPDX-License-Identifier: Apache-2.0 OR MIT -// -// Ported from bolt-rs (https://github.com/bcc-research/bolt-rs, -// `ligerito_recursive.rs`). -//! Ligerito: multilevel multilinear PCS. +//! Ligerito with `K = GF(2)[x]/(x^64+x^4+x^3+x+1)` and +//! `E = K[y]/(y^3+y+1)`. +//! +//! The committed message is a +//! vector of [`F64`] values; every verifier challenge, sumcheck message, basis +//! poly, and post-fold witness is [`F192`]-valued. //! -//! Ported from bolt-rs (`ligerito_recursive.rs`) onto Flock primitives: -//! `F128` (GHASH irreducible), [`AdditiveNttF128`] (LCH novel basis, -//! byte-identical to bolt-rs's FFT), SHA-256 merkle from [`crate::merkle`], -//! and the shared [`fiat_shamir::sponge::Sponge`] for Fiat-Shamir. +//! Type map relative to the original: +//! - committed message / L0 codeword / L0 opened rows: `F64` (8 bytes) +//! - challenges, sumcheck messages, folded witnesses, deeper-level codewords +//! and opened rows, `b_initial`, betas, alphas, `yr`: `F192` (24 bytes) +//! - the RS-encoding evaluation domain and all LCH twiddles stay in K, so the +//! deeper-level (E-valued) encodes use K-twiddles via the mixed product +//! [`F192::mul_base`] (3 PMULL) instead of a full E multiplication. //! -//! Soundness regimes (our paper App. C.3): unique decoding (Thm `ca-udr`, -//! BCHKS25 Cor. 1.4 — the ONE shipped configuration, see [`SECURITY_BITS`]) -//! and Johnson list decoding with out-of-domain binding (Thm `ca-johnson`, -//! BCHKS25 Thm 4.6 + Johnson interleaved list bound — hand-built configs -//! only). See [`SoundnessRegime`]. +//! Deliberate divergences from the original (each noted inline too): +//! - Buffers use plain `Vec` allocation where the original recycles through +//! `crate::scratch` (no F64/F192 pool exists yet). +//! - The prover/commit timing instrumentation answers to `LIGERITO_TRACE` +//! (instead of the original's `LIG_PROVE_TRACE` / `FLOCK_COMMIT_TIMING`). //! -//! ## Protocol -//! 1. Commit f^0: reshape into `num_interleaved × msg_cols`, RS-encode each -//! lane to `block_len = msg_cols · 2^log_inv_rate`, merkle over codeword -//! positions (one position across all lanes = one leaf). -//! 2. Partial-eval f^0 with `initial_k` challenges → f^1. -//! 3. Commit f^1. -//! 4. Open `num_queries` rows of f^0; build induced sumcheck basis poly. -//! 5. For each level step i: -//! a. Run k_i sumcheck rounds. -//! b. Last step: send remaining poly + open f^i. -//! c. Else: commit f^{i+2}, open f^{i+1}, induce next basis, glue. - -use fiat_shamir::sponge::Sponge; -use crate::{ProverState, VerifierState}; -use primitives::{field::F128, multilinear::{build_eq, eq_eval, mle_eval}, pretty_integer}; +//! Basis induction mirrors the original's two strategies: the dense +//! per-query LCH expansion and the sparse transposed-NTT fast path +//! (`induce_sumcheck_poly_via_ntt_base`), with the SAME auto-dispatch size +//! heuristic at L0 (deeper levels stay dense, exactly like the original). +//! +//! Soundness note: [`LigeritoSecurityConfig`] analyzes the actual challenge +//! field size `q = 2^192`; the committed alphabet remains `K = GF(2^64)`. + use crate::merkle::{self, Hash}; -use crate::ntt::additive_ntt_f128::AdditiveNttF128; +use crate::ntt::AdditiveNttF64; +use fiat_shamir::Sponge; +use primitives::{ + field::{F64, F192, F192BaseUnreduced, F192Unreduced}, + multilinear::eq_eval, + pretty_integer, +}; use serde::{Deserialize, Serialize}; +pub use super::ligerito_config::*; + // =================================================================== -// Config +// Sponge helpers: E = F192 straight off the shared Fiat-Shamir sponge // =================================================================== +// +// The sponge's scalars ARE E-elements (three K-limbs per sampled scalar), +// so sampling/observing here is the sponge API verbatim; the helpers only +// keep the K files' call sites uniform. -// The ONE Ligerito configuration this repo ships (the old `Secure` profile): -// rate-1/2 unique-decoding regime (list size 1, no OOD binding), 120-bit -// round-by-round soundness. `SoundnessRegime::JohnsonOod` machinery survives -// only for hand-built configs (analysis / tests). - -/// Round-by-round soundness target (bits): every round must individually -/// clear this level (total security = min over rounds, per the Fiat-Shamir / -/// `soundcalc` convention). -pub const SECURITY_BITS: usize = 120; - -/// L0 code rate index: `rho_0 = 2^-LOG_INV_RATE_0` (rate 1/2). -pub const LOG_INV_RATE_0: usize = 1; - -/// Query-phase grinding bits: with `g` bits ground, the per-level queries only -/// need to cover `SECURITY_BITS - g` bits (validation rule 3) — about 15% -/// fewer queries, which recursion feels directly (the query walk dominates a -/// guest). -pub const QUERY_GRINDING_BITS: usize = 18; - -pub const INITIAL_FOLDING_FATOR: usize = 6; -pub const SUBSEQUENT_FOLDING_FACTORS: usize = 3; - -/// Folding stops once at most this many variables remain: the residual -/// polynomial (`yr`, at most `2^RESIDUAL_MAX_LOG` coefficients) is sent in -/// clear instead of committed and folded further. -pub const RESIDUAL_MAX_LOG: usize = 5; - -#[derive(Clone, Debug)] -pub struct LigeritoConfig { - pub log_inv_rates: Vec, - pub level_steps: usize, - pub initial_log_msg_cols: usize, - pub initial_log_num_interleaved: usize, - pub initial_k: usize, - pub level_log_msg_cols: Vec, - pub level_ks: Vec, - /// Per-level query counts (L0, L1, ..., L_r). Length = level_steps + 1. - /// [`LigeritoSecurityConfig::derive_config`] fills these from the - /// per-level soundness analysis. - pub queries: Vec, - /// Per-level **query-phase** PoW grinding bits (L0, L1, ..., L_r), ground - /// post-commit/pre-queries. Length = level_steps + 1. Each bit here - /// substitutes for ~1/log₂(1/(1−γ)) queries at that level. - pub grinding_bits: Vec, - /// Per-level **fold-challenge** PoW grinding bits (L0, ..., L_r), ground - /// immediately before EACH of the level's fold challenges (so a level - /// with `k` folds does `k` grinds of this many bits). Boosts the - /// proximity-gap term, which lives on the fold challenges. Length = - /// level_steps + 1. - pub fold_grinding_bits: Vec, - /// Per-commit-level out-of-domain samples (L0, ..., L_r), taken right - /// after the level's Merkle root enters the transcript. `[0]` must be 0: - /// L0 is bound by the opening's own (post-commit, random-point) - /// evaluation claim. Length = level_steps + 1. - pub ood_samples: Vec, +#[inline] +fn sample_ext(sponge: &mut Sponge) -> F192 { + sponge.sample() } -/// The per-level shape table a [`LigeritoConfig`] implies for a -/// `log_n`-variable opening — the numbers every consumer of the multilevel -/// protocol (the verifier itself, recursion harnesses) otherwise re-derives. -#[derive(Clone, Debug)] -pub struct LevelShapes { - /// Level count (`level_steps + 1`). - pub levels: usize, - /// Fold count per level: `initial_k` then `level_ks`. - pub ks: Vec, - /// Log message columns entering each level's fold (`log_n - initial_k`, - /// then descending by each level's `k`). - pub log_msg_cols: Vec, - /// Committed block length per level (`msg_cols * inv_rate`). - pub block_len: Vec, - /// The residual cube dimension left after every fold. - pub yr_log_n: usize, +fn sample_ext_vec(sponge: &mut Sponge, n: usize) -> Vec { + sponge.sample_vec(n) } -impl LigeritoConfig { - /// See [`LevelShapes`]. - pub fn level_shapes(&self, log_n: usize) -> LevelShapes { - let r = self.level_steps; - let ks: Vec = std::iter::once(self.initial_k).chain(self.level_ks.iter().copied()).collect(); - let mut log_msg_cols = vec![log_n - self.initial_k]; - for i in 0..r { - log_msg_cols.push(log_msg_cols[i] - self.level_ks[i]); - } - let mut block_len = vec![1usize << (self.initial_log_msg_cols + self.log_inv_rates[0])]; - for i in 0..r { - block_len.push(1usize << (self.level_log_msg_cols[i] + self.log_inv_rates[i + 1])); - } - LevelShapes { - levels: r + 1, - ks, - yr_log_n: *log_msg_cols.last().unwrap(), - log_msg_cols, - block_len, - } +#[inline] +fn observe_ext(sponge: &mut Sponge, e: F192) { + sponge.observe(e); +} + +/// Bind a Merkle root into the transcript as two `F192` scalars rather than +/// as a byte string. Binds the root before any challenge exactly as `absorb_bytes` +/// would; keeping the scalar form matches the recursion guest's replay. +fn observe_root(sponge: &mut Sponge, root: &crate::merkle::Hash) { + for s in crate::merkle::hash_to_scalars(root) { + observe_ext(sponge, s); } } +// =================================================================== +// Multilinear helpers over E +// =================================================================== -/// Proximity loss `ε*` for the UDR (unique-decoding regime) analysis. It -/// would back the proximity radius off to `γ = δ/2 − ε*` (δ = 1 − ρ the -/// code's relative distance); set to `0`, so we decode to the full -/// unique-decoding radius `γ = δ/2` with no backoff. Per our paper's Appendix -/// C.3 (Theorem `ca-udr`, BCHKS25 Cor. 1.4) the proximity-gap exceptional set -/// is then `a = γ·n + 1` — length-dependent (see [`paper_thm_1_4_log_a`]), so -/// `eps_pg = 128 − log₂ a` shrinks ~1 bit per witness doubling and is -/// recovered by `fold_grinding_bits`. -pub const UDR_PROXIMITY_LOSS: f64 = 0.0; +/// Build the eq-MLE table at `point` in E^d, LSB-first: mirror of +/// `lincheck::build_eq_table` with F192 arithmetic. +pub fn build_eq_table_ext(point: &[F192]) -> Vec { + let d = point.len(); + let mut out: Vec = Vec::with_capacity(1usize << d); + out.push(F192::ONE); + for j in 0..d { + let r_j = point[j]; + let len = 1usize << j; + out.resize(2 * len, F192::ZERO); + for i in 0..len { + let v = out[i]; + let high = v * r_j; + out[i + len] = high; + out[i] = v + high; + } + } + out +} -/// Soundness (in bits) the query phase must close on its own at every level -/// (the "100 bits from queries always" policy). -#[cfg(test)] -const UDR_TARGET_BITS: f64 = 100.0; - -/// Number of queries for 100-bit soundness in the **unique-decoding regime** -/// at rate `2^(-log_inv_rate)`: `γ = δ/2 = (1−ρ)/2`, per-query soundness -/// `log₂(1/(1−γ))` (see [`udr_per_query_bits`]). Within the unique decoding -/// radius the prover is pinned to a single codeword, so there is no list and -/// no union-bound term — queries close the full target by themselves. -/// Per-query soundness saturates below 1 bit (`γ < 1/2`), so slimmer codes -/// bottom out near `UDR_TARGET_BITS` queries: 243 at rate 1/2, 148 at 1/4, -/// 121 at 1/8, 110 at 1/16, 105 at 1/32. -#[cfg(test)] -pub fn udr_queries(log_inv_rate: usize) -> usize { - assert!(log_inv_rate > 0, "log_inv_rate=0 (rate 1) has no soundness"); - let per_q = udr_per_query_bits_asymptotic(log_inv_rate); - (UDR_TARGET_BITS / per_q).ceil() as usize +/// Evaluate an E-valued multilinear table at an E-valued point, LSB-first. +fn mle_eval_ext(table: &[F192], point: &[F192]) -> F192 { + assert_eq!(table.len(), 1usize << point.len()); + let mut folded = table.to_vec(); + for &challenge in point { + let half = folded.len() / 2; + for row in 0..half { + let lo = folded[2 * row]; + let hi = folded[2 * row + 1]; + folded[row] = lo + challenge * (lo + hi); + } + folded.truncate(half); + } + folded[0] } -/// Level-ladder shape: per-level dims (index 0 = L0) plus the residual. -struct LadderShape { - log_inv_rates: Vec, - log_msg_cols: Vec, - log_num_interleaved: Vec, - k_levels: Vec, - yr_log_n: usize, +/// Parallel mirror of [`build_eq_table_ext`]: identical LSB-first doubling +/// recurrence, byte-identical output, with each level's independent +/// iterations fanned out across rayon threads once the level is large enough +/// to amortize dispatch. Structure copied from the extension-field layer's +/// `ring_switch::build_eq_parallel`. +pub fn build_eq_table_ext_parallel(point: &[F192]) -> Vec { + let mut out = primitives::alloc_uninit(1usize << point.len()); + build_eq_table_ext_seeded(point, F192::ONE, &mut out); + // SAFETY: the doubling recurrence initializes every table entry. + unsafe { primitives::assume_init(out) } } -/// Shared shape derivation behind [`LigeritoSecurityConfig::derive_config`]: [`LEVEL_K`]-bit level folds with the -/// rate index increasing by ≥ 1 per level, bumped further whenever the block -/// length couldn't accommodate `queries_at_rate(rate)` distinct queries. -fn derive_ladder_shape( - log_n: usize, - initial_k: usize, - log_inv_rate: usize, - queries_at_rate: &dyn Fn(usize) -> usize, -) -> Result { - if log_n <= initial_k { - return Err("log_n must be > initial_k".into()); - } - let mut shape = LadderShape { - log_inv_rates: vec![log_inv_rate], - log_msg_cols: vec![log_n - initial_k], - log_num_interleaved: vec![initial_k], - k_levels: vec![initial_k], - yr_log_n: 0, - }; - let mut n_running = log_n - initial_k; - let mut rate_running = log_inv_rate; - if (1usize << (n_running + rate_running)) < queries_at_rate(rate_running) { - return Err("L0 block_len < queries — log_n too small for chosen rate".into()); - } - while n_running > RESIDUAL_MAX_LOG { - let k = SUBSEQUENT_FOLDING_FACTORS.min(n_running); - let log_msg_cols_next = n_running - k; - let mut next_rate = rate_running + 1; - loop { - if (1usize << (log_msg_cols_next + next_rate)) >= queries_at_rate(next_rate) { - break; - } - next_rate += 1; - if next_rate > 20 { - return Err("could not find feasible level rate (level too deep)".into()); - } - } - shape.log_inv_rates.push(next_rate); - shape.log_msg_cols.push(log_msg_cols_next); - shape.log_num_interleaved.push(k); - shape.k_levels.push(k); - n_running -= k; - rate_running = next_rate; - } - if shape.k_levels.len() < 2 { - return Err("log_n too small: needs at least 2 fold levels".into()); - } - shape.yr_log_n = n_running; - Ok(shape) +trait EqTableSlot { + fn put(&mut self, value: F192); + unsafe fn get(&self) -> F192; } -// =================================================================== -// Security configuration schema -// =================================================================== -// -// Auditable, per-level spec for a Ligerito instance: query count, grinding -// bits, slack-from-Johnson, and the proximity-gap analysis the parameters -// were derived under. Designed to be (de)serializable so it can live in a -// TOML/JSON file alongside the prover/verifier code. - -/// Which proximity-gap analysis a level's parameters were derived under. -/// Determines which formulas the implementation should verify against the -/// declared (η, queries, grinding) tuple. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SoundnessRegime { - /// Unique decoding radius: γ = δ/2 (δ = 1 − ρ the code's relative - /// distance; no proximity-loss backoff). Theorem `ca-udr` of our paper's - /// Appendix C.3 (adapted from Ben-Sasson–Carmon–Haböck–Kopparty–Saraf - /// "On Proximity Gaps for Reed–Solomon Codes", 2025, Corollary 1.4): the - /// exceptional set is `a = γ·n + 1`, growing with the codeword length `n`, - /// so the proximity-gap term is recovered per level by `fold_grinding_bits` - /// rather than coming out 0. `eta` is `None` for this regime. - Udr, - /// Johnson radius with explicit slack `η` (γ = (1 − √ρ) − η) **with - /// out-of-domain binding**. Theorem 1.5 of the same paper gives the - /// proximity-gap exceptional set `a = O_ρ(n / η^5)`; the level's - /// `fold_grinding_bits` should be ≥ (target_bits − log₂(q/a)). - /// Binding to a single codeword of the (Johnson-bounded) interleaved list - /// is via `ood_samples` explicit multilinear OOD evaluations — except at - /// L0, where the opening's own post-commit random evaluation claim plays - /// the OOD role (union over the list, `L·μ/q`), so `ood_samples = 0`. - /// - /// Note there is deliberately no plain `Johnson` variant: without OOD - /// binding the query phase pays a union bound over the interleaved list - /// (≈ 19–52 bits here), which our query counts do not include. A config - /// claiming Johnson soundness without OOD accounting would be unsound. - JohnsonOod, +impl EqTableSlot for F192 { + #[inline(always)] + fn put(&mut self, value: F192) { + *self = value; + } + + #[inline(always)] + unsafe fn get(&self) -> F192 { + *self + } } -/// Where in a level's Fiat-Shamir transcript the grinding step lands. -/// Currently only one choice; reserved for future protocol variants. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum GrindingStep { - /// Grind happens after the level's Merkle root is observed but before - /// query positions are sampled. Standard FRI/STARK pattern. - PostCommitPreQueries, +impl EqTableSlot for std::mem::MaybeUninit { + #[inline(always)] + fn put(&mut self, value: F192) { + self.write(value); + } + + #[inline(always)] + unsafe fn get(&self) -> F192 { + // SAFETY: the doubling recurrence reads only the prefix initialized by + // earlier levels. + unsafe { self.assume_init_read() } + } } -/// Parameters for a single level in the multilevel Ligerito ladder. -/// L0 = the upstream `pcs::commit` output (reused, not re-committed); -/// L1 .. L_{r−1} are the level commits; the final residual `yr` block -/// is described separately in [`FinalBlockConfig`]. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct LigeritoLevelConfig { - /// PCS rate at this level: codeword expansion factor = 2^log_inv_rate. - pub log_inv_rate: usize, - /// Message dimension at this level (log of number of F128 columns in - /// the codeword). `log_msg_cols + log_inv_rate = log_2(block_len)`. - pub log_msg_cols: usize, - /// Log of lane width per Merkle leaf at this level. For L0 = `initial_k`; - /// for L_i (i ≥ 1) = the previous level's `k`. - pub log_num_interleaved: usize, - /// Number of sumcheck folds taken at this level. For L0 = `initial_k` - /// (the lane fold); for L_i (i ≥ 1) = the level fold k_{i−1}. - pub k: usize, - /// Which proximity-gap analysis the (eta, queries, grinding_bits) - /// tuple was derived under. Determines the formulas the implementation - /// validates against. - pub regime: SoundnessRegime, - /// Slack from the Johnson radius. Required for the `JohnsonOod` regime; - /// must be `None` for `Udr`. - pub eta: Option, - /// Proximity loss `ε*` for the UDR radius `γ = δ/2 − ε*` (our paper - /// App. C.3 / BCHKS25 Cor. 1.4); `0` in the shipped configs (full - /// unique-decoding radius δ/2, no backoff). Required for `Udr`; must be - /// `None` for `JohnsonOod`. The exceptional set is `a = γ·n + 1`, - /// length-dependent (see [`paper_thm_1_4_log_a`]). - #[serde(default)] - pub proximity_loss: Option, - /// Number of codeword position queries opened at this level (the FRI - /// query phase). Bounds the per-query soundness term `(1−γ)^Q`. - pub queries: usize, - /// **Query-phase** PoW grinding bits, ground post-commit/pre-queries - /// (see [`GrindingStep`]). Each bit substitutes for - /// ~1/log₂(1/(1−γ)) queries at this level. - pub grinding_bits: usize, - /// **Fold-challenge** PoW grinding bits, ground immediately before EACH - /// of this level's `k` fold challenges. Boosts the - /// proximity-gap term (which lives on the fold challenges): - /// `eps_pg + fold_grinding_bits ≥ target`. - #[serde(default)] - pub fold_grinding_bits: usize, - /// Out-of-domain samples taken right after this level's commit enters - /// the transcript (`JohnsonOod` only). Each binds the prover to a single - /// codeword of the interleaved list via a multilinear evaluation claim. - /// Must be 0 at L0 (bound by the opening's own post-commit evaluation - /// claim) and ≥ 1 at deeper `JohnsonOod` levels. - #[serde(default)] - pub ood_samples: usize, - /// Security target this level guarantees, post-grinding. - pub target_security_bits: usize, - /// Diagnostic — `log₂(q/a)` under the chosen regime. The implementation - /// should assert this matches the formula at startup, modulo rounding. - pub expected_eps_pg_bits: f64, - /// Diagnostic — `Q · log₂(1/(1−γ))`. Should be ≥ - /// `target_security_bits − grinding_bits`. - pub expected_eps_query_bits: f64, - /// Diagnostic — OOD binding bits (`JohnsonOod` only): - /// `s·(128 − log₂μ) − (2·log₂L − 1)` for explicit samples, or - /// `128 − log₂L − log₂μ` for the implicit L0 binding, where `L` is the - /// Johnson interleaved list size and `μ` the level's variable count. - #[serde(default)] - pub expected_eps_ood_bits: Option, +/// In-place seeded core of [`build_eq_table_ext_parallel`]: fills +/// `out[..2^point.len()]` with `seed * eq(point, .)`. +/// +/// Seeding folds a batching scalar into the table for free: every entry is +/// `seed` times a product of point factors, and field multiplication is +/// exact and associative, so the result equals the post-multiplied table +/// byte for byte while skipping one full multiply pass. `out` must have +/// length exactly `2^point.len()`; every slot is written before any is read, so +/// a reused scratch buffer is fine. +pub fn build_eq_table_ext_seeded_into(point: &[F192], seed: F192, out: &mut [F192]) { + build_eq_table_ext_seeded(point, seed, out); } -/// Descriptor for the final-residual block (`yr`) sent in the clear at the -/// end of the last fold level. It has no commit and no queries, so the -/// only meaningful parameter is its dimension. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct FinalBlockConfig { - /// `log_2(|yr|)` — number of F128 values sent in the clear. The last - /// fold level's sumcheck stops at this dim instead of folding to 1. - pub yr_log_n: usize, +pub(crate) fn build_eq_table_ext_seeded_uninit(point: &[F192], seed: F192, out: &mut [std::mem::MaybeUninit]) { + build_eq_table_ext_seeded(point, seed, out); } -/// Complete security spec for one Ligerito instance, covering a single -/// `(hash, m)` pair. Designed to round-trip cleanly via serde (TOML/JSON). -/// -/// **Validation invariants** (checked by [`Self::validate`]): -/// 1. `initial_k + Σ levels[1..].k + final_block.yr_log_n == log_n`. -/// 2. Each level's `expected_eps_pg_bits` is consistent with the declared -/// regime and `eta` (within tolerance). -/// 3. Each level's `expected_eps_query_bits ≥ target_security_bits − -/// grinding_bits` (queries cover what grinding doesn't). -/// 4. `eta` is `Some` iff regime ∈ {Johnson, JohnsonOod}; `None` for Udr. -/// 5. `log_msg_cols`, `log_num_interleaved`, `k` match the -/// level-shape constraint (each level's input dim equals the -/// previous level's `log_msg_cols`). -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct LigeritoSecurityConfig { - /// Block-encoder log size: m = log₂(witness bit count). - pub m: usize, - /// Packed-witness log dim (`= m − LOG_PACKING = m − 7`). - pub log_n: usize, - /// L0 lane fold. Must equal the upstream `PcsParams::log_batch_size` so - /// the L0 commit can be reused without re-committing. - pub initial_k: usize, - /// Round-by-round security target (bits): validate() asserts every error - /// term at every round (round-by-round soundness) clears at least this - /// much. Total security is the *minimum* over rounds — the notion that - /// governs Fiat-Shamir security (cf. Ethereum's `soundcalc`) — so there is - /// deliberately no whole-protocol union bound over terms. - pub target_security_bits: usize, - /// Identifier of the proximity-gap analysis used. Self-documents which - /// theorem the per-level parameters were derived from. Example: - /// `"ben_sasson_2025_thm_4_6"`. - pub analysis_version: String, - /// Field of the protocol. Example: `"f128"`. - pub field: String, - /// Hash function used by Merkle + FS sponge. Example: `"sha256"`. - pub hash: String, - /// Where in the per-level FS transcript grinding is placed. - pub grinding_step: GrindingStep, - /// Per-level parameters, in order L0, L1, L2, .... - pub levels: Vec, - /// Final residual block descriptor. - pub final_block: FinalBlockConfig, +fn build_eq_table_ext_seeded(point: &[F192], seed: F192, out: &mut [S]) { + use rayon::prelude::*; + let n = point.len(); + assert_eq!(out.len(), 1usize << n, "out must have length 2^point.len()"); + out[0].put(seed); + // Threshold below which rayon dispatch overhead beats the parallel work + // (same floor as the extension-field layer's `build_eq_parallel`). + const PAR_THRESHOLD: usize = 1 << 12; + for j in 0..n { + let r_j = point[j]; + let half = 1usize << j; + let (lo, hi_rest) = out.split_at_mut(half); + let hi = &mut hi_rest[..half]; + if half < PAR_THRESHOLD { + for (lo_x, hi_x) in lo.iter_mut().zip(hi.iter_mut()) { + // SAFETY: `lo` was initialized before this level starts. + let v = unsafe { lo_x.get() }; + let high = v * r_j; + hi_x.put(high); + lo_x.put(v + high); + } + } else { + lo.par_iter_mut().zip(hi.par_iter_mut()).for_each(|(lo_x, hi_x)| { + // SAFETY: `lo` was initialized before this level starts. + let v = unsafe { lo_x.get() }; + let high = v * r_j; + hi_x.put(high); + lo_x.put(v + high); + }); + } + } } -/// Default field size used for soundness analysis: `q = 2^128` (our F128). -const ANALYSIS_LOG_Q: f64 = 128.0; +/// Partially evaluate the multilinear extension of `evals` at the first +/// `rs.len()` (LSB) variables. Mirror of `ligerito::partial_eval_lsb`. +#[cfg(test)] +pub(crate) fn partial_eval_lsb_ext(evals: &[F192], rs: &[F192]) -> Vec { + let mut cur = evals.to_vec(); + for &r in rs { + let one_plus_r = F192::ONE + r; + let half = cur.len() / 2; + let mut next = Vec::with_capacity(half); + for i in 0..half { + next.push(cur[2 * i] * one_plus_r + cur[2 * i + 1] * r); + } + cur = next; + } + cur +} -/// Round a float to one decimal place. Used to round paper-predicted -/// soundness diagnostics so the generated TOMLs stay readable. -fn round1(x: f64) -> f64 { - (x * 10.0).round() / 10.0 +/// Mixed inner product `Σ_i b[i] · witness[i]` (E x K via `mul_base`). The +/// evaluation-claim `target` for a K-witness against an E-basis. +pub fn inner_product_base_ext(witness: &[F64], b: &[F192]) -> F192 { + use rayon::prelude::*; + assert_eq!(witness.len(), b.len()); + const PAR_THRESHOLD: usize = 4096; + if witness.len() < PAR_THRESHOLD { + return witness + .iter() + .zip(b.iter()) + .map(|(&w, &e)| e.mul_base(w)) + .fold(F192::ZERO, |a, v| a + v); + } + witness + .par_iter() + .zip(b.par_iter()) + .with_min_len(PAR_THRESHOLD / 4) + .map(|(&w, &e)| e.mul_base(w)) + .reduce(|| F192::ZERO, |a, v| a + v) } -/// Bit-level tolerance when comparing declared diagnostics -/// (`expected_eps_pg_bits` / `expected_eps_query_bits`) against the value -/// computed from the regime's formulas. Set generously enough that rounding -/// in the TOML doesn't cause spurious failures, but tightly enough that an -/// incorrect declaration of η, Q, or grinding can't slip through. -const PAPER_COMPAT_TOL_BITS: f64 = 0.6; - -/// Proximity-gap exceptional set for the list-decoding (Johnson) regime, per -/// our paper's Appendix C.3 (Theorem `ca-johnson`, adapted from BCHKS25 -/// Theorem 4.6). For a Reed–Solomon code of rate `ρ`, codeword length `n`, -/// and Johnson slack `η` (proximity radius `γ = 1 − √ρ − η`), the MCA error is -/// `a/|F|` with -/// -/// `a = [2(m+½)^5 + 3(m+½)·γ·ρ] / (3·ρ^{3/2}) · n + (m+½)/√ρ`, -/// -/// where `η = 1 − √ρ − γ` and `m = max(⌈√ρ/(2η)⌉, 3)`. Returns `log₂ a`. -/// -/// This is the per-fold-step MCA error, stated for a two-row interleaved word -/// (`C ∈ F^{2×n}`). The ℓ-round lane fold of a `2^ℓ`-interleaved word adds a -/// row-union factor via App. C.3's Lemma `mca-commutes`; see -/// [`paper_johnson_log_a`]. -fn paper_thm_ca_johnson_log_a(log_inv_rate: usize, eta: f64, log_msg_cols: usize) -> f64 { - let rho = (-(log_inv_rate as f64)).exp2(); - let sqrt_rho = rho.sqrt(); - let gamma = 1.0 - sqrt_rho - eta; - // m = ⌈√ρ/(2η)⌉ where η = 1−√ρ−γ, floored at 3. - let m_param = ((sqrt_rho / (2.0 * eta)).ceil() as usize).max(3) as f64; - let half = m_param + 0.5; - let half5 = half.powi(5); - let numerator = 2.0 * half5 + 3.0 * half * gamma * rho; - let denominator = 3.0 * rho.powf(1.5); - let n = ((log_msg_cols + log_inv_rate) as f64).exp2(); - let a = (numerator / denominator) * n + half / sqrt_rho; - a.log2() +#[inline] +fn log2_pow2(n: usize) -> usize { + assert!(n.is_power_of_two() && n > 0, "length must be a positive power of 2"); + n.trailing_zeros() as usize } -/// Johnson-regime proximity-gap `log₂ a` for a level, including the row-union -/// factor from our paper's Appendix C.3 (Lemma `mca-commutes`, "MCA commutes -/// with list decoding"). -/// -/// The base MCA error `ε = a_RLC/|F|` from [`paper_thm_ca_johnson_log_a`] is -/// stated for a two-row interleaved word (one fold step). Folding a -/// `2^ℓ`-interleaved word (ℓ = `log_num_interleaved`) over its ℓ lane-fold -/// rounds pays a row union: by the lemma, round `i` incurs `2^{ℓ-i}·ε`, so the -/// worst round (`i = 1`) pays the factor `2^{ℓ-1}` = (interleaving factor)/2. -/// We bind the per-level grinding to that worst round, returning -/// `log₂(2^{ℓ-1}·a_RLC) = log₂ a_RLC + (ℓ-1)`. -/// -/// `ℓ ≤ 1` (`L ≤ 2`) means no row union; the `(ℓ-1)` penalty clamps to 0. -fn paper_johnson_log_a( - log_inv_rate: usize, - eta: f64, - log_msg_cols: usize, - log_num_interleaved: usize, -) -> f64 { - let base = paper_thm_ca_johnson_log_a(log_inv_rate, eta, log_msg_cols); - // Row-union factor 2^{ℓ-1} (worst round i=1 of the ℓ-round lane fold), - // ℓ = log_num_interleaved. In bits: (ℓ-1), clamped ≥ 0. - let row_union_penalty = (log_num_interleaved as f64 - 1.0).max(0.0); - base + row_union_penalty +// =================================================================== +// Config reuse +// =================================================================== + +/// Derive `(ProverConfig, VerifierConfig)` for a K-witness of `2^log_n` F64 +/// elements, using the production 128-bit Johnson/OOD profile at +/// `m = log_n + LOG_PACKING`. +pub fn configs_for(log_n: usize) -> Result<(ProverConfig, VerifierConfig), String> { + configs_for_rate(log_n, crate::ligerito::LOG_INV_RATE_0) } -/// Per-query log₂(1/(1−γ)) under the Johnson regime: each query closes -/// `log_2(1/(1-γ))` bits of soundness against a γ-far adversary. -fn paper_per_query_bits(log_inv_rate: usize, eta: f64) -> f64 { - let rho = (-(log_inv_rate as f64)).exp2(); - let gamma = 1.0 - rho.sqrt() - eta; - (1.0 / (1.0 - gamma)).log2() +/// As [`configs_for`], with an explicit L0 inverse-rate logarithm. +pub fn configs_for_rate(log_n: usize, log_inv_rate: usize) -> Result<(ProverConfig, VerifierConfig), String> { + let sec = LigeritoSecurityConfig::derive_config_with_log_inv_rate(log_n + crate::LOG_PACKING, log_inv_rate)?; + sec.to_prover_verifier_configs() } -/// UDR proximity radius: the **maximum** allowed by our paper's App. C.3 -/// (Theorem `ca-udr`, BCHKS25 Cor. 1.4), whose valid range is -/// `[δ/3, δ/2 − 3/(δ·n)]`. We take the top of the range, -/// -/// `γ = δ/2 − 3/(δ·n) − ε*`, -/// -/// where `δ = 1 − ρ` is the code's relative minimum distance, -/// `n = 2^(log_msg_cols + log_inv_rate)` the codeword length, and `ε*` -/// (`proximity_loss`) optional extra slack below the maximum (`0` in shipped -/// configs → exactly the maximal radius). The `3/(δ·n)` backoff is the -/// theorem-mandated minimum and shrinks with the codeword length. -fn udr_gamma(log_inv_rate: usize, log_msg_cols: usize, proximity_loss: f64) -> f64 { - let rho = (-(log_inv_rate as f64)).exp2(); - let delta = 1.0 - rho; - let n = ((log_msg_cols + log_inv_rate) as f64).exp2(); - delta / 2.0 - 3.0 / (delta * n) - proximity_loss +// =================================================================== +// Commit: F64 message -> interleaved RS codeword -> Merkle root +// =================================================================== + +/// Public commitment for an `F64` message: the L0 Merkle root plus the shape +/// parameters needed to re-derive block lengths. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Commitment { + pub root: Hash, + /// log2 of the committed message length in F64 elements. + pub log_msg_len: usize, + /// Lanes per Merkle leaf (log2); must equal the Ligerito `initial_k`. + pub log_batch_size: usize, + pub log_inv_rate: usize, } -/// Per-query log₂(1/(1−γ)) under the UDR regime at the maximal radius -/// `γ = δ/2 − 3/(δ·n) − ε*` (see [`udr_gamma`]). -fn udr_per_query_bits(log_inv_rate: usize, log_msg_cols: usize, proximity_loss: f64) -> f64 { - let gamma = udr_gamma(log_inv_rate, log_msg_cols, proximity_loss); - (1.0 / (1.0 - gamma)).log2() +/// Prover-side state retained after commit for the opening phase. The message +/// itself is not stored; the caller retains it for opening. +pub struct ProverData { + pub codeword: Vec, + pub merkle_tree: Vec, } -/// Asymptotic (n → ∞) UDR per-query soundness at `γ = δ/2`, dropping the -/// finite-length `3/(δ·n)` backoff. Length-agnostic — used for ladder-shape -/// feasibility (and the test-only `udr_queries`); the per-level configs use the -/// n-aware [`udr_per_query_bits`]. The dropped backoff slightly *under*-counts -/// queries, but the per-level block-length check in `derive_config` (and the -/// `+5` feasibility padding) catch any shape that wouldn't hold the real, -/// n-aware query count. -fn udr_per_query_bits_asymptotic(log_inv_rate: usize) -> f64 { - let rho = (-(log_inv_rate as f64)).exp2(); - let gamma = (1.0 - rho) / 2.0; - (1.0 / (1.0 - gamma)).log2() +/// Fill `codeword` with `2^r` replicas of `msg`: the exact state after the +/// first `r` forward-NTT layers on the zero-padded coefficient vector +/// `[msg, 0, ..., 0]`. Pair with `forward_transform_*_from_layer(.., r)`. +fn replicate_message_fill_uninit(codeword: &mut [std::mem::MaybeUninit], msg: &[T]) { + use rayon::prelude::*; + let msg_len = msg.len(); + debug_assert!(codeword.len().is_multiple_of(msg_len)); + const COPY_CHUNK: usize = 1 << 16; + let copy = |dst: &mut [std::mem::MaybeUninit], src: &[T]| { + // SAFETY: source and destination are disjoint, have the same length, + // and each destination slot is written exactly once. + unsafe { std::ptr::copy_nonoverlapping(src.as_ptr(), dst.as_mut_ptr().cast(), dst.len()) }; + }; + if msg_len >= COPY_CHUNK { + codeword.par_chunks_mut(COPY_CHUNK).enumerate().for_each(|(i, dst)| { + let src_off = (i * COPY_CHUNK) % msg_len; + copy(dst, &msg[src_off..src_off + dst.len()]); + }); + } else { + for replica in codeword.chunks_mut(msg_len) { + copy(replica, msg); + } + } } -/// UDR proximity-gap exceptional set, per our paper's Appendix C.3 -/// (Theorem `ca-udr`, adapted from BCHKS25 Corollary 1.4): at proximity -/// radius `γ` (here the maximal `γ = δ/2 − 3/(δ·n)`; see [`udr_gamma`]) the -/// exceptional set is +/// Commit to an `F64` message: replicate it `2^log_inv_rate` times, run the interleaved additive +/// NTT over F_{2^64} from layer `log_inv_rate`, then Merkle-commit one leaf +/// per codeword position (= `2^log_batch_size` F64 = `2^log_batch_size * 8` +/// bytes per leaf). /// -/// `a = γ·n + 1`, -/// -/// where `n = 2^(log_msg_cols + log_inv_rate)` is the codeword length at this -/// level. The `log₂ a ≈ log₂(γ·n)` term therefore **grows with the codeword -/// length**, so larger witnesses give a smaller `eps_pg = 128 − log₂ a` and -/// need proportionally more `fold_grinding_bits` to hold a fixed target. -/// Callers add **no** row-union penalty in this regime: the unique-decoding -/// list has size 1, so (per Diamond and Gruen) MCA-commutes holds with error -/// ε directly, unlike the Johnson regime's `2^{ℓ-1}` factor. This replaced an -/// earlier length-independent `a ≤ 2/ε*` form, which did not match the paper's -/// stated bound. -fn paper_thm_1_4_log_a(log_inv_rate: usize, log_msg_cols: usize, proximity_loss: f64) -> f64 { - let gamma = udr_gamma(log_inv_rate, log_msg_cols, proximity_loss); - let n = ((log_msg_cols + log_inv_rate) as f64).exp2(); - (gamma * n + 1.0).log2() +/// `message.len()` must be a power of two `>= 2^log_batch_size`. +pub fn commit(message: &[F64], log_batch_size: usize, log_inv_rate: usize) -> (Commitment, ProverData) { + let log_msg_len = log2_pow2(message.len()); + assert!(log_msg_len >= log_batch_size, "message too small for log_batch_size"); + assert!(log_inv_rate >= 1, "log_inv_rate must be >= 1 for a non-trivial RS code"); + let log_dim = log_msg_len - log_batch_size; + let k_code = log_dim + log_inv_rate; + let num_ntts = 1usize << log_batch_size; + let n_positions = 1usize << k_code; + let codeword_len = n_positions * num_ntts; + + let mut codeword = primitives::alloc_uninit(codeword_len); + replicate_message_fill_uninit(&mut codeword, message); + // SAFETY: the replicate fill initializes every codeword element. + let mut codeword = unsafe { primitives::assume_init(codeword) }; + + // Optional phase timing (LIGERITO_TRACE): one env lookup per commit, no + // work when unset. + let trace = std::env::var_os("LIGERITO_TRACE").is_some(); + let t_ntt = std::time::Instant::now(); + tracing::info_span!("NTT", kind = "base encode", log_domain = k_code, lanes = num_ntts).in_scope(|| { + let ntt = AdditiveNttF64::standard(k_code); + ntt.forward_transform_interleaved_from_layer(&mut codeword, num_ntts, log_inv_rate); + }); + let ntt_elapsed = t_ntt.elapsed(); + let t_merkle = std::time::Instant::now(); + + // Merkle commitment, zero-copy over the codeword bytes. + // SAFETY: F64 is repr(transparent) over u64; a `[F64]` slice is therefore + // a contiguous little-endian u64 byte image (8 bytes each), identical to + // an explicit `to_le_bytes()` serialization on this (LE) target. The cast + // covers exactly `codeword.len() * size_of::()` initialized bytes. + let codeword_bytes: &[u8] = unsafe { + core::slice::from_raw_parts( + codeword.as_ptr() as *const u8, + codeword.len() * core::mem::size_of::(), + ) + }; + let merkle_tree = merkle::merkle_tree(codeword_bytes, n_positions); + let root = *merkle_tree.last().expect("merkle tree non-empty"); + if trace { + let k_code = pretty_integer(k_code); + let num_ntts = pretty_integer(num_ntts); + eprintln!( + "[lig-commit] k_code={k_code} lanes={num_ntts}: ntt = {:.4} s, merkle = {:.4} s", + ntt_elapsed.as_secs_f64(), + t_merkle.elapsed().as_secs_f64(), + ); + } + + ( + Commitment { + root, + log_msg_len, + log_batch_size, + log_inv_rate, + }, + ProverData { codeword, merkle_tree }, + ) } -/// Johnson-bound list size of the *interleaved* RS code at radius -/// `θ = 1 − √ρ − η`, in log₂. Independent of the interleaving factor. -/// -/// Interleaving preserves relative distance — `V^{⊙m}` has the base code's -/// distance `δ = 1 − ρ` — and only enlarges the alphabet (to `q^m`). The -/// Johnson bound depends solely on (distance, radius, alphabet size), so the -/// interleaved list size at any radius *below* the Johnson radius `1 − √ρ` -/// is bounded by the very same single-code Johnson list size -/// -/// `L_int ≤ L_base ≤ 1/(2·η·√ρ)`, -/// -/// with no dependence on `m` and, crucially, no `L_base^r` blow-up. -/// -/// The general GGR (Gopalan–Guruswami–Raghavendra, Thm 2.5) interleaved bound -/// `L_int ≤ C(b+r, r)·L_base^r` is only needed to push the list-decoding -/// radius *past* the Johnson bound toward `δ`. Ligerito deliberately sits at -/// `θ = 1 − √ρ − η`, strictly below the Johnson radius by slack `η > 0`, so -/// that regime never applies and the plain Johnson bound is both correct and -/// far tighter (it dominates GGR throughout the regime RS can reach). -fn johnson_interleaved_list_log2(log_inv_rate: usize, eta: f64) -> f64 { - debug_assert!( - eta > 0.0, - "η must be > 0 to stay strictly below the Johnson radius" - ); - let rho = (-(log_inv_rate as f64)).exp2(); - let sqrt_rho = rho.sqrt(); - let l_base = 1.0 / (2.0 * eta * sqrt_rho); - l_base.log2() +// =================================================================== +// Interleaved forward additive NTT over E with K-twiddles +// =================================================================== +// +// Deeper Ligerito levels RS-encode an E-valued (folded) witness on the SAME +// K-domain: the twiddles are F64, and each butterfly multiply is the mixed +// product `v.mul_base(twiddle)` (3 PMULL). Structure copied from +// `ntt::additive_ntt_f64`'s interleaved transform, with constants re-derived +// for 24-byte elements. + +pub(crate) fn forward_transform_interleaved_ext_from_layer( + ntt: &AdditiveNttF64, + data: &mut [F192], + num_ntts: usize, + start_layer: usize, +) { + assert!(num_ntts.is_power_of_two() && num_ntts > 0); + let n_total = data.len(); + assert_eq!(n_total % num_ntts, 0); + let log_d = log2_pow2(n_total / num_ntts); + assert!(log_d <= ntt.log_domain_size()); + assert!(start_layer <= log_d); + + forward_transform_interleaved_ext_parallel_from_layer(ntt, data, num_ntts, start_layer); } -/// OOD binding bits for a `JohnsonOod` level. `mu_vars` is the level's -/// multilinear variable count (`log_msg_cols + log_num_interleaved`). -/// -/// - `ood_samples ≥ 1` (explicit samples): the bad event is two distinct -/// list elements agreeing on all `s` random points of `F^μ` -/// (Schwartz–Zippel, total degree ≤ μ), union over pairs: -/// `bits = s·(128 − log₂ μ) − (2·log₂ L_int − 1)`. -/// - `ood_samples = 0` (L0's implicit binding): the opening's own evaluation -/// claim at a post-commit random point pins the prover to one claimed -/// value, so the union is over the list (not pairs): -/// `bits = 128 − log₂ L_int − log₂ μ`. -fn paper_ood_bits(log_inv_rate: usize, eta: f64, mu_vars: usize, ood_samples: usize) -> f64 { - let log2_l = johnson_interleaved_list_log2(log_inv_rate, eta); - let log2_mu = (mu_vars as f64).log2(); - if ood_samples == 0 { - ANALYSIS_LOG_Q - log2_l - log2_mu - } else { - ood_samples as f64 * (ANALYSIS_LOG_Q - log2_mu) - (2.0 * log2_l - 1.0) +/// Scalar reference for the E-valued interleaved forward NTT (test oracle and +/// small-input path). +fn forward_transform_interleaved_ext_scalar_from_layer( + ntt: &AdditiveNttF64, + data: &mut [F192], + num_ntts: usize, + start_layer: usize, +) { + let n_total = data.len(); + let log_d = log2_pow2(n_total / num_ntts); + + for layer in start_layer..log_d { + let num_blocks = 1usize << layer; + let block_size = 1usize << (log_d - layer); + let block_size_half = block_size >> 1; + let block_elems = block_size * num_ntts; + for block in 0..num_blocks { + let twiddle = ntt.twiddle(layer, block); + let block_start = block * block_elems; + for row in 0..block_size_half { + let off_top = block_start + row * num_ntts; + let off_bot = off_top + block_size_half * num_ntts; + for lane in 0..num_ntts { + let v = data[off_bot + lane]; + let new_u = data[off_top + lane] + v.mul_base(twiddle); + data[off_top + lane] = new_u; + data[off_bot + lane] = v + new_u; + } + } + } } } -impl LigeritoLevelConfig { - /// Compute the proximity-gap and per-query soundness bits this level is - /// expected to deliver under its declared regime. Returns - /// `(eps_pg_bits, eps_query_bits)` where: - /// eps_pg_bits = log₂(q/a) under the regime's threshold-a formula - /// eps_query_bits = Q · log₂(1/(1−γ)) - /// - /// Used by [`LigeritoSecurityConfig::validate`] to assert the declared - /// `expected_*_bits` diagnostics are consistent with the regime's - /// canonical formulas (i.e., the config is compatible with the paper). - pub fn paper_predicted_bits(&self) -> (f64, f64) { - match self.regime { - SoundnessRegime::JohnsonOod => { - let eta = self.eta.expect("JohnsonOod must have eta"); - // App. C.3 Lemma `mca-commutes`: the ℓ-round lane fold of a - // 2^ℓ-interleaved word (ℓ = log_num_interleaved) pays a - // row-union factor 2^{ℓ-i} at round i; the worst round (i=1) - // gives 2^{ℓ-1}, on top of the base ca-johnson MCA error. - let log_a = paper_johnson_log_a( - self.log_inv_rate, - eta, - self.log_msg_cols, - self.log_num_interleaved, +/// Parallel interleaved forward NTT over E, cache-blocked like the F64 twin: +/// top layers sweep the full buffer (fused two-layer passes, row-parallel), +/// deep layers run as cache-resident sub-NTTs in parallel. Constants are +/// derived from the actual F192 element size. +fn forward_transform_interleaved_ext_parallel_from_layer( + ntt: &AdditiveNttF64, + data: &mut [F192], + num_ntts: usize, + start_layer: usize, +) { + use rayon::prelude::*; + let n_total = data.len(); + let log_d = log2_pow2(n_total / num_ntts); + + // Target sub-group ~2 MB; each position is `num_ntts` F192 elements. + const TARGET_SUBGROUP_LOG_BYTES: usize = 21; + let log_bytes_per_position = log2_ceil(num_ntts * core::mem::size_of::()); + let target_log_positions = TARGET_SUBGROUP_LOG_BYTES.saturating_sub(log_bytes_per_position); + let cache_n_top = log_d.saturating_sub(target_log_positions); + + const PARALLEL_FLOOR_LOG_D: usize = 12; + const MIN_SUB_LOG: usize = 8; + let n_top = if log_d >= PARALLEL_FLOOR_LOG_D { + let want_subs_log = log2_pow2(rayon::current_num_threads().next_power_of_two()); + let max_n_top = log_d.saturating_sub(MIN_SUB_LOG); + cache_n_top.max(want_subs_log.min(max_n_top)) + } else { + cache_n_top + }; + if n_top == 0 || log_d < 8 { + forward_transform_interleaved_ext_scalar_from_layer(ntt, data, num_ntts, start_layer); + return; + } + + // Top layers: full-buffer sweeps, fusing two layers where possible. + let mut layer = start_layer.min(n_top); + while layer < n_top { + let num_blocks = 1usize << layer; + let block_size = 1usize << (log_d - layer); + let block_elems = block_size * num_ntts; + + if layer + 1 < n_top && block_size >= 4 { + let quarter = block_size >> 2; + for block in 0..num_blocks { + let t_outer = ntt.twiddle(layer, block); + let t_inner_a = ntt.twiddle(layer + 1, 2 * block); + let t_inner_b = ntt.twiddle(layer + 1, 2 * block + 1); + let start = block * block_elems; + butterfly_interleaved_ext_fused_2layer_par_rows( + &mut data[start..start + block_elems], + t_outer, + t_inner_a, + t_inner_b, + quarter, + num_ntts, ); - let eps_pg = ANALYSIS_LOG_Q - log_a; - // Per-query soundness WITHOUT a list union bound — the OOD - // binding (see `paper_ood_bits`) pins the prover to a single - // codeword of the interleaved list before queries are drawn. - let per_q = paper_per_query_bits(self.log_inv_rate, eta); - let eps_query = self.queries as f64 * per_q; - (eps_pg, eps_query) } - SoundnessRegime::Udr => { - // App. C.3 Thm `ca-udr` (BCHKS25 Cor. 1.4): a = γ·n + 1 for - // radius γ = δ/2 (ε* = 0, no backoff). - let proximity_loss = self - .proximity_loss - .expect("Udr regime must carry proximity_loss"); - // No row-union penalty in the unique-decoding regime: the list - // has size 1, so (per Diamond and Gruen) the MCA-commutes step - // holds with error ε directly — the Johnson regime's 2^{ℓ-1} - // row union is unnecessary. So eps_pg = 128 − log₂ a. - let log_a = - paper_thm_1_4_log_a(self.log_inv_rate, self.log_msg_cols, proximity_loss); - let eps_pg = ANALYSIS_LOG_Q - log_a; - let per_q = - udr_per_query_bits(self.log_inv_rate, self.log_msg_cols, proximity_loss); - let eps_query = self.queries as f64 * per_q; - (eps_pg, eps_query) + layer += 2; + } else { + let block_size_half = block_size >> 1; + for block in 0..num_blocks { + let t = ntt.twiddle(layer, block); + let start = block * block_elems; + butterfly_interleaved_ext_block_par_rows( + &mut data[start..start + block_elems], + t, + block_size_half, + num_ntts, + ); } + layer += 1; } } - /// OOD binding bits this level is expected to deliver (`JohnsonOod` - /// only; `None` for `Udr`, where the unique-decoding list has size 1 and - /// no binding step exists). See [`paper_ood_bits`]. - pub fn paper_predicted_ood_bits(&self) -> Option { - match self.regime { - SoundnessRegime::JohnsonOod => { - let eta = self.eta.expect("JohnsonOod must have eta"); - let mu = self.log_msg_cols + self.log_num_interleaved; - Some(paper_ood_bits(self.log_inv_rate, eta, mu, self.ood_samples)) + // Deep layers: parallel cache-resident sub-NTTs. + let sub_size_positions = 1usize << (log_d - n_top); + let sub_elems = sub_size_positions * num_ntts; + data.par_chunks_mut(sub_elems) + .enumerate() + .for_each(|(sub_idx, sub_data)| { + for layer in n_top.max(start_layer)..log_d { + let layer_in_sub = layer - n_top; + let num_blocks_in_sub = 1usize << layer_in_sub; + let block_size = 1usize << (log_d - layer); + let block_size_half = block_size >> 1; + let block_elems = block_size * num_ntts; + for block_in_sub in 0..num_blocks_in_sub { + let global_block = sub_idx * num_blocks_in_sub + block_in_sub; + let twiddle = ntt.twiddle(layer, global_block); + let block_start = block_in_sub * block_elems; + let block = &mut sub_data[block_start..block_start + block_elems]; + butterfly_interleaved_ext_block(block, twiddle, block_size_half, num_ntts); + } } - SoundnessRegime::Udr => None, - } + }); +} + +fn butterfly_interleaved_ext_block_par_rows(block: &mut [F192], twiddle: F64, block_size_half: usize, num_ntts: usize) { + use rayon::prelude::*; + const PARALLEL_ROW_THRESHOLD: usize = 1024; + if block_size_half < PARALLEL_ROW_THRESHOLD { + butterfly_interleaved_ext_block(block, twiddle, block_size_half, num_ntts); + return; } + let half_offset = block_size_half * num_ntts; + let (top, bot) = block.split_at_mut(half_offset); + top.par_chunks_mut(num_ntts) + .zip(bot.par_chunks_mut(num_ntts)) + .for_each(|(top_row, bot_row)| { + butterfly_ext_lanes(top_row, bot_row, twiddle); + }); } -impl LigeritoSecurityConfig { - /// Validate that the config is internally consistent and matches the - /// declared analysis. Returns the first violation found, if any. - pub fn validate(&self) -> Result<(), String> { - if self.log_n + 7 != self.m { - return Err(format!( - "log_n ({}) + LOG_PACKING (7) != m ({})", - self.log_n, self.m - )); - } +/// Fused 2-layer butterfly, row-parallel; see the F64 twin for the shape. +fn butterfly_interleaved_ext_fused_2layer_par_rows( + block: &mut [F192], + t_outer: F64, + t_inner_a: F64, + t_inner_b: F64, + quarter: usize, + num_ntts: usize, +) { + use rayon::prelude::*; + const PARALLEL_ROW_THRESHOLD: usize = 512; + let stride = quarter * num_ntts; + debug_assert_eq!(block.len(), 4 * stride); - // Level shape: initial_k + Σ k (L1+) + yr_log_n = log_n. - let levels_level_k_sum: usize = self.levels.iter().skip(1).map(|lv| lv.k).sum(); - let yr_log_n = self.final_block.yr_log_n; - if self.initial_k + levels_level_k_sum + yr_log_n != self.log_n { - return Err(format!( - "shape mismatch: initial_k ({}) + Σ k ({}) + yr_log_n ({}) = {} ≠ log_n ({})", - self.initial_k, - levels_level_k_sum, - yr_log_n, - self.initial_k + levels_level_k_sum + yr_log_n, - self.log_n, - )); - } + let do_one = |row_a: &mut [F192], row_b: &mut [F192], row_c: &mut [F192], row_d: &mut [F192]| { + butterfly_ext_fused_lanes(row_a, row_b, row_c, row_d, t_outer, t_inner_a, t_inner_b); + }; - // L0 must have k = initial_k and log_num_interleaved = initial_k. - let l0 = self - .levels - .first() - .ok_or_else(|| "empty levels".to_string())?; - if l0.k != self.initial_k { - return Err(format!( - "L0.k ({}) must equal initial_k ({})", - l0.k, self.initial_k - )); - } - if l0.log_num_interleaved != self.initial_k { - return Err(format!( - "L0.log_num_interleaved ({}) must equal initial_k ({})", - l0.log_num_interleaved, self.initial_k - )); + let (top_half, bot_half) = block.split_at_mut(2 * stride); + let (q1, q2) = top_half.split_at_mut(stride); + let (q3, q4) = bot_half.split_at_mut(stride); + + if quarter < PARALLEL_ROW_THRESHOLD { + for r in 0..quarter { + let off = r * num_ntts; + let (q1r, _) = q1[off..].split_at_mut(num_ntts); + let (q2r, _) = q2[off..].split_at_mut(num_ntts); + let (q3r, _) = q3[off..].split_at_mut(num_ntts); + let (q4r, _) = q4[off..].split_at_mut(num_ntts); + do_one(q1r, q2r, q3r, q4r); } + } else { + q1.par_chunks_mut(num_ntts) + .zip(q2.par_chunks_mut(num_ntts)) + .zip(q3.par_chunks_mut(num_ntts)) + .zip(q4.par_chunks_mut(num_ntts)) + .for_each(|(((row_a, row_b), row_c), row_d)| { + do_one(row_a, row_b, row_c, row_d); + }); + } +} - // Per-level checks. - let mut dim_in = self.log_n; - for (i, lv) in self.levels.iter().enumerate() { - // Shape: log_msg_cols + log_num_interleaved = dim_in. - if lv.log_msg_cols + lv.log_num_interleaved != dim_in { - return Err(format!( - "L{i}: log_msg_cols ({}) + log_num_interleaved ({}) ≠ input dim ({dim_in})", - lv.log_msg_cols, lv.log_num_interleaved - )); - } - - // eta presence matches regime. - match (lv.regime, lv.eta) { - (SoundnessRegime::Udr, Some(_)) => { - return Err(format!("L{i}: regime=udr but eta is set")); - } - (SoundnessRegime::JohnsonOod, None) => { - return Err(format!("L{i}: regime requires eta but eta is None")); - } - _ => {} - } - - // proximity_loss presence matches regime (UDR-only). - match (lv.regime, lv.proximity_loss) { - (SoundnessRegime::Udr, None) => { - return Err(format!("L{i}: regime=udr but proximity_loss is missing")); - } - (SoundnessRegime::Udr, Some(eps)) if eps < 0.0 => { - return Err(format!("L{i}: proximity_loss must be ≥ 0, got {eps}")); - } - (SoundnessRegime::JohnsonOod, Some(_)) => { - return Err(format!("L{i}: proximity_loss is only valid for regime=udr")); - } - _ => {} - } +#[inline] +fn butterfly_interleaved_ext_block(block: &mut [F192], twiddle: F64, block_size_half: usize, num_ntts: usize) { + let half_offset = block_size_half * num_ntts; + let (top, bot) = block.split_at_mut(half_offset); + for r in 0..block_size_half { + let off = r * num_ntts; + butterfly_ext_lanes(&mut top[off..off + num_ntts], &mut bot[off..off + num_ntts], twiddle); + } +} - // OOD samples match regime: UDR has no list, so no OOD; under - // JohnsonOod every level past L0 needs explicit samples, while - // L0 is bound by the opening's own post-commit evaluation claim. - match lv.regime { - SoundnessRegime::Udr if lv.ood_samples != 0 => { - return Err(format!( - "L{i}: regime=udr but ood_samples={} (unique decoding \ - has list size 1 — no OOD binding step exists)", - lv.ood_samples - )); - } - SoundnessRegime::JohnsonOod if i == 0 && lv.ood_samples != 0 => { - return Err(format!( - "L0: ood_samples={} but L0 is bound by the opening's \ - own evaluation claim (must be 0)", - lv.ood_samples - )); - } - SoundnessRegime::JohnsonOod if i > 0 && lv.ood_samples == 0 => { - return Err(format!( - "L{i}: regime=johnson_ood requires ood_samples ≥ 1 \ - past L0 (the query counts assume single-codeword \ - binding)" - )); - } - _ => {} +/// Butterfly all extension-field lanes in a row pair. The production layout +/// has eight interleaved NTTs, which the AVX-512 path transposes from eight +/// AoS `F192`s into three coefficient vectors before multiplying. +#[inline] +fn butterfly_ext_lanes(top: &mut [F192], bot: &mut [F192], twiddle: F64) { + debug_assert_eq!(top.len(), bot.len()); + #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] + { + let vectors = top.len() / 8; + // SAFETY: target features are enabled at compile time and every call + // addresses exactly eight readable and writable F192 values. + unsafe { + for i in 0..vectors { + butterfly_ext_lanes_avx512(top.as_mut_ptr().add(8 * i), bot.as_mut_ptr().add(8 * i), twiddle.0); } + } + for lane in 8 * vectors..top.len() { + let v = bot[lane]; + let new_u = top[lane] + v.mul_base(twiddle); + top[lane] = new_u; + bot[lane] = v + new_u; + } + } + #[cfg(not(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f")))] + { + for lane in 0..top.len() { + let v = bot[lane]; + let new_u = top[lane] + v.mul_base(twiddle); + top[lane] = new_u; + bot[lane] = v + new_u; + } + } +} - // OOD diagnostic matches regime + formula. - match (lv.regime, lv.expected_eps_ood_bits) { - (SoundnessRegime::Udr, Some(_)) => { - return Err(format!("L{i}: regime=udr but expected_eps_ood_bits is set")); - } - (SoundnessRegime::JohnsonOod, None) => { - return Err(format!( - "L{i}: regime=johnson_ood requires expected_eps_ood_bits" - )); - } - (SoundnessRegime::JohnsonOod, Some(declared)) => { - let pred = lv - .paper_predicted_ood_bits() - .expect("JohnsonOod has an OOD prediction"); - if (declared - pred).abs() > PAPER_COMPAT_TOL_BITS { - return Err(format!( - "L{i}: expected_eps_ood_bits ({declared:.2}) doesn't \ - match prediction ({pred:.2}); tolerance ±{:.2} bits.", - PAPER_COMPAT_TOL_BITS - )); - } - } - _ => {} +#[inline] +fn butterfly_ext_fused_lanes( + row_a: &mut [F192], + row_b: &mut [F192], + row_c: &mut [F192], + row_d: &mut [F192], + t_outer: F64, + t_inner_a: F64, + t_inner_b: F64, +) { + debug_assert_eq!(row_a.len(), row_b.len()); + debug_assert_eq!(row_a.len(), row_c.len()); + debug_assert_eq!(row_a.len(), row_d.len()); + #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] + { + let vectors = row_a.len() / 8; + // SAFETY: target features are enabled at compile time and every call + // addresses exactly eight elements in each of four disjoint rows. + unsafe { + for i in 0..vectors { + butterfly_ext_fused_lanes_avx512( + row_a.as_mut_ptr().add(8 * i), + row_b.as_mut_ptr().add(8 * i), + row_c.as_mut_ptr().add(8 * i), + row_d.as_mut_ptr().add(8 * i), + t_outer.0, + t_inner_a.0, + t_inner_b.0, + ); } + } + for lane in 8 * vectors..row_a.len() { + let mut a = row_a[lane]; + let mut b = row_b[lane]; + let mut c = row_c[lane]; + let mut d = row_d[lane]; + let new_a = a + c.mul_base(t_outer); + c += new_a; + a = new_a; + let new_b = b + d.mul_base(t_outer); + d += new_b; + b = new_b; + let new_a2 = a + b.mul_base(t_inner_a); + b += new_a2; + a = new_a2; + let new_c2 = c + d.mul_base(t_inner_b); + d += new_c2; + c = new_c2; + row_a[lane] = a; + row_b[lane] = b; + row_c[lane] = c; + row_d[lane] = d; + } + } + #[cfg(not(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f")))] + { + butterfly_ext_lanes(row_a, row_c, t_outer); + butterfly_ext_lanes(row_b, row_d, t_outer); + butterfly_ext_lanes(row_a, row_b, t_inner_a); + butterfly_ext_lanes(row_c, row_d, t_inner_b); + } +} - // Paper-compatibility: the declared expected_*_bits must agree - // with what the regime's formula predicts (within tolerance). - // Asserts the config was actually derived from the paper, not - // hand-waved into compliance. - let (pg_pred, q_pred) = lv.paper_predicted_bits(); - if (lv.expected_eps_pg_bits - pg_pred).abs() > PAPER_COMPAT_TOL_BITS { - return Err(format!( - "L{i}: expected_eps_pg_bits ({:.2}) doesn't match \ - {analysis} prediction ({:.2}); tolerance ±{:.2} bits. \ - Re-derive Q, eta, or grinding so the declared diagnostic \ - matches the formula.", - lv.expected_eps_pg_bits, - pg_pred, - PAPER_COMPAT_TOL_BITS, - analysis = self.analysis_version, - )); - } - if (lv.expected_eps_query_bits - q_pred).abs() > PAPER_COMPAT_TOL_BITS { - return Err(format!( - "L{i}: expected_eps_query_bits ({:.2}) doesn't match \ - {analysis} prediction ({:.2}); tolerance ±{:.2} bits.", - lv.expected_eps_query_bits, - q_pred, - PAPER_COMPAT_TOL_BITS, - analysis = self.analysis_version, - )); - } +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[derive(Clone, Copy)] +struct F192x8 { + c0: core::arch::x86_64::__m512i, + c1: core::arch::x86_64::__m512i, + c2: core::arch::x86_64::__m512i, +} - // Security: queries cover the gap left by grinding. - if lv.target_security_bits > lv.grinding_bits - && lv.expected_eps_query_bits + 1e-3 - < (lv.target_security_bits - lv.grinding_bits) as f64 - { - return Err(format!( - "L{i}: expected_eps_query_bits ({:.2}) < target ({}) - grinding ({}) = {}", - lv.expected_eps_query_bits, - lv.target_security_bits, - lv.grinding_bits, - lv.target_security_bits - lv.grinding_bits - )); - } +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "avx512f")] +unsafe fn load_f192x8_avx512(ptr: *const F192) -> F192x8 { + use core::arch::x86_64::*; + + // The 24 coefficients occupy three contiguous ZMM registers. Each result + // first selects from the first two registers, then fills its tail from the + // third register. + unsafe { + let p = ptr.cast::(); + let x0 = _mm512_loadu_si512(p.cast()); + let x1 = _mm512_loadu_si512(p.add(8).cast()); + let x2 = _mm512_loadu_si512(p.add(16).cast()); + + let c0_head = _mm512_permutex2var_epi64(x0, _mm512_set_epi64(0, 0, 15, 12, 9, 6, 3, 0), x1); + let c0_tail = _mm512_permutexvar_epi64(_mm512_set_epi64(5, 2, 0, 0, 0, 0, 0, 0), x2); + let c1_head = _mm512_permutex2var_epi64(x0, _mm512_set_epi64(0, 0, 0, 13, 10, 7, 4, 1), x1); + let c1_tail = _mm512_permutexvar_epi64(_mm512_set_epi64(6, 3, 0, 0, 0, 0, 0, 0), x2); + let c2_head = _mm512_permutex2var_epi64(x0, _mm512_set_epi64(0, 0, 0, 14, 11, 8, 5, 2), x1); + let c2_tail = _mm512_permutexvar_epi64(_mm512_set_epi64(7, 4, 1, 0, 0, 0, 0, 0), x2); + + F192x8 { + c0: _mm512_mask_mov_epi64(c0_head, 0xc0, c0_tail), + c1: _mm512_mask_mov_epi64(c1_head, 0xe0, c1_tail), + c2: _mm512_mask_mov_epi64(c2_head, 0xe0, c2_tail), + } + } +} - // Per-application proximity gap + fold-challenge grinding must - // reach target. (The pg bad event lives on the fold challenges, - // so only the fold grind — done before each fold challenge — - // boosts it; the query-phase grind does not.) - if lv.expected_eps_pg_bits + lv.fold_grinding_bits as f64 + 1e-3 - < lv.target_security_bits as f64 - { - return Err(format!( - "L{i}: expected_eps_pg_bits ({:.2}) + fold_grinding ({}) < target ({})", - lv.expected_eps_pg_bits, lv.fold_grinding_bits, lv.target_security_bits - )); - } +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "avx512f")] +unsafe fn store_f192x8_avx512(ptr: *mut F192, value: F192x8) { + use core::arch::x86_64::*; - // OOD binding must reach target on its own (no grind covers it; - // escalate ood_samples instead). - if let Some(ood) = lv.expected_eps_ood_bits - && ood + 1e-3 < lv.target_security_bits as f64 - { - return Err(format!( - "L{i}: expected_eps_ood_bits ({ood:.2}) < target ({}); \ - increase ood_samples", - lv.target_security_bits - )); - } + unsafe { + let x0_head = _mm512_permutex2var_epi64(value.c0, _mm512_set_epi64(10, 2, 0, 9, 1, 0, 8, 0), value.c1); + let x0_tail = _mm512_permutexvar_epi64(_mm512_set_epi64(0, 0, 1, 0, 0, 0, 0, 0), value.c2); - if lv.target_security_bits < self.target_security_bits { - return Err(format!( - "L{i}: target_security_bits ({}) < global target ({})", - lv.target_security_bits, self.target_security_bits - )); - } + let x1_head = _mm512_permutex2var_epi64(value.c0, _mm512_set_epi64(5, 0, 12, 4, 0, 11, 3, 0), value.c1); + let x1_tail = _mm512_permutexvar_epi64(_mm512_set_epi64(0, 4, 0, 0, 3, 0, 0, 2), value.c2); - // Advance dim_in for next level: subtract k (the folds at this level). - dim_in -= lv.k; - } + let x2_head = _mm512_permutex2var_epi64(value.c0, _mm512_set_epi64(0, 15, 7, 0, 14, 6, 0, 13), value.c1); + let x2_tail = _mm512_permutexvar_epi64(_mm512_set_epi64(7, 0, 0, 6, 0, 0, 5, 0), value.c2); - if dim_in != yr_log_n { - return Err(format!( - "after consuming all levels, dim_in ({dim_in}) ≠ yr_log_n ({yr_log_n})" - )); - } + let p = ptr.cast::(); + _mm512_storeu_si512(p.cast(), _mm512_mask_mov_epi64(x0_head, 0x24, x0_tail)); + _mm512_storeu_si512(p.add(8).cast(), _mm512_mask_mov_epi64(x1_head, 0x49, x1_tail)); + _mm512_storeu_si512(p.add(16).cast(), _mm512_mask_mov_epi64(x2_head, 0x92, x2_tail)); + } +} - // Round-by-round soundness: each error term at each round is checked - // against `target_security_bits` in the per-level loop above. Total - // security is the minimum over rounds (the Fiat-Shamir-relevant notion; - // cf. Ethereum's `soundcalc`), so there is intentionally no - // whole-protocol union bound summed across terms. - Ok(()) - } - - /// Derive THE security config at witness size `m`: Udr regime, rate - /// `2^-LOG_INV_RATE_0`, ε* = 1e-3, [`SECURITY_BITS`] bits per round under - /// **round-by-round soundness** — every error term (pg + fold grinding, - /// query + query grinding) clears the target individually, and the - /// protocol's security is the *minimum* over rounds — the notion that - /// governs Fiat-Shamir security (cf. Ethereum's `soundcalc`), not a - /// whole-protocol union bound over terms. - pub fn derive_config(m: usize) -> Result { - let target_bits = SECURITY_BITS; - let log_inv_rate = LOG_INV_RATE_0; - // Query-phase grinding trades prover PoW for query count (see - // [`QUERY_GRINDING_BITS`]): 120-bit rounds with 18 bits ground, so - // queries cover 102. - let query_grind: usize = QUERY_GRINDING_BITS; - let log_n = m - .checked_sub(crate::LOG_PACKING) - .ok_or_else(|| format!("m ({m}) < LOG_PACKING (7)"))?; - let initial_k = INITIAL_FOLDING_FATOR; - - // Length-agnostic per-query estimate for ladder-shape feasibility - // (the per-level codeword length `n` is not known until the shape is - // fixed): the asymptotic γ = δ/2; the actual per-level config below - // uses the n-aware `udr_per_query_bits`. - let per_query_bits_feas = udr_per_query_bits_asymptotic; - - // Shape derivation needs per-level query counts for block-length - // feasibility before the level count (and hence the exact per-term - // target) is known. Use a conservative target of target_bits + 5 - // (≥ log₂(3 terms · 10 levels)); the final counts are ≤ this. - let t_feas = target_bits as f64 + 5.0; - let queries_feas = |rate: usize| -> usize { - ((t_feas - query_grind as f64).max(1.0) / per_query_bits_feas(rate)).ceil() as usize - }; - let shape = derive_ladder_shape(log_n, initial_k, log_inv_rate, &queries_feas)?; - let n_levels = shape.log_inv_rates.len(); - - // Round-by-round target: every error term (pg, query, ood) at every - // round must individually clear `target_bits`. Round-by-round soundness - // — the notion that governs the Fiat-Shamir security of the IOP — is the - // *minimum* security level over rounds, not the sum, so there is - // deliberately NO `log₂(#terms)` union-bound headroom. This matches the - // convention Ethereum's `soundcalc` uses for hash-based zkEVM IOPs - // (total security = min over rounds). It also keeps the proximity-gap - // fold grinding (especially L0's, the dominant prover cost) at the - // round-by-round minimum rather than paying ~4 bits of union slack that - // buys nothing. - let t = target_bits as f64; - - let mut levels = Vec::with_capacity(n_levels); - for i in 0..n_levels { - let rate = shape.log_inv_rates[i]; - let cols = shape.log_msg_cols[i]; - let ilv = shape.log_num_interleaved[i]; - // Actual per-level per-query bits: n-aware (maximal radius). - let per_q = udr_per_query_bits(rate, cols, UDR_PROXIMITY_LOSS); - let queries = ((t - query_grind as f64).max(1.0) / per_q).ceil() as usize; - if queries > (1usize << (cols + rate)) { - return Err(format!( - "L{i}: {queries} queries exceed block length 2^{}", - cols + rate - )); - } - let eps_query = queries as f64 * per_q; - - // No row-union penalty in the unique-decoding regime (list size - // 1): per Diamond and Gruen, MCA-commutes holds with error ε - // directly (vs the Johnson regime's 2^{ℓ-1} factor). - let _ = ilv; - let eps_pg = ANALYSIS_LOG_Q - paper_thm_1_4_log_a(rate, cols, UDR_PROXIMITY_LOSS); - let (regime, eta, proximity_loss, ood_samples, eps_ood) = - (SoundnessRegime::Udr, None, Some(UDR_PROXIMITY_LOSS), 0usize, None); - let fold_grinding_bits = (t - eps_pg).ceil().max(0.0) as usize; - - levels.push(LigeritoLevelConfig { - log_inv_rate: rate, - log_msg_cols: cols, - log_num_interleaved: ilv, - k: shape.k_levels[i], - regime, - eta, - proximity_loss, - queries, - grinding_bits: query_grind, - fold_grinding_bits, - ood_samples, - target_security_bits: target_bits, - expected_eps_pg_bits: round1(eps_pg), - expected_eps_query_bits: round1(eps_query), - expected_eps_ood_bits: eps_ood, - }); - } +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx512f")] +unsafe fn mul_base_f192x8_avx512(value: F192x8, twiddle: u64) -> F192x8 { + use core::arch::x86_64::*; - let analysis_version = "no_row_union_over_ben_sasson_2025_cor_1_4"; - let cfg = Self { - m, - log_n, - initial_k, - target_security_bits: target_bits, - analysis_version: analysis_version.into(), - field: "f128".into(), - hash: "sha256".into(), - grinding_step: GrindingStep::PostCommitPreQueries, - levels, - final_block: FinalBlockConfig { - yr_log_n: shape.yr_log_n, - }, - }; - cfg.validate()?; - Ok(cfg) + #[inline] + #[target_feature(enable = "vpclmulqdq", enable = "avx512f")] + unsafe fn mul_coeff(value: __m512i, tw: __m512i, r: __m512i) -> __m512i { + let even = _mm512_clmulepi64_epi128::<0x00>(value, tw); + let even_t = _mm512_clmulepi64_epi128::<0x01>(even, r); + let even_u = _mm512_clmulepi64_epi128::<0x01>(even_t, r); + let even = _mm512_xor_si512(_mm512_xor_si512(even, even_t), even_u); + + let odd = _mm512_clmulepi64_epi128::<0x11>(value, tw); + let odd_t = _mm512_clmulepi64_epi128::<0x01>(odd, r); + let odd_u = _mm512_clmulepi64_epi128::<0x01>(odd_t, r); + let odd = _mm512_shuffle_epi32::<0x4e>(_mm512_xor_si512(_mm512_xor_si512(odd, odd_t), odd_u)); + _mm512_mask_blend_epi64(0xaa, even, odd) } - /// Build the protocol configuration from this security config. - pub fn to_config(&self) -> Result { - self.validate()?; - let level_ks: Vec = self - .levels - .iter() - .skip(1) - .map(|lv| lv.k) - .collect(); - Ok(LigeritoConfig { - log_inv_rates: self.levels.iter().map(|lv| lv.log_inv_rate).collect(), - level_steps: level_ks.len(), - initial_log_msg_cols: self.levels[0].log_msg_cols, - initial_log_num_interleaved: self.initial_k, - initial_k: self.initial_k, - level_log_msg_cols: self - .levels - .iter() - .skip(1) - .map(|lv| lv.log_msg_cols) - .collect(), - level_ks, - queries: self.levels.iter().map(|lv| lv.queries).collect(), - grinding_bits: self.levels.iter().map(|lv| lv.grinding_bits).collect(), - fold_grinding_bits: self - .levels - .iter() - .map(|lv| lv.fold_grinding_bits) - .collect(), - ood_samples: self.levels.iter().map(|lv| lv.ood_samples).collect(), - }) + unsafe { + let tw = _mm512_set1_epi64(twiddle as i64); + let r = _mm512_set1_epi64(0x1b); + F192x8 { + c0: mul_coeff(value.c0, tw, r), + c1: mul_coeff(value.c1, tw, r), + c2: mul_coeff(value.c2, tw, r), + } } } -// =================================================================== -// Proof -// =================================================================== - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct LevelProof { - /// One row per query, each of `num_interleaved` F128 entries. Rows are - /// emitted in **sorted** query-position order so they align with the - /// merkle multi-proof. - pub opened_rows: Vec>, - /// Single octopus multi-proof shared across all queries at this level. - pub merkle_proof: Vec, +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "avx512f")] +unsafe fn butterfly_f192x8_avx512(top: &mut F192x8, bot: &mut F192x8, twiddle: u64) { + use core::arch::x86_64::_mm512_xor_si512; + + unsafe { + let product = mul_base_f192x8_avx512(*bot, twiddle); + top.c0 = _mm512_xor_si512(top.c0, product.c0); + top.c1 = _mm512_xor_si512(top.c1, product.c1); + top.c2 = _mm512_xor_si512(top.c2, product.c2); + bot.c0 = _mm512_xor_si512(bot.c0, top.c0); + bot.c1 = _mm512_xor_si512(bot.c1, top.c1); + bot.c2 = _mm512_xor_si512(bot.c2, top.c2); + } } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct FinalProof { - /// Same sorted-by-position convention as [`LevelProof`]. - pub opened_rows: Vec>, - pub merkle_proof: Vec, +/// Eight F192 butterflies with a shared base-field twiddle. +/// +/// # Safety +/// Requires VPCLMULQDQ + AVX-512F; `top` and `bot` must each address eight +/// readable and writable F192 values. +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx512f")] +unsafe fn butterfly_ext_lanes_avx512(top: *mut F192, bot: *mut F192, twiddle: u64) { + unsafe { + let mut u = load_f192x8_avx512(top); + let mut v = load_f192x8_avx512(bot); + butterfly_f192x8_avx512(&mut u, &mut v, twiddle); + store_f192x8_avx512(top, u); + store_f192x8_avx512(bot, v); + } } -/// The Ligerito opening object: ONLY the hash-bearing hint data (opened rows -/// plus Merkle multi-proofs), which the verifier checks against roots rather -/// than observes. Every scalar the verifier must bind (sumcheck messages, -/// OOD values, the final `yr`, the level roots, the PoW nonces) rides the -/// shared transcript stream via `add_scalar`/`next_scalar`/`grind`, bound at -/// its protocol point like every other transmitted value. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct LigeritoProof { - pub initial_proof: LevelProof, - pub level_proofs: Vec, - pub final_proof: FinalProof, +/// Fused two-layer butterfly over eight F192 lanes. Four rows stay in SoA ZMM +/// form across both layers, so each row pays the AoS transpose only once. +/// +/// # Safety +/// Requires VPCLMULQDQ + AVX-512F; each pointer must address eight readable +/// and writable F192 values, and the four regions must be disjoint. +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx512f")] +unsafe fn butterfly_ext_fused_lanes_avx512( + row_a: *mut F192, + row_b: *mut F192, + row_c: *mut F192, + row_d: *mut F192, + t_outer: u64, + t_inner_a: u64, + t_inner_b: u64, +) { + unsafe { + let mut a = load_f192x8_avx512(row_a); + let mut b = load_f192x8_avx512(row_b); + let mut c = load_f192x8_avx512(row_c); + let mut d = load_f192x8_avx512(row_d); + butterfly_f192x8_avx512(&mut a, &mut c, t_outer); + butterfly_f192x8_avx512(&mut b, &mut d, t_outer); + butterfly_f192x8_avx512(&mut a, &mut b, t_inner_a); + butterfly_f192x8_avx512(&mut c, &mut d, t_inner_b); + store_f192x8_avx512(row_a, a); + store_f192x8_avx512(row_b, b); + store_f192x8_avx512(row_c, c); + store_f192x8_avx512(row_d, d); + } } // =================================================================== -// Multilinear helpers +// LCH novel-basis evaluations over K (mirror of ligerito's extension-field block) // =================================================================== - -/// Multilinear extension of `evals` at the boolean cube of dimension `n`, -/// LSB-first indexing: `eval(b_0, …, b_{n-1}) = evals[b_0 + 2·b_1 + …]`. -/// -/// Partially evaluate at the first `k` variables (the LSB end): given -/// challenges `rs ∈ F^k`, returns the length-`2^{n-k}` table -/// `f(rs[0], …, rs[k-1], x_k, …, x_{n-1})`. -/// -/// Matches [`build_eq`] LSB-first convention (and bolt-rs's -/// `partial_eval` Julia convention). -#[cfg(test)] -pub(crate) fn partial_eval_lsb(evals: &[F128], rs: &[F128]) -> Vec { - let mut cur = evals.to_vec(); - for &r in rs { - let one_plus_r = F128::ONE + r; - let half = cur.len() / 2; - // Pair (cur[2i], cur[2i+1]) collapses to cur[2i]·(1+r) + cur[2i+1]·r. - // LSB-first ⇒ adjacent pairs are bit_0 = 0 vs 1. - let mut next = Vec::with_capacity(half); - for i in 0..half { - next.push(cur[2 * i] * one_plus_r + cur[2 * i + 1] * r); - } - cur = next; - } - cur -} - -// =================================================================== -// LCH novel-basis evaluations (ported from bolt-rs `fft.rs`) -// =================================================================== -// -// Same subspace-polynomial recurrence `s_{i+1}(x) = s_i(x)² + s_i(v_i)·s_i(x)` -// as Flock's `AdditiveNttF128`, but we expose the evaluation at an arbitrary -// point — which the NTT doesn't currently surface publicly. Standard basis only -// (v_i = 2^i, embedded as `F128::new(1 << i, 0)`). +// +// The subspace-polynomial recurrence runs entirely over the K evaluation +// domain (F64 values); results are lifted into E with `mul_base` only where +// they scale E-accumulators. Standard basis only (v_i = x^i = F64(1 << i)). #[inline] -fn next_s(s: F128, s_at_root: F128) -> F128 { +fn next_s(s: F64, s_at_root: F64) -> F64 { s * s + s_at_root * s } -/// `sks_vks[k] = s_k(v_k)` for `k = 0..=log_n`. Length `log_n + 1`. -/// Only depends on `log_n`, so callers cache. -pub fn eval_sk_at_vks(log_n: usize) -> Vec { - let mut sks_vks = vec![F128::ZERO; log_n + 1]; - sks_vks[0] = F128::ONE; +/// `sks_vks[k] = s_k(v_k)` for `k = 0..=log_n`, over K. Mirror of +/// `ligerito::eval_sk_at_vks`. Public for the recursion harness, which dumps +/// these vanishing-polynomial values as guest hints (base-field, embedded into +/// the tower with both extension limbs zero). +pub fn eval_sk_at_vks(log_n: usize) -> Vec { + let mut sks_vks = vec![F64::ZERO; log_n + 1]; + sks_vks[0] = F64::ONE; if log_n == 0 { return sks_vks; } - let mut layer: Vec = (1..=log_n).map(|i| F128::new(1u64 << i, 0)).collect(); + let mut layer: Vec = (1..=log_n).map(|i| F64(1u64 << i)).collect(); let mut cur_len = log_n; for i in 0..log_n { for j in 0..cur_len { @@ -1130,20 +905,16 @@ pub fn eval_sk_at_vks(log_n: usize) -> Vec { sks_vks } -/// Write into `basis` the **normalized** LCH novel-basis polynomials -/// `X̂_j(x) = Π_{k: bit_k(j)=1} Ŵ_k(x)` for `j ∈ [0, 2^log_n)`, each scaled by -/// `alpha`. `Ŵ_k = s_k / s_k(v_k)` is normalized to match Flock's NTT twiddles. -/// -/// `sks_at_x` is a scratch buffer of length `≥ log_n`. `sks_vks` is from -/// [`eval_sk_at_vks`]; `inv_sks_vks[k] = sks_vks[k].inv()` precomputed once -/// across many queries. +/// Write into `basis` the normalized LCH novel-basis polynomials evaluated at +/// `x` (a K point), each scaled by the E-value `alpha`. The `sks_at_x` +/// recurrence stays in K; the basis expansion lifts into E via `mul_base`. fn evaluate_scaled_basis_inplace( - sks_at_x: &mut [F128], - basis: &mut [F128], - sks_vks: &[F128], - inv_sks_vks: &[F128], - x: F128, - alpha: F128, + sks_at_x: &mut [F64], + basis: &mut [F192], + sks_vks: &[F64], + inv_sks_vks: &[F64], + x: F64, + alpha: F192, ) { let log_n = basis.len().trailing_zeros() as usize; debug_assert_eq!(basis.len(), 1 << log_n); @@ -1155,7 +926,7 @@ fn evaluate_scaled_basis_inplace( for i in 1..log_n { sks_at_x[i] = next_s(sks_at_x[i - 1], sks_vks[i - 1]); } - // Normalize: Ŵ_i(x) = s_i(x) / s_i(v_i) + // Normalize: W-hat_i(x) = s_i(x) / s_i(v_i) for i in 0..log_n { sks_at_x[i] *= inv_sks_vks[i]; } @@ -1166,197 +937,119 @@ fn evaluate_scaled_basis_inplace( let s_at_x = sks_at_x[k]; let current_len = 1 << k; for i in 0..current_len { - basis[i + current_len] = s_at_x * basis[i]; + basis[i + current_len] = basis[i].mul_base(s_at_x); } } } // =================================================================== -// induce_sumcheck_poly — the per-level basis-poly builder. +// induce_sumcheck_poly: dense path (base-field rows at L0, E rows deeper) // =================================================================== // -// Given Q opened rows of the previous commitment at query positions and the -// post-partial-eval challenges `v_challenges`, builds: -// basis_poly[j] = Σ_i α^i · Ŵ_j(q_i_field) -// enforced_sum = Σ_i α^i · ⟨row_i, eq(v_challenges, ·)⟩ -// -// The verifier reconstructs both independently from public inputs and checks -// the sumcheck claim Σ_j f(j) · basis_poly[j] = enforced_sum at the residual. - -/// Compute just the `enforced_sum` half of [`induce_sumcheck_poly`]: -/// `enforced_sum = Σ_i eq(α, i_bin) · ⟨opened_rows[i], eq(v_challenges, ·)⟩` -/// Cheap: O(num_queries × num_interleaved). Verifier needs this at level -/// intro time (before residual challenges are known). -pub fn induce_sumcheck_enforced_sum( - opened_rows: &[Vec], - v_challenges: &[F128], - queries: &[usize], - alpha: &[F128], -) -> F128 { - assert_eq!(opened_rows.len(), queries.len()); - let eq = build_eq(v_challenges); - let n_queries = queries.len(); - let alpha_weights: Vec = if n_queries == 0 { - Vec::new() - } else { - build_eq(alpha).into_iter().take(n_queries).collect() - }; - let mut sum = F128::ZERO; - for (i, row) in opened_rows.iter().enumerate() { - debug_assert_eq!(row.len(), eq.len()); - let dot: F128 = row - .iter() - .zip(eq.iter()) - .map(|(&r, &e)| r * e) - .fold(F128::ZERO, |a, v| a + v); - sum += alpha_weights[i] * dot; - } - sum -} - -/// `⌈log₂ n⌉`. Number of bits needed to index `n` items. Used to size the -/// per-level `alpha` slice for the eq-tensor basis-induction combination. -#[inline] -pub fn log2_ceil(n: usize) -> usize { - if n <= 1 { - 0 - } else { - (n - 1).ilog2() as usize + 1 - } -} - -/// **Succinct** evaluator for the induced basis poly's MLE at residual points. -/// Replaces `induce_sumcheck_poly` + `partial_eval_lsb` in the verifier: -/// instead of materializing the dense `2^log_msg_cols` basis_poly, evaluates -/// its MLE directly using the closed-form identity: -/// `MLE(basis_poly)(p) = Σ_i α^i · Π_k (1 + p[k] · (1 + Ŵ_k(q_i)))` -/// where each `q_i` is the field embedding of `queries[i]`. -/// -/// `ris_for_basis` is the fixed prefix of the residual point (the ris range -/// that would have been passed to `partial_eval_lsb(basis_poly, ris_for_basis)`). -/// Length must be `log_msg_cols - yr_log_n`. The function returns evaluations -/// at `2^yr_log_n` points: `ris_for_basis ++ y_bits` for `y ∈ [0, 2^yr_log_n)`. -/// -/// Cost: O(num_queries × yr_log_n × 2^yr_log_n + num_queries × log_msg_cols), -/// vs the dense path's O(num_queries × log_msg_cols × 2^log_msg_cols). At m=30 -/// L0 with 221 queries, log_msg_cols=17, yr_log_n=4: ~18k ops vs ~500M ops. -pub fn induce_sumcheck_evaluate_at_residual( +// The sparse transposed-NTT fast path plus the auto dispatch that selects it +// at L0 lives in the "Transposed-NTT fast path" section below, mirroring the +// original's strategy split. + +/// Level-0 induce: opened rows are F64. `basis_poly[j] = Σ_i eq(α, i) · +/// W-hat_j(q_i)`, `enforced_sum = Σ_i eq(α, i) · ` +/// with the row dot done via `mul_base`. Mirror of the dense +/// `ligerito::induce_sumcheck_poly` (per-thread chunked accumulation). +pub(crate) fn induce_sumcheck_poly_base( log_msg_cols: usize, - sks_vks: &[F128], + sks_vks: &[F64], + opened_rows: &[Vec], + v_challenges: &[F192], queries: &[usize], - alpha: &[F128], - ris_for_basis: &[F128], - yr_log_n: usize, -) -> Vec { + alpha: &[F192], +) -> (Vec, F192) { use rayon::prelude::*; - assert_eq!(ris_for_basis.len() + yr_log_n, log_msg_cols); + let n = 1usize << log_msg_cols; let n_queries = queries.len(); - let yr_len = 1usize << yr_log_n; + assert_eq!(opened_rows.len(), n_queries); + debug_assert_eq!( + v_challenges.len(), + opened_rows + .first() + .map(|r| r.len().trailing_zeros() as usize) + .unwrap_or(0) + ); + + let eq = build_eq_table_ext(v_challenges); - // Per-query weights are the eq-tensor coefficients `eq(α, i_bin)` for - // `i ∈ {0,1}^{⌈log₂ n_queries⌉}` (LSB-first), padded with zeros for - // indices ≥ n_queries. Replaces the legacy α^i Vandermonde scheme; - // soundness bound goes from `Q/q` (univariate S-Z) to `⌈log₂ Q⌉/q` - // (multilinear S-Z), matching the rest of the multilinear protocol. - let alpha_pows: Vec = if n_queries == 0 { + let alpha_pows: Vec = if n_queries == 0 { Vec::new() } else { - let table = build_eq(alpha); + let table = build_eq_table_ext(alpha); debug_assert!(table.len() >= n_queries); table.into_iter().take(n_queries).collect() }; - let inv_sks_vks: Vec = sks_vks + let inv_sks_vks: Vec = sks_vks .iter() - .map(|&v| if v.is_zero() { F128::ZERO } else { v.inv() }) + .map(|&v| if v.is_zero() { F64::ZERO } else { v.inv() }) .collect(); - let prefix_len = ris_for_basis.len(); + let n_threads = rayon::current_num_threads().max(1); + let chunk_size = (n_queries + n_threads - 1) / n_threads.max(1); - // Per-query precomputation: Ŵ_k(q) for all k, then split into prefix - // product (fixed scalar) and suffix Ŵ values (varied per y). - struct PerQuery { - prefix_prod: F128, - suffix_w: Vec, // length = yr_log_n - } - let compute_query = |&q: &usize| -> PerQuery { - let q_field = F128::new(q as u64, 0); - // Compute s_k(q_field) recursively, then normalize by 1/s_k(v_k). - let mut sks_at_x = Vec::with_capacity(log_msg_cols.max(1)); - if log_msg_cols > 0 { - sks_at_x.push(q_field); - for k in 1..log_msg_cols { - sks_at_x.push(next_s(sks_at_x[k - 1], sks_vks[k - 1])); - } - for k in 0..log_msg_cols { - sks_at_x[k] *= inv_sks_vks[k]; + let partials: Vec<(Vec, F192)> = (0..n_threads) + .into_par_iter() + .map(|t| { + let start = t * chunk_size; + let end = (start + chunk_size).min(n_queries); + if start >= end { + return (vec![F192::ZERO; n], F192::ZERO); } - } - // Prefix product: Π_{k prefix_len { - sks_at_x[prefix_len..].to_vec() - } else { - Vec::new() - }; - PerQuery { - prefix_prod, - suffix_w, - } - }; - // This runs once per fold level over tiny verify-sized inputs - // (`queries` ≈ tens; `yr_len` ≤ 2^5 since the residual folds to ≤5 bits), so - // a rayon dispatch per level costs more than the field work itself (measured - // ~0.47 ms serial vs ~0.75 ms parallel for the whole residual eval at m=30). - // Stay serial below the crossover — mirror of merkle.rs's `SERIAL_LEVEL_NODES`. - const PAR_FLOOR: usize = 1024; - let per_query: Vec = if n_queries > PAR_FLOOR { - queries.par_iter().map(compute_query).collect() - } else { - queries.iter().map(compute_query).collect() - }; + let mut accum_basis = vec![F192::ZERO; n]; + let mut local_basis = vec![F192::ZERO; n]; + let mut sks_at_x = vec![F64::ZERO; log_msg_cols.max(1)]; + let mut local_sum = F192::ZERO; - // For each residual position y, accumulate the suffix product per query. - let compute_y = |y: usize| -> F128 { - let mut sum = F128::ZERO; - for i in 0..n_queries { - let pq = &per_query[i]; - let mut suffix_prod = F128::ONE; - for j in 0..yr_log_n { - let p_j = if (y >> j) & 1 == 1 { - F128::ONE - } else { - F128::ZERO - }; - suffix_prod *= F128::ONE + p_j * (F128::ONE + pq.suffix_w[j]); + for i in start..end { + let row = &opened_rows[i]; + let q = queries[i]; + let ap = alpha_pows[i]; + + // Mixed dot: E eq-weights times K row entries. + let dot: F192 = row + .iter() + .zip(eq.iter()) + .map(|(&r, &e)| e.mul_base(r)) + .fold(F192::ZERO, |a, v| a + v); + local_sum += dot * ap; + + let q_field = F64(q as u64); + evaluate_scaled_basis_inplace(&mut sks_at_x, &mut local_basis, sks_vks, &inv_sks_vks, q_field, ap); + for (acc, &v) in accum_basis.iter_mut().zip(local_basis.iter()) { + *acc += v; + } } - sum += alpha_pows[i] * pq.prefix_prod * suffix_prod; + (accum_basis, local_sum) + }) + .collect(); + + let mut basis_poly = vec![F192::ZERO; n]; + let mut enforced_sum = F192::ZERO; + for (lb, ls) in partials { + for (acc, &v) in basis_poly.iter_mut().zip(lb.iter()) { + *acc += v; } - sum - }; - if yr_len > PAR_FLOOR { - (0..yr_len).into_par_iter().map(compute_y).collect() - } else { - (0..yr_len).map(compute_y).collect() + enforced_sum += ls; } + + (basis_poly, enforced_sum) } -/// `queries` are **0-indexed** codeword positions. `q_field = F128::new(q, 0)`. -/// -/// Parallel: each thread takes a chunk of queries, builds a partial basis_poly -/// accumulator + partial enforced_sum, then we reduce. The per-query work -/// (eq-dot + LCH novel-basis expansion) is independent of other queries. -pub(crate) fn induce_sumcheck_poly( +/// Deeper-level induce: opened rows are E-valued. Same structure as +/// [`induce_sumcheck_poly_base`] with a pure-E row dot. +pub(crate) fn induce_sumcheck_poly_ext( log_msg_cols: usize, - sks_vks: &[F128], - opened_rows: &[Vec], - v_challenges: &[F128], + sks_vks: &[F64], + opened_rows: &[Vec], + v_challenges: &[F192], queries: &[usize], - alpha: &[F128], -) -> (Vec, F128) { + alpha: &[F192], +) -> (Vec, F192) { use rayon::prelude::*; let n = 1usize << log_msg_cols; let n_queries = queries.len(); @@ -1369,66 +1062,51 @@ pub(crate) fn induce_sumcheck_poly( .unwrap_or(0) ); - let eq = build_eq(v_challenges); // length 2^v_challenges.len() = num_interleaved + let eq = build_eq_table_ext(v_challenges); - // Per-query weights are the eq-tensor coefficients `eq(α, i_bin)` for - // `i ∈ {0,1}^{⌈log₂ n_queries⌉}` (LSB-first), truncated to the first - // `n_queries` indices. Replaces the legacy α^i Vandermonde scheme; - // matches the multilinear S-Z structure used by the lane fold. - let alpha_pows: Vec = if n_queries == 0 { + let alpha_pows: Vec = if n_queries == 0 { Vec::new() } else { - let table = build_eq(alpha); + let table = build_eq_table_ext(alpha); debug_assert!(table.len() >= n_queries); table.into_iter().take(n_queries).collect() }; - // Precompute inv_sks_vks once across all queries and threads. - let inv_sks_vks: Vec = sks_vks + let inv_sks_vks: Vec = sks_vks .iter() - .map(|&v| if v.is_zero() { F128::ZERO } else { v.inv() }) + .map(|&v| if v.is_zero() { F64::ZERO } else { v.inv() }) .collect(); - // Per-thread chunked accumulation: each thread accumulates a partial - // basis_poly (length n) and a partial enforced_sum, then we reduce. let n_threads = rayon::current_num_threads().max(1); let chunk_size = (n_queries + n_threads - 1) / n_threads.max(1); - let partials: Vec<(Vec, F128)> = (0..n_threads) + let partials: Vec<(Vec, F192)> = (0..n_threads) .into_par_iter() .map(|t| { let start = t * chunk_size; let end = (start + chunk_size).min(n_queries); if start >= end { - return (vec![F128::ZERO; n], F128::ZERO); + return (vec![F192::ZERO; n], F192::ZERO); } - let mut accum_basis = vec![F128::ZERO; n]; - // Per-thread scratch reused across this chunk's queries. - let mut local_basis = vec![F128::ZERO; n]; - let mut sks_at_x = vec![F128::ZERO; log_msg_cols.max(1)]; - let mut local_sum = F128::ZERO; + let mut accum_basis = vec![F192::ZERO; n]; + let mut local_basis = vec![F192::ZERO; n]; + let mut sks_at_x = vec![F64::ZERO; log_msg_cols.max(1)]; + let mut local_sum = F192::ZERO; for i in start..end { let row = &opened_rows[i]; let q = queries[i]; let ap = alpha_pows[i]; - let dot: F128 = row + let dot: F192 = row .iter() .zip(eq.iter()) .map(|(&r, &e)| r * e) - .fold(F128::ZERO, |a, v| a + v); + .fold(F192::ZERO, |a, v| a + v); local_sum += dot * ap; - let q_field = F128::new(q as u64, 0); - evaluate_scaled_basis_inplace( - &mut sks_at_x, - &mut local_basis, - sks_vks, - &inv_sks_vks, - q_field, - ap, - ); + let q_field = F64(q as u64); + evaluate_scaled_basis_inplace(&mut sks_at_x, &mut local_basis, sks_vks, &inv_sks_vks, q_field, ap); for (acc, &v) in accum_basis.iter_mut().zip(local_basis.iter()) { *acc += v; } @@ -1437,9 +1115,8 @@ pub(crate) fn induce_sumcheck_poly( }) .collect(); - // Reduce across threads. - let mut basis_poly = vec![F128::ZERO; n]; - let mut enforced_sum = F128::ZERO; + let mut basis_poly = vec![F192::ZERO; n]; + let mut enforced_sum = F192::ZERO; for (lb, ls) in partials { for (acc, &v) in basis_poly.iter_mut().zip(lb.iter()) { *acc += v; @@ -1450,200 +1127,259 @@ pub(crate) fn induce_sumcheck_poly( (basis_poly, enforced_sum) } -/// Transposed forward additive NTT, `Fᵀ`, in place over `2^log_d` coefficients. -/// Forward butterfly is `M=[[1,t],[1,t+1]]`; transpose `Mᵀ=[[1,1],[t,t+1]]` is -/// `s=a+b; top=s; bot=t·s+b`, applied in **reverse** layer order. (Baseline: -/// one parallel sweep per layer.) -fn transpose_forward_ntt(ntt: &AdditiveNttF128, data: &mut [F128], log_d: usize) { - use rayon::prelude::*; - debug_assert_eq!(data.len(), 1usize << log_d); - debug_assert!(log_d <= ntt.log_domain_size()); - let n_threads = rayon::current_num_threads().max(1); - for layer in (0..log_d).rev() { - let num_blocks = 1usize << layer; - let block_size = 1usize << (log_d - layer); - let bsh = block_size >> 1; - if num_blocks >= n_threads { - data.par_chunks_mut(block_size) - .enumerate() - .for_each(|(block, chunk)| { - let t = ntt.twiddle(layer, block); - let (top, bot) = chunk.split_at_mut(bsh); - for (a_ref, b_ref) in top.iter_mut().zip(bot.iter_mut()) { - let a = *a_ref; - let b = *b_ref; - let s = a + b; - *a_ref = s; - *b_ref = t * s + b; - } - }); - } else { - for block in 0..num_blocks { - let t = ntt.twiddle(layer, block); - let chunk = &mut data[block * block_size..(block + 1) * block_size]; - let (top, bot) = chunk.split_at_mut(bsh); - top.par_iter_mut() - .zip(bot.par_iter_mut()) - .for_each(|(a_ref, b_ref)| { - let a = *a_ref; - let b = *b_ref; - let s = a + b; - *a_ref = s; - *b_ref = t * s + b; - }); - } - } +/// Compute just the `enforced_sum` half of the L0 induce (mirror of +/// `ligerito::induce_sumcheck_enforced_sum`, F64 rows via `mul_base`): +/// `enforced_sum = Σ_i eq(α, i) · ` +/// Cheap: O(num_queries x num_interleaved). The succinct verifier needs this +/// at level intro time (before the residual challenges are known). +pub(crate) fn induce_sumcheck_enforced_sum_base( + opened_rows: &[Vec], + v_challenges: &[F192], + queries: &[usize], + alpha: &[F192], +) -> F192 { + assert_eq!(opened_rows.len(), queries.len()); + let eq = build_eq_table_ext(v_challenges); + let n_queries = queries.len(); + let alpha_weights: Vec = if n_queries == 0 { + Vec::new() + } else { + build_eq_table_ext(alpha).into_iter().take(n_queries).collect() + }; + let mut sum = F192::ZERO; + for (i, row) in opened_rows.iter().enumerate() { + debug_assert_eq!(row.len(), eq.len()); + let dot: F192 = row + .iter() + .zip(eq.iter()) + .map(|(&r, &e)| e.mul_base(r)) + .fold(F192::ZERO, |a, v| a + v); + sum += alpha_weights[i] * dot; + } + sum +} + +/// Deeper-level counterpart of [`induce_sumcheck_enforced_sum_base`]: +/// E-valued opened rows, pure-E row dot. +pub(crate) fn induce_sumcheck_enforced_sum_ext( + opened_rows: &[Vec], + v_challenges: &[F192], + queries: &[usize], + alpha: &[F192], +) -> F192 { + assert_eq!(opened_rows.len(), queries.len()); + let eq = build_eq_table_ext(v_challenges); + let n_queries = queries.len(); + let alpha_weights: Vec = if n_queries == 0 { + Vec::new() + } else { + build_eq_table_ext(alpha).into_iter().take(n_queries).collect() + }; + let mut sum = F192::ZERO; + for (i, row) in opened_rows.iter().enumerate() { + debug_assert_eq!(row.len(), eq.len()); + let dot: F192 = row + .iter() + .zip(eq.iter()) + .map(|(&r, &e)| r * e) + .fold(F192::ZERO, |a, v| a + v); + sum += alpha_weights[i] * dot; } + sum } -/// `Fᵀ`-based fast path for [`induce_sumcheck_poly`]: scatter per-query weights -/// into the codeword domain, apply `Fᵀ`, keep the low `2^log_msg_cols` outputs. -/// Byte-identical output to [`induce_sumcheck_poly`]. -pub(crate) fn induce_sumcheck_poly_via_ntt( +/// SUCCINCT evaluator for the induced basis poly's MLE at residual points +/// (mirror of `ligerito::induce_sumcheck_evaluate_at_residual`). Replaces the +/// dense basis + `partial_eval_lsb` in the verifier via the closed form: +/// `MLE(basis_poly)(p) = Σ_i eq(α, i) · Π_k (1 + p[k] · (1 + W-hat_k(q_i)))` +/// where `q_i = F64(queries[i])` and the K-valued `W-hat_k(q_i)` lifts into E +/// through the char-2 factor. `ris_for_basis` is the fixed residual prefix +/// (length `log_msg_cols - yr_log_n`); returns evaluations at the `2^yr_log_n` +/// points `ris_for_basis ++ y_bits`. +pub(crate) fn induce_sumcheck_evaluate_at_residual( log_msg_cols: usize, - log_inv_rate: usize, - opened_rows: &[Vec], - v_challenges: &[F128], + sks_vks: &[F64], queries: &[usize], - alpha: &[F128], -) -> (Vec, F128) { - let n = 1usize << log_msg_cols; - let log_block = log_msg_cols + log_inv_rate; - let block_len = 1usize << log_block; + alpha: &[F192], + ris_for_basis: &[F192], + yr_log_n: usize, +) -> Vec { + use rayon::prelude::*; + assert_eq!(ris_for_basis.len() + yr_log_n, log_msg_cols); let n_queries = queries.len(); - assert_eq!(opened_rows.len(), n_queries); + let yr_len = 1usize << yr_log_n; - let eq = build_eq(v_challenges); - let alpha_pows: Vec = if n_queries == 0 { + let alpha_pows: Vec = if n_queries == 0 { Vec::new() } else { - let table = build_eq(alpha); + let table = build_eq_table_ext(alpha); debug_assert!(table.len() >= n_queries); table.into_iter().take(n_queries).collect() }; - let mut enforced_sum = F128::ZERO; - for i in 0..n_queries { - let dot: F128 = opened_rows[i] - .iter() - .zip(eq.iter()) - .map(|(&r, &e)| r * e) - .fold(F128::ZERO, |a, v| a + v); - enforced_sum += dot * alpha_pows[i]; - } + let inv_sks_vks: Vec = sks_vks + .iter() + .map(|&v| if v.is_zero() { F64::ZERO } else { v.inv() }) + .collect(); - let mut coeffs = if log_block == 0 { - let mut c = vec![F128::ZERO; block_len]; - for i in 0..n_queries { - c[queries[i]] += alpha_pows[i]; + let prefix_len = ris_for_basis.len(); + + // Per-query precomputation: W-hat_k(q) for all k over K, split into a + // fixed prefix product (E scalar) and the suffix W-hat values varied per y. + struct PerQuery { + prefix_prod: F192, + suffix_w: Vec, // length = yr_log_n + } + let compute_query = |&q: &usize| -> PerQuery { + let q_field = F64(q as u64); + let mut sks_at_x = Vec::with_capacity(log_msg_cols.max(1)); + if log_msg_cols > 0 { + sks_at_x.push(q_field); + for k in 1..log_msg_cols { + sks_at_x.push(next_s(sks_at_x[k - 1], sks_vks[k - 1])); + } + for k in 0..log_msg_cols { + sks_at_x[k] *= inv_sks_vks[k]; + } } - c + // Prefix product: Π_{k prefix_len { + sks_at_x[prefix_len..].to_vec() + } else { + Vec::new() + }; + PerQuery { prefix_prod, suffix_w } + }; + // Once per recursion level over verify-sized inputs; stay serial below + // the rayon dispatch crossover (mirror of the original's PAR_FLOOR). + const PAR_FLOOR: usize = 1024; + let per_query: Vec = if n_queries > PAR_FLOOR { + queries.par_iter().map(compute_query).collect() } else { - let ntt = AdditiveNttF128::standard(log_block); - transpose_forward_ntt_sparse(&ntt, queries, &alpha_pows, log_block) + queries.iter().map(compute_query).collect() }; - coeffs.truncate(n); - (coeffs, enforced_sum) -} -/// Cost-based dispatch between the dense [`induce_sumcheck_poly`] and the -/// sparse-NTT [`induce_sumcheck_poly_via_ntt`]. -/// -/// The dense path costs `O(n_queries · 2^log_msg_cols)`; the NTT path costs one -/// pass over the `2^(log_msg_cols+log_inv_rate)` codeword domain, `O(2^log_block -/// · log_block)`. The `2^log_msg_cols` factor cancels, so the NTT wins exactly -/// when there are enough queries to amortize the codeword pass against the rate -/// blow-up and depth: -/// `n_queries > C · 2^log_inv_rate · log_block` (C≈4: the NTT is ~2× -/// costlier per op — memory-bound, multi-pass — plus margin so we only switch -/// when clearly ahead). In the multilevel PCS this fires only at the top level -/// (large message domain, many queries); deeper levels stay dense. -/// -/// Both paths are byte-identical (see `induce_sumcheck_poly_via_ntt_matches_dense`), -/// so a mis-dispatch only costs time. Tuned/validated at blake m=30. -pub(crate) fn induce_sumcheck_poly_auto( - log_msg_cols: usize, - log_inv_rate: usize, - sks_vks: &[F128], - opened_rows: &[Vec], - v_challenges: &[F128], - queries: &[usize], - alpha: &[F128], -) -> (Vec, F128) { - let log_block = log_msg_cols + log_inv_rate; - let use_ntt = - log_msg_cols >= 12 && queries.len() > 4 * (1usize << log_inv_rate) * log_block.max(1); - if use_ntt { - induce_sumcheck_poly_via_ntt( - log_msg_cols, - log_inv_rate, - opened_rows, - v_challenges, - queries, - alpha, - ) + // For each residual position y, accumulate the suffix product per query. + let compute_y = |y: usize| -> F192 { + let mut sum = F192::ZERO; + for i in 0..n_queries { + let pq = &per_query[i]; + let mut suffix_prod = F192::ONE; + for j in 0..yr_log_n { + let p_j = if (y >> j) & 1 == 1 { F192::ONE } else { F192::ZERO }; + suffix_prod *= F192::ONE + p_j * (F192::ONE + F192::from(pq.suffix_w[j])); + } + sum += alpha_pows[i] * pq.prefix_prod * suffix_prod; + } + sum + }; + if yr_len > PAR_FLOOR { + (0..yr_len).into_par_iter().map(compute_y).collect() } else { - induce_sumcheck_poly( - log_msg_cols, - sks_vks, - opened_rows, - v_challenges, - queries, - alpha, - ) + (0..yr_len).map(compute_y).collect() } } -/// Sparse-prefix variant of [`transpose_forward_ntt`]: exploits that the input -/// has only `positions.len()` nonzeros and that the first `k` transpose steps -/// (forward layers `log_d-1 .. log_d-k`, pairing distances `1 .. 2^(k-1)`) mix -/// only **within** `2^k`-aligned windows. We process just the windows that -/// contain a nonzero (a dense `2^k` transpose each), densify, then run the -/// remaining steps as full dense sweeps. Output is identical to -/// `transpose_forward_ntt` applied to the scattered input. -fn transpose_forward_ntt_sparse( - ntt: &AdditiveNttF128, - positions: &[usize], - values: &[F128], - log_d: usize, -) -> Vec { - let _span = tracing::info_span!( - "NTT", - kind = "transpose induce", - log_domain = %pretty_integer(log_d), - nonzero = %pretty_integer(positions.len()) - ) - .entered(); - use rayon::prelude::*; - use std::collections::HashMap; - let n = 1usize << log_d; - // No prefix for small domains — just scatter + full dense transpose. - let k = if log_d >= 12 { 8usize.min(log_d) } else { 0 }; +// =================================================================== +// Transposed-NTT fast path for basis induction (mirror of the original) +// =================================================================== + +/// Transposed forward additive NTT, `F^T`, in place over `2^log_d` E-values +/// with K-twiddles. Forward butterfly is `M = [[1, t], [1, t+1]]`; transpose +/// `M^T = [[1, 1], [t, t+1]]` is `s = a + b; top = s; bot = t*s + b` (here +/// `s.mul_base(t) + b`), applied in reverse layer order. Mirror of +/// `ligerito::transpose_forward_ntt` (one parallel sweep per layer). +fn transpose_forward_ntt_ext(ntt: &AdditiveNttF64, data: &mut [F192], log_d: usize) { + use rayon::prelude::*; + debug_assert_eq!(data.len(), 1usize << log_d); + debug_assert!(log_d <= ntt.log_domain_size()); + let n_threads = rayon::current_num_threads().max(1); + for layer in (0..log_d).rev() { + let num_blocks = 1usize << layer; + let block_size = 1usize << (log_d - layer); + let bsh = block_size >> 1; + if num_blocks >= n_threads { + data.par_chunks_mut(block_size).enumerate().for_each(|(block, chunk)| { + let t = ntt.twiddle(layer, block); + let (top, bot) = chunk.split_at_mut(bsh); + for (a_ref, b_ref) in top.iter_mut().zip(bot.iter_mut()) { + let a = *a_ref; + let b = *b_ref; + let s = a + b; + *a_ref = s; + *b_ref = s.mul_base(t) + b; + } + }); + } else { + for block in 0..num_blocks { + let t = ntt.twiddle(layer, block); + let chunk = &mut data[block * block_size..(block + 1) * block_size]; + let (top, bot) = chunk.split_at_mut(bsh); + top.par_iter_mut().zip(bot.par_iter_mut()).for_each(|(a_ref, b_ref)| { + let a = *a_ref; + let b = *b_ref; + let s = a + b; + *a_ref = s; + *b_ref = s.mul_base(t) + b; + }); + } + } + } +} + +/// Sparse-prefix variant of [`transpose_forward_ntt_ext`]: the input has only +/// `positions.len()` nonzeros and the first `k` transpose steps (forward +/// layers `log_d-1 .. log_d-k`, pairing distances `1 .. 2^(k-1)`) mix only +/// WITHIN `2^k`-aligned windows. We process just the windows that contain a +/// nonzero (a dense `2^k` transpose each, disjoint so window-parallel), +/// densify, then run the remaining steps as full dense sweeps. Output is +/// identical to `transpose_forward_ntt_ext` on the scattered input. Mirror +/// of `ligerito::transpose_forward_ntt_sparse`. +fn transpose_forward_ntt_sparse_ext( + ntt: &AdditiveNttF64, + positions: &[usize], + values: &[F192], + log_d: usize, +) -> Vec { + let _span = tracing::info_span!( + "NTT", + kind = "transpose induce", + log_domain = log_d, + nonzero = positions.len() + ) + .entered(); + use rayon::prelude::*; + use std::collections::HashMap; + let n = 1usize << log_d; + // No prefix for small domains: just scatter + full dense transpose. + let k = if log_d >= 12 { 8usize.min(log_d) } else { 0 }; if k == 0 { - let mut data = vec![F128::ZERO; n]; + let mut data = vec![F192::ZERO; n]; for (&p, &v) in positions.iter().zip(values) { data[p] += v; } if log_d > 0 { - transpose_forward_ntt(ntt, &mut data, log_d); + transpose_forward_ntt_ext(ntt, &mut data, log_d); } return data; } let wmask = (1usize << k) - 1; // Group nonzeros into 2^k windows. - let mut windows: HashMap> = HashMap::new(); + let mut windows: HashMap> = HashMap::new(); for (&p, &v) in positions.iter().zip(values) { - let buf = windows - .entry(p >> k) - .or_insert_with(|| vec![F128::ZERO; 1 << k]); + let buf = windows.entry(p >> k).or_insert_with(|| vec![F192::ZERO; 1 << k]); buf[p & wmask] += v; } // Steps s = 0..k-1 within each active window, in parallel (windows disjoint). - let win_vec: Vec<(usize, Vec)> = windows.into_iter().collect(); - let processed: Vec<(usize, Vec)> = win_vec + let win_vec: Vec<(usize, Vec)> = windows.into_iter().collect(); + let processed: Vec<(usize, Vec)> = win_vec .into_par_iter() .map(|(w, mut buf)| { for s in 0..k { @@ -1660,7 +1396,7 @@ fn transpose_forward_ntt_sparse( let b = buf[base + r + bsh]; let sab = a + b; buf[base + r] = sab; - buf[base + r + bsh] = t * sab + b; + buf[base + r + bsh] = sab.mul_base(t) + b; } } } @@ -1670,7 +1406,7 @@ fn transpose_forward_ntt_sparse( // Densify (active windows only; the rest stay zero, which is the correct // post-step-(k-1) state for an all-zero window). - let mut data = vec![F128::ZERO; n]; + let mut data = vec![F192::ZERO; n]; for (w, buf) in processed { data[(w << k)..((w + 1) << k)].copy_from_slice(&buf); } @@ -1682,75 +1418,142 @@ fn transpose_forward_ntt_sparse( let block_size = 1usize << (log_d - layer); let bsh = block_size >> 1; if num_blocks >= n_threads { - data.par_chunks_mut(block_size) - .enumerate() - .for_each(|(block, chunk)| { - let t = ntt.twiddle(layer, block); - let (top, bot) = chunk.split_at_mut(bsh); - for (a_ref, b_ref) in top.iter_mut().zip(bot.iter_mut()) { - let a = *a_ref; - let b = *b_ref; - let sab = a + b; - *a_ref = sab; - *b_ref = t * sab + b; - } - }); + data.par_chunks_mut(block_size).enumerate().for_each(|(block, chunk)| { + let t = ntt.twiddle(layer, block); + let (top, bot) = chunk.split_at_mut(bsh); + for (a_ref, b_ref) in top.iter_mut().zip(bot.iter_mut()) { + let a = *a_ref; + let b = *b_ref; + let sab = a + b; + *a_ref = sab; + *b_ref = sab.mul_base(t) + b; + } + }); } else { for block in 0..num_blocks { let t = ntt.twiddle(layer, block); let chunk = &mut data[block * block_size..(block + 1) * block_size]; let (top, bot) = chunk.split_at_mut(bsh); - top.par_iter_mut() - .zip(bot.par_iter_mut()) - .for_each(|(a_ref, b_ref)| { - let a = *a_ref; - let b = *b_ref; - let sab = a + b; - *a_ref = sab; - *b_ref = t * sab + b; - }); + top.par_iter_mut().zip(bot.par_iter_mut()).for_each(|(a_ref, b_ref)| { + let a = *a_ref; + let b = *b_ref; + let sab = a + b; + *a_ref = sab; + *b_ref = sab.mul_base(t) + b; + }); } } } data } +/// `F^T`-based fast path for [`induce_sumcheck_poly_base`]: scatter per-query +/// E-weights into the codeword domain, apply `F^T` with K-twiddles, keep the +/// low `2^log_msg_cols` outputs. Byte-identical output to the dense path +/// (pinned by `induce_via_ntt_matches_dense`). Mirror of +/// `ligerito::induce_sumcheck_poly_via_ntt` with the L0 mixed row dot. +pub(crate) fn induce_sumcheck_poly_via_ntt_base( + log_msg_cols: usize, + log_inv_rate: usize, + opened_rows: &[Vec], + v_challenges: &[F192], + queries: &[usize], + alpha: &[F192], +) -> (Vec, F192) { + let n = 1usize << log_msg_cols; + let log_block = log_msg_cols + log_inv_rate; + let block_len = 1usize << log_block; + let n_queries = queries.len(); + assert_eq!(opened_rows.len(), n_queries); + + let eq = build_eq_table_ext(v_challenges); + let alpha_pows: Vec = if n_queries == 0 { + Vec::new() + } else { + let table = build_eq_table_ext(alpha); + debug_assert!(table.len() >= n_queries); + table.into_iter().take(n_queries).collect() + }; + + let mut enforced_sum = F192::ZERO; + for i in 0..n_queries { + let dot: F192 = opened_rows[i] + .iter() + .zip(eq.iter()) + .map(|(&r, &e)| e.mul_base(r)) + .fold(F192::ZERO, |a, v| a + v); + enforced_sum += dot * alpha_pows[i]; + } + + let mut coeffs = if log_block == 0 { + let mut c = vec![F192::ZERO; block_len]; + for i in 0..n_queries { + c[queries[i]] += alpha_pows[i]; + } + c + } else { + let ntt = AdditiveNttF64::standard(log_block); + transpose_forward_ntt_sparse_ext(&ntt, queries, &alpha_pows, log_block) + }; + coeffs.truncate(n); + (coeffs, enforced_sum) +} + +/// The original's cost-based dispatch heuristic, verbatim: the dense path +/// costs `O(n_queries * 2^log_msg_cols)`, the NTT path one pass over the +/// `2^log_block` codeword domain, so the NTT wins exactly when +/// `n_queries > 4 * 2^log_inv_rate * log_block`. Same constants as the +/// original so both field versions choose the same strategy at the same +/// shapes. +#[inline] +pub(crate) fn induce_use_ntt_heuristic(log_msg_cols: usize, log_inv_rate: usize, n_queries: usize) -> bool { + let log_block = log_msg_cols + log_inv_rate; + log_msg_cols >= 12 && n_queries > 4 * (1usize << log_inv_rate) * log_block.max(1) +} + +/// Dispatch between the dense [`induce_sumcheck_poly_base`] and the sparse +/// [`induce_sumcheck_poly_via_ntt_base`] for L0 (base-field rows). Mirror of +/// `ligerito::induce_sumcheck_poly_auto`: in the recursive PCS this fires +/// only at the top level (large message domain, many queries); deeper levels +/// stay dense. Both paths produce identical output, so a mis-dispatch only +/// costs time. +pub(crate) fn induce_sumcheck_poly_auto_base( + log_msg_cols: usize, + log_inv_rate: usize, + sks_vks: &[F64], + opened_rows: &[Vec], + v_challenges: &[F192], + queries: &[usize], + alpha: &[F192], +) -> (Vec, F192) { + if induce_use_ntt_heuristic(log_msg_cols, log_inv_rate, queries.len()) { + induce_sumcheck_poly_via_ntt_base(log_msg_cols, log_inv_rate, opened_rows, v_challenges, queries, alpha) + } else { + induce_sumcheck_poly_base(log_msg_cols, sks_vks, opened_rows, v_challenges, queries, alpha) + } +} + // =================================================================== -// ligero_commit +// ligero_commit for E-valued (folded) witnesses // =================================================================== -/// Codeword + Merkle tree for one Ligerito commitment level. -/// -/// `mat` is row-major: `mat[pos * num_interleaved + lane]` for -/// `pos ∈ [0, block_len)`, `lane ∈ [0, num_interleaved)`. Each row -/// (one `pos` across all lanes) is one Merkle leaf. -pub struct LigeroWitness { - pub mat: Vec, +/// Codeword + Merkle tree for one deeper Ligerito commitment level. +/// `mat[pos * num_interleaved + lane]`; each row (one `pos` across all lanes) +/// is one Merkle leaf of `num_interleaved * 16` bytes. +pub(crate) struct LigeroWitness { + pub mat: Vec, pub tree: Vec, pub block_len: usize, pub num_interleaved: usize, } -// Recycle the codeword matrix (128 MB for L1 at m=29) through the scratch -// pool when a level's witness is replaced/dropped. -impl Drop for LigeroWitness { - fn drop(&mut self) { - primitives::scratch::give_f128(std::mem::take(&mut self.mat)); - } -} - -// SumcheckProver owns the two witness-sized polynomials of the open (the -// packed witness `f` and the γ-combined basis) — recycle both on drop. -impl Drop for SumcheckProver { - fn drop(&mut self) { - primitives::scratch::give_f128(std::mem::take(&mut self.f)); - primitives::scratch::give_f128(std::mem::take(&mut self.combined_basis)); - } -} +// No Drop/scratch-pool recycling here (divergence from the original's +// `LigeroWitness`): there is no F192 scratch pool, and deeper-level matrices +// are small relative to L0. impl LigeroWitness { #[inline] - pub fn row(&self, pos: usize) -> &[F128] { + pub fn row(&self, pos: usize) -> &[F192] { let start = pos * self.num_interleaved; &self.mat[start..start + self.num_interleaved] } @@ -1761,21 +1564,15 @@ impl LigeroWitness { } } -/// Reshape `poly` (length `num_interleaved · msg_cols`) into a -/// `block_len × num_interleaved` SoA matrix, RS-encode each lane via the -/// LCH additive NTT (non-systematic: pad message with zeros to `block_len`, -/// then forward-transform), and Merkle-commit the rows. -/// -/// `poly` layout: **LSB-first lane index** — `poly[col * num_interleaved + lane]`. -/// The first `log_num_interleaved` LSB variables of the multilinear poly are the -/// lane indices, so `partial_eval_lsb(poly, lane_challenges)` produces the -/// next-level poly directly. This composes cleanly with sumcheck folds. -pub fn ligero_commit( - poly: &[F128], +/// Commit an extension-field polynomial at a recursive level: replicate the +/// LSB-lane-layout message into all `2^log_inv_rate` sub-blocks, RS-encode +/// each lane with the K-twiddle mixed-product NTT, and Merkle over rows. +pub(crate) fn ligero_commit_ext( + poly: &[F192], log_msg_cols: usize, log_num_interleaved: usize, log_inv_rate: usize, - ntt: &AdditiveNttF128, + ntt: &AdditiveNttF64, ) -> LigeroWitness { let msg_cols = 1usize << log_msg_cols; let num_interleaved = 1usize << log_num_interleaved; @@ -1784,36 +1581,47 @@ pub fn ligero_commit( assert_eq!(poly.len(), num_interleaved * msg_cols); assert!(log_block_len <= ntt.log_domain_size()); - // LSB-lane layout: input matches the SoA layout `data[pos * num_interleaved + lane]` - // directly. The first `log_inv_rate` NTT layers on the zero-padded - // coefficients are pure copies, so fill the matrix with 2^log_inv_rate - // replicas of `poly` (same write cost as copy + zero-fill) and start the - // transform past those layers — see `pcs::commit::replicate_message_fill`. + // Plain allocation (scratch-pool divergence; see module docs). let codeword_len = block_len * num_interleaved; - let mut mat = primitives::scratch::take_f128(codeword_len); - super::commit::replicate_message_fill(&mut mat, poly); - - // RS-encode every lane in one call (each lane is one independent NTT). + let mut mat = primitives::alloc_uninit(codeword_len); + replicate_message_fill_uninit(&mut mat, poly); + // SAFETY: the replicate fill initializes every matrix element. + let mut mat = unsafe { primitives::assume_init(mat) }; + + // Optional per-level NTT/Merkle split (LIGERITO_TRACE): one env lookup per + // commit level, no work when unset. + let trace = std::env::var_os("LIGERITO_TRACE").is_some(); + let t_ntt = std::time::Instant::now(); tracing::info_span!( "NTT", kind = "extension encode", - log_domain = %pretty_integer(log_block_len), - lanes = %pretty_integer(num_interleaved) + log_domain = log_block_len, + lanes = num_interleaved ) - .in_scope(|| { - ntt.forward_transform_interleaved_from_layer(&mut mat, num_interleaved, log_inv_rate) - }); - - // Merkle over rows. One leaf = `num_interleaved` consecutive F128 = 16·num_interleaved bytes. - let leaf_size_bytes = num_interleaved * core::mem::size_of::(); - let data_bytes: &[u8] = unsafe { - core::slice::from_raw_parts( - mat.as_ptr() as *const u8, - mat.len() * core::mem::size_of::(), - ) - }; + .in_scope(|| forward_transform_interleaved_ext_from_layer(ntt, &mut mat, num_interleaved, log_inv_rate)); + let ntt_elapsed = t_ntt.elapsed(); + let t_merkle = std::time::Instant::now(); + + // Merkle over rows, zero-copy. + // SAFETY: F192 is repr(C) with three u64 limbs (24 bytes, no padding); + // a `[F192]` slice is its contiguous byte image. The cast covers exactly + // `mat.len() * size_of::()` initialized + // bytes. + let leaf_size_bytes = num_interleaved * core::mem::size_of::(); + let data_bytes: &[u8] = + unsafe { core::slice::from_raw_parts(mat.as_ptr() as *const u8, mat.len() * core::mem::size_of::()) }; debug_assert_eq!(data_bytes.len(), block_len * leaf_size_bytes); let tree = merkle::merkle_tree(data_bytes, block_len); + if trace { + let log_block_len = pretty_integer(log_block_len); + let num_interleaved = pretty_integer(num_interleaved); + eprintln!( + "[lig] recursive_commit(log_block={log_block_len}, lanes={num_interleaved}): \ + ntt = {:.4} s, merkle = {:.4} s", + ntt_elapsed.as_secs_f64(), + t_merkle.elapsed().as_secs_f64(), + ); + } LigeroWitness { mat, @@ -1824,62 +1632,34 @@ pub fn ligero_commit( } // =================================================================== -// Stateful sumcheck — Flock (u_0, u_2) convention +// Stateful sumcheck over E with a two-phase (Base then Ext) witness // =================================================================== // -// Per-round quadratic q(X) = u_0 + u_1·X + u_2·X² with the sumcheck constraint -// q(0) + q(1) = T_r (T_r = running sum-claim entering this round) -// Verifier derives u_1 = T_r + u_2 (char 2). Round eval at challenge r: -// q(r) = u_0 + r·(T_r + u_2) + r²·u_2 = u_0 + r·T_r + (r + r²)·u_2 +// Same (u_0, u_2) convention as the original: per-round quadratic +// q(X) = u_0 + u_1 X + u_2 X^2 with q(0) + q(1) = T_r, verifier derives +// u_1 = T_r + u_2 (char 2), round eval q(r) = u_0 + r T_r + (r + r^2) u_2. // -// Ligerito extends plain sumcheck with two ops at level boundaries: -// -// introduce_new(b_new, h): -// Prover commits to a new basis poly b_new with its own claimed sum h -// (verifier-computable from the open-rows induce step). Sends (u_0, u_2) -// for the inner product f·b_new at the current (already-folded) dim. -// -// glue(α): -// Combine the running round-quadratic with the introduced one as -// running := running + α·to_glue. New sum-claim becomes T_r + α·h. - -/// Send one `(u_0, u_2)` sumcheck message on the stream (bound as written). -fn add_sumcheck_msg(ps: &mut ProverState, msg: &SumcheckMessage) { - ps.add_scalar(msg.u_0); - ps.add_scalar(msg.u_2); -} - -/// Read one `(u_0, u_2)` sumcheck message off the stream (bound as read). -fn next_sumcheck_msg(vs: &mut VerifierState<'_>) -> Option { - let u_0 = vs.next_scalar().ok()?; - let u_2 = vs.next_scalar().ok()?; - Some(SumcheckMessage { u_0, u_2 }) -} - -/// Read a Merkle root off the stream (two scalars, bound as read). -fn next_root(vs: &mut VerifierState<'_>) -> Option { - Some(merkle::scalars_to_hash(&vs.next_scalars(2).ok()?)) -} +// Round 0 pairs the K-witness with the E-basis via `mul_base`; the first fold +// lifts the witness into E and all later rounds are pure E. -/// (u_0, u_2) per round — what the prover sends. +/// (u_0, u_2) per round in E. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct SumcheckMessage { - pub u_0: F128, - pub u_2: F128, + pub u_0: F192, + pub u_2: F192, } -/// Round-quadratic in coefficient form `c + b·X + a·X²`. Used by the verifier -/// to track the running quadratic across fold / introduce_new / glue. +/// Round-quadratic in coefficient form `c + b X + a X^2` (verifier side). #[derive(Clone, Copy, Debug)] struct RoundQuad { - c: F128, // u_0 - b: F128, // u_1 (X coeff) — derived from T_r and u_2 - a: F128, // u_2 (X² coeff) + c: F192, // u_0 + b: F192, // u_1 (X coeff), derived from T_r and u_2 + a: F192, // u_2 (X^2 coeff) } impl RoundQuad { #[inline] - fn from_msg(msg: SumcheckMessage, t_r: F128) -> Self { + fn from_msg(msg: SumcheckMessage, t_r: F192) -> Self { Self { c: msg.u_0, b: t_r + msg.u_2, @@ -1887,11 +1667,11 @@ impl RoundQuad { } } #[inline] - fn eval(&self, r: F128) -> F128 { - self.c + r * self.b + r * r * self.a + fn eval(&self, r: F192) -> F192 { + (self.a * r + self.b) * r + self.c } #[inline] - fn fold(p1: &Self, p2: &Self, alpha: F128) -> Self { + fn fold(p1: &Self, p2: &Self, alpha: F192) -> Self { Self { c: p1.c + alpha * p2.c, b: p1.b + alpha * p2.b, @@ -1900,38 +1680,9 @@ impl RoundQuad { } } -/// Compute `(u_0, u_2)` for `u(X) = Σ_x f(X, x) · b(X, x)` where `X` is the -/// LSB variable. Parallel reduction across pair indices. -/// -/// Uses a SINGLE combined basis poly. (Previously took `&[Vec]` and -/// summed at every pair index; collapsing to one basis happens at glue time.) -/// One message-pair term, batched: `(f0·b0, (f0+f1)·(b0+b1))` as a 2-wide -/// CLMUL on x86_64 with VPCLMULQDQ, scalar muls elsewhere. -#[inline] -fn msg_pair_products(f0: F128, f1: F128, b0: F128, b1: F128) -> (F128, F128) { - #[cfg(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx2" - ))] - { - // SAFETY: vpclmulqdq+avx2 statically enabled by the cfg gate. - let p = unsafe { - primitives::field::gf2_128::x86_64::ghash_mul_vec2_clmul([f0, f0 + f1], [b0, b0 + b1]) - }; - (p[0], p[1]) - } - #[cfg(not(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx2" - )))] - { - (f0 * b0, (f0 + f1) * (b0 + b1)) - } -} - -fn round_msg_lsb(f: &[F128], b: &[F128]) -> SumcheckMessage { +/// Round message for the mixed phase: `f` in K, `b` in E. All products are +/// `mul_base` (2 PMULL each). +fn round_msg_lsb_base(f: &[F64], b: &[F192]) -> SumcheckMessage { use rayon::prelude::*; let n = f.len(); debug_assert!(n.is_power_of_two() && n >= 2); @@ -1939,40 +1690,49 @@ fn round_msg_lsb(f: &[F128], b: &[F128]) -> SumcheckMessage { const PAR_THRESHOLD: usize = 4096; let half = n / 2; + // Deferred reduction: XOR-accumulate the raw mul_base lane products + // (2 PMULL per term, no reduction tail) and reduce once per accumulator — + // reduction commutes with XOR, so the message is bit-identical. if half < PAR_THRESHOLD { - let mut u_0 = F128::ZERO; - let mut u_2 = F128::ZERO; + let mut u_0 = F192BaseUnreduced::ZERO; + let mut u_2 = F192BaseUnreduced::ZERO; for j in 0..half { - let (p0, p2) = msg_pair_products(f[2 * j], f[2 * j + 1], b[2 * j], b[2 * j + 1]); - u_0 += p0; - u_2 += p2; + let f0 = f[2 * j]; + let f1 = f[2 * j + 1]; + let b0 = b[2 * j]; + let b1 = b[2 * j + 1]; + u_0 ^= b0.mul_base_unreduced(f0); + u_2 ^= (b0 + b1).mul_base_unreduced(f0 + f1); } - return SumcheckMessage { u_0, u_2 }; + return SumcheckMessage { + u_0: u_0.reduce(), + u_2: u_2.reduce(), + }; } let (u_0, u_2) = (0..half) .into_par_iter() .with_min_len(PAR_THRESHOLD / 4) - .map(|j| msg_pair_products(f[2 * j], f[2 * j + 1], b[2 * j], b[2 * j + 1])) - .reduce( - || (F128::ZERO, F128::ZERO), - |(a0, a2), (b0, b2)| (a0 + b0, a2 + b2), - ); + .fold( + || (F192BaseUnreduced::ZERO, F192BaseUnreduced::ZERO), + |(a0, a2), j| { + let f0 = f[2 * j]; + let f1 = f[2 * j + 1]; + let b0 = b[2 * j]; + let b1 = b[2 * j + 1]; + ( + a0 ^ b0.mul_base_unreduced(f0), + a2 ^ (b0 + b1).mul_base_unreduced(f0 + f1), + ) + }, + ) + .map(|(a0, a2)| (a0.reduce(), a2.reduce())) + .reduce(|| (F192::ZERO, F192::ZERO), |(a0, a2), (b0, b2)| (a0 + b0, a2 + b2)); SumcheckMessage { u_0, u_2 } } -/// Fused round message + full inner product: returns `round_msg_lsb(f, b)` -/// alongside `y = Σ_x f(x)·b(x)`, computed in a single pass over `(f, b)`. -/// -/// Used by OOD binding, where `b = build_eq(z)` and `y` is the claimed MLE -/// eval `f̂(z)`. Folding `f` against `z` separately (`mle_eval_inline`) then -/// re-reading `f` against `b` in `round_msg_lsb` costs two passes over the -/// 2^n witness; this collapses them into one (the phase is memory-bandwidth -/// bound, so a saved pass is a near-proportional win). The `u_0` term `f0·b0` -/// is shared between the message and the eval, so `y` costs one extra mul per -/// pair. Bit-identical to the unfused path: F128 sums are exact and order- -/// independent, so `y == mle_eval_inline(f, z)`. -fn round_msg_and_eval_lsb(f: &[F128], b: &[F128]) -> (SumcheckMessage, F128) { +/// Round message for the pure-E phase. Mirror of `ligerito::round_msg_lsb`. +fn round_msg_lsb_ext(f: &[F192], b: &[F192]) -> SumcheckMessage { use rayon::prelude::*; let n = f.len(); debug_assert!(n.is_power_of_two() && n >= 2); @@ -1980,144 +1740,152 @@ fn round_msg_and_eval_lsb(f: &[F128], b: &[F128]) -> (SumcheckMessage, F128) { const PAR_THRESHOLD: usize = 4096; let half = n / 2; - let term = |j: usize| -> (F128, F128, F128) { - let f0 = f[2 * j]; - let f1 = f[2 * j + 1]; - let b0 = b[2 * j]; - let b1 = b[2 * j + 1]; - let e0 = f0 * b0; - // (u_0 term, u_2 term, y term = f0·b0 + f1·b1). - (e0, (f0 + f1) * (b0 + b1), e0 + f1 * b1) - }; + // Deferred reduction: XOR-accumulate the unreduced Karatsuba parts + // (3 PMULL per term) and reduce once per accumulator — bit-identical. if half < PAR_THRESHOLD { - let (mut u_0, mut u_2, mut y) = (F128::ZERO, F128::ZERO, F128::ZERO); + let mut u_0 = F192Unreduced::ZERO; + let mut u_2 = F192Unreduced::ZERO; for j in 0..half { - let (a0, a2, ay) = term(j); - u_0 += a0; - u_2 += a2; - y += ay; + let f0 = f[2 * j]; + let f1 = f[2 * j + 1]; + let b0 = b[2 * j]; + let b1 = b[2 * j + 1]; + u_0 ^= f0.mul_unreduced(b0); + u_2 ^= (f0 + f1).mul_unreduced(b0 + b1); } - return (SumcheckMessage { u_0, u_2 }, y); + return SumcheckMessage { + u_0: u_0.reduce(), + u_2: u_2.reduce(), + }; } - let (u_0, u_2, y) = (0..half) + let (u_0, u_2) = (0..half) .into_par_iter() .with_min_len(PAR_THRESHOLD / 4) - .map(term) - .reduce( - || (F128::ZERO, F128::ZERO, F128::ZERO), - |(a0, a2, ay), (b0, b2, by)| (a0 + b0, a2 + b2, ay + by), - ); - (SumcheckMessage { u_0, u_2 }, y) + .fold( + || (F192Unreduced::ZERO, F192Unreduced::ZERO), + |(a0, a2), j| { + let f0 = f[2 * j]; + let f1 = f[2 * j + 1]; + let b0 = b[2 * j]; + let b1 = b[2 * j + 1]; + (a0 ^ f0.mul_unreduced(b0), a2 ^ (f0 + f1).mul_unreduced(b0 + b1)) + }, + ) + .map(|(a0, a2)| (a0.reduce(), a2.reduce())) + .reduce(|| (F192::ZERO, F192::ZERO), |(a0, a2), (b0, b2)| (a0 + b0, a2 + b2)); + SumcheckMessage { u_0, u_2 } } -/// Partially evaluate `evals` at LSB variable = `r`, in place. Halves length. -/// Parallel for large arrays. Test oracle for the fused fold below; the -/// production path uses `fold_and_msg_lsb` instead. -#[cfg(test)] -fn partial_eval_lsb_one(evals: &mut Vec, r: F128) { +/// Build the round message and the full inner product in one pass. For an OOD +/// basis `b = eq(z, ·)`, the inner product is the claimed MLE evaluation. +fn round_msg_and_eval_lsb_ext(f: &[F192], b: &[F192]) -> (SumcheckMessage, F192) { use rayon::prelude::*; - let n = evals.len(); + let n = f.len(); debug_assert!(n.is_power_of_two() && n >= 2); - let half = n / 2; - let one_plus_r = F128::ONE + r; + debug_assert_eq!(b.len(), n); + let term = |j: usize| { + let f0 = f[2 * j]; + let f1 = f[2 * j + 1]; + let b0 = b[2 * j]; + let b1 = b[2 * j + 1]; + let e0 = f0 * b0; + (e0, (f0 + f1) * (b0 + b1), e0 + f1 * b1) + }; const PAR_THRESHOLD: usize = 4096; - if half < PAR_THRESHOLD { - for j in 0..half { - let v0 = evals[2 * j]; - let v1 = evals[2 * j + 1]; - evals[j] = v0 * one_plus_r + v1 * r; - } - evals.truncate(half); - return; - } - - // Parallel: produce a fresh halved Vec then swap in. Doing it in-place with - // par_iter on overlapping indices is dicey; allocate the halved output and - // swap (cheap vs the fold itself). - let folded: Vec = (0..half) - .into_par_iter() - .with_min_len(PAR_THRESHOLD / 4) - .map(|j| evals[2 * j] * one_plus_r + evals[2 * j + 1] * r) - .collect(); - *evals = folded; + let half = n / 2; + let (u_0, u_2, y) = if half < PAR_THRESHOLD { + (0..half) + .map(term) + .fold((F192::ZERO, F192::ZERO, F192::ZERO), |(a0, a2, ay), (b0, b2, by)| { + (a0 + b0, a2 + b2, ay + by) + }) + } else { + (0..half) + .into_par_iter() + .with_min_len(PAR_THRESHOLD / 4) + .map(term) + .reduce( + || (F192::ZERO, F192::ZERO, F192::ZERO), + |(a0, a2, ay), (b0, b2, by)| (a0 + b0, a2 + b2, ay + by), + ) + }; + (SumcheckMessage { u_0, u_2 }, y) } -/// Fused fold + next-round message in a SINGLE parallel pass. -/// -/// Replaces the three separate passes a sumcheck fold otherwise needs -/// (`partial_eval_lsb_one(f)` + `partial_eval_lsb_one(b)` + `round_msg_lsb`): -/// each chunk folds its slice of `f` and `b` at `r` (LSB variable) AND -/// accumulates that slice's `(u_0, u_2)` contribution to the message for the -/// *next* round — over the freshly-folded values, computed while they are -/// still in registers. One fork-join instead of three, and ~⅓ less memory -/// traffic (the folded arrays are not re-read to build the message). -/// -/// Returns `(folded_f, folded_b, next_msg)` where `next_msg = round_msg_lsb -/// (folded_f, folded_b)`. Bit-identical to the unfused sequence. -/// Fold one `(f, b)` pair against the same `r`, in the one-mul char-2 -/// interpolation form `x0 + r·(x0+x1)` (bit-identical to `x0·(1+r) + x1·r`, -/// half the muls). The two streams' muls share `r`, so on x86_64 with -/// VPCLMULQDQ they run as one 2-wide CLMUL. +/// Output buffer for an initial-sumcheck fold. On x86_64 these ~100 MB F192 +/// vectors are drawn from (and later returned to, in [`SumcheckProver::fold`]) +/// the process-global scratch pool, so repeated proves reuse resident pages +/// instead of faulting a fresh mapping each round; other targets keep the +/// fresh-alloc path (pooling measured slower on aarch64). Credit: flock +/// (flock-core) scratch-pool reuse for the initial-sumcheck fold buffers. #[inline] -fn fold_fb_pair(f0: F128, f1: F128, b0: F128, b1: F128, r: F128) -> (F128, F128) { - #[cfg(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx2" - ))] +fn fold_out_buf(n: usize) -> Vec { + #[cfg(target_arch = "x86_64")] { - // SAFETY: vpclmulqdq+avx2 statically enabled by the cfg gate. - let p = unsafe { - primitives::field::gf2_128::x86_64::ghash_mul_vec2_clmul([r, r], [f0 + f1, b0 + b1]) - }; - (f0 + p[0], b0 + p[1]) + primitives::scratch::take_f192(n) } - #[cfg(not(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx2" - )))] + #[cfg(not(target_arch = "x86_64"))] { - (f0 + r * (f0 + f1), b0 + r * (b0 + b1)) + // SAFETY: zero is a valid F192 value. + unsafe { primitives::alloc_zeroed_vec(n) } } } -fn fold_and_msg_lsb(f: &[F128], b: &[F128], r: F128) -> (Vec, Vec, SumcheckMessage) { +/// Fused fold + next-round message for the FIRST fold (mixed phase): the +/// K-witness folds into E (`(1+r).mul_base(f0) + r.mul_base(f1)`), the basis +/// folds in E, and the next-round message is built over the freshly folded +/// E values in the same pass. Mirror of `ligerito::fold_and_msg_lsb`. +fn fold_and_msg_lsb_base(f: &[F64], b: &[F192], r: F192) -> (Vec, Vec, SumcheckMessage) { use rayon::prelude::*; let n = f.len(); debug_assert!(n.is_power_of_two() && n >= 2); debug_assert_eq!(b.len(), n); let half = n / 2; + // Characteristic-two interpolation needs one product rather than two: + // `x0·(1+r) + x1·r = x0 + r·(x0+x1)`. The witness uses the mixed K×E + // product; the basis uses a full E product. Both are bit-identical to the + // two-product form and each still performs just one reduction. + let fold_f = |j: usize| -> F192 { F192::from(f[2 * j]) + r.mul_base(f[2 * j] + f[2 * j + 1]) }; + let fold_b = |j: usize| -> F192 { b[2 * j] + r * (b[2 * j] + b[2 * j + 1]) }; const PAR_THRESHOLD: usize = 4096; if half < PAR_THRESHOLD { let mut nf = Vec::with_capacity(half); let mut nb = Vec::with_capacity(half); for j in 0..half { - let (nfj, nbj) = fold_fb_pair(f[2 * j], f[2 * j + 1], b[2 * j], b[2 * j + 1], r); - nf.push(nfj); - nb.push(nbj); + nf.push(fold_f(j)); + nb.push(fold_b(j)); } - let mut u_0 = F128::ZERO; - let mut u_2 = F128::ZERO; + let mut u_0 = F192Unreduced::ZERO; + let mut u_2 = F192Unreduced::ZERO; let mut k = 0; while k + 1 < half { - let (p0, p2) = msg_pair_products(nf[k], nf[k + 1], nb[k], nb[k + 1]); - u_0 += p0; - u_2 += p2; + let f0 = nf[k]; + let f1 = nf[k + 1]; + let b0 = nb[k]; + let b1 = nb[k + 1]; + u_0 ^= f0.mul_unreduced(b0); + u_2 ^= (f0 + f1).mul_unreduced(b0 + b1); k += 2; } - return (nf, nb, SumcheckMessage { u_0, u_2 }); + return ( + nf, + nb, + SumcheckMessage { + u_0: u_0.reduce(), + u_2: u_2.reduce(), + }, + ); } - // Parallel path: `half` is a power of two ≥ PAR_THRESHOLD and CHUNK is a + // Parallel path: `half` is a power of two >= PAR_THRESHOLD and CHUNK is a // power of two, so every chunk has even length and starts at an even - // global index — message pairs (2k, 2k+1) never straddle a chunk boundary. + // global index (message pairs never straddle a chunk boundary). const CHUNK: usize = 2048; - let mut nf = primitives::alloc_uninit_vec::(half); - let mut nb = primitives::alloc_uninit_vec::(half); + let mut nf: Vec = fold_out_buf(half); + let mut nb: Vec = fold_out_buf(half); let (u_0, u_2) = nf .par_chunks_mut(CHUNK) .zip(nb.par_chunks_mut(CHUNK)) @@ -2125,116 +1893,213 @@ fn fold_and_msg_lsb(f: &[F128], b: &[F128], r: F128) -> (Vec, Vec, S .map(|(ci, (fc, bc))| { let base = ci * CHUNK; let len = fc.len(); - let mut u0 = F128::ZERO; - let mut u2 = F128::ZERO; - // Fold this slice, then pair up the just-folded values for the msg. + let mut u0 = F192Unreduced::ZERO; + let mut u2 = F192Unreduced::ZERO; for t in 0..len { let j = base + t; - let (nfj, nbj) = fold_fb_pair(f[2 * j], f[2 * j + 1], b[2 * j], b[2 * j + 1], r); - fc[t] = nfj; - bc[t] = nbj; + fc[t] = fold_f(j); + bc[t] = fold_b(j); } let mut k = 0; while k + 1 < len { - let (p0, p2) = msg_pair_products(fc[k], fc[k + 1], bc[k], bc[k + 1]); - u0 += p0; - u2 += p2; + let f0 = fc[k]; + let f1 = fc[k + 1]; + let b0 = bc[k]; + let b1 = bc[k + 1]; + u0 ^= f0.mul_unreduced(b0); + u2 ^= (f0 + f1).mul_unreduced(b0 + b1); k += 2; } - (u0, u2) + (u0.reduce(), u2.reduce()) }) - .reduce( - || (F128::ZERO, F128::ZERO), - |(a0, a2), (c0, c2)| (a0 + c0, a2 + c2), + .reduce(|| (F192::ZERO, F192::ZERO), |(a0, a2), (c0, c2)| (a0 + c0, a2 + c2)); + (nf, nb, SumcheckMessage { u_0, u_2 }) +} + +/// Fused fold + next-round message for the pure-E phase. Mirror of +/// `ligerito::fold_and_msg_lsb`. +fn fold_and_msg_lsb_ext(f: &[F192], b: &[F192], r: F192) -> (Vec, Vec, SumcheckMessage) { + use rayon::prelude::*; + let n = f.len(); + debug_assert!(n.is_power_of_two() && n >= 2); + debug_assert_eq!(b.len(), n); + let half = n / 2; + + // One-multiply characteristic-two interpolation, as in + // [`fold_and_msg_lsb_base`]. + let fold_pair = |x0: F192, x1: F192| -> F192 { x0 + r * (x0 + x1) }; + const PAR_THRESHOLD: usize = 4096; + if half < PAR_THRESHOLD { + let mut nf = Vec::with_capacity(half); + let mut nb = Vec::with_capacity(half); + for j in 0..half { + nf.push(fold_pair(f[2 * j], f[2 * j + 1])); + nb.push(fold_pair(b[2 * j], b[2 * j + 1])); + } + let mut u_0 = F192Unreduced::ZERO; + let mut u_2 = F192Unreduced::ZERO; + let mut k = 0; + while k + 1 < half { + let f0 = nf[k]; + let f1 = nf[k + 1]; + let b0 = nb[k]; + let b1 = nb[k + 1]; + u_0 ^= f0.mul_unreduced(b0); + u_2 ^= (f0 + f1).mul_unreduced(b0 + b1); + k += 2; + } + return ( + nf, + nb, + SumcheckMessage { + u_0: u_0.reduce(), + u_2: u_2.reduce(), + }, ); + } + + const CHUNK: usize = 2048; + let mut nf: Vec = fold_out_buf(half); + let mut nb: Vec = fold_out_buf(half); + let (u_0, u_2) = nf + .par_chunks_mut(CHUNK) + .zip(nb.par_chunks_mut(CHUNK)) + .enumerate() + .map(|(ci, (fc, bc))| { + let base = ci * CHUNK; + let len = fc.len(); + let mut u0 = F192Unreduced::ZERO; + let mut u2 = F192Unreduced::ZERO; + for t in 0..len { + let j = base + t; + fc[t] = fold_pair(f[2 * j], f[2 * j + 1]); + bc[t] = fold_pair(b[2 * j], b[2 * j + 1]); + } + let mut k = 0; + while k + 1 < len { + let f0 = fc[k]; + let f1 = fc[k + 1]; + let b0 = bc[k]; + let b1 = bc[k + 1]; + u0 ^= f0.mul_unreduced(b0); + u2 ^= (f0 + f1).mul_unreduced(b0 + b1); + k += 2; + } + (u0.reduce(), u2.reduce()) + }) + .reduce(|| (F192::ZERO, F192::ZERO), |(a0, a2), (c0, c2)| (a0 + c0, a2 + c2)); (nf, nb, SumcheckMessage { u_0, u_2 }) } -pub struct SumcheckProver { - f: Vec, - /// Single combined basis poly. After every `glue(β)`, the introduced - /// `b_new` is folded into here as `combined_basis += β · b_new`. This - /// keeps fold cost O(1 + 1) = (f + combined_basis) regardless of how - /// many level intro/glue pairs have happened. - combined_basis: Vec, - t_r: F128, - pending_glue: Option<(Vec, F128)>, +/// Two-phase witness: the committed K-message (borrowed from the caller, it +/// is only read until the first fold) before the first fold, an owned +/// E-vector afterwards. +enum Witness<'a> { + Base(&'a [F64]), + Ext(Vec), +} + +/// Mirror of `ligerito::SumcheckProver` with the two-phase witness. +pub struct SumcheckProver<'a> { + f: Witness<'a>, + /// Single combined basis poly: after every `glue(beta)` the introduced + /// basis is folded in as `combined_basis += beta * b_new`. + combined_basis: Vec, + t_r: F192, + transcript: Vec, + round: usize, + pending_glue: Option<(Vec, F192)>, } -impl SumcheckProver { - pub fn new(f: Vec, b1: Vec, h1: F128) -> (Self, SumcheckMessage) { +impl<'a> SumcheckProver<'a> { + pub fn new(f: &'a [F64], b1: Vec, h1: F192) -> (Self, SumcheckMessage) { + let _span = tracing::info_span!("Sumcheck round", round = 0, log_size = f.len().trailing_zeros()).entered(); assert_eq!(f.len(), b1.len()); - let inst = Self { - f, + let msg = round_msg_lsb_base(f, &b1); + let mut inst = Self { + f: Witness::Base(f), combined_basis: b1, t_r: h1, + transcript: Vec::new(), + round: 0, pending_glue: None, }; - let msg = round_msg_lsb(&inst.f, &inst.combined_basis); + inst.transcript.push(msg); (inst, msg) } - /// Like [`Self::new`] but skips the initial `round_msg_lsb` pass over - /// `(f, b1)` because the caller already computed `(u_0, u_2)` while - /// building `b1` (saves a 256 MB read pass at m=30 BLAKE3). Used by - /// `multilevel_prover_with_basis` to consume the round0 prime that - /// `compute_combined_basis_and_target` produces for free. - pub fn new_with_first_msg( - f: Vec, - b1: Vec, - h1: F128, - first_msg: SumcheckMessage, - ) -> (Self, SumcheckMessage) { - assert_eq!(f.len(), b1.len()); - let inst = Self { - f, - combined_basis: b1, - t_r: h1, - pending_glue: None, + pub fn fold(&mut self, r: F192) -> SumcheckMessage { + self.round += 1; + let log_size = match &self.f { + Witness::Base(f) => f.len().trailing_zeros(), + Witness::Ext(f) => f.len().trailing_zeros(), }; - (inst, first_msg) - } - - pub fn fold(&mut self, r: F128) -> SumcheckMessage { - // Fused: fold f and combined_basis at r AND build the next-round - // message in one parallel pass (was three passes). See - // [`fold_and_msg_lsb`]. - let (nf, nb, msg) = fold_and_msg_lsb(&self.f, &self.combined_basis, r); - self.f = nf; - self.combined_basis = nb; + let _span = tracing::info_span!("Sumcheck round", round = self.round, log_size).entered(); + let (nf, nb, msg) = match &self.f { + Witness::Base(f) => fold_and_msg_lsb_base(f, &self.combined_basis, r), + Witness::Ext(f) => fold_and_msg_lsb_ext(f, &self.combined_basis, r), + }; + // Swap the freshly folded buffers in and reclaim the consumed ones. On + // x86_64 the old E buffers return to the scratch pool so the next + // round's `fold_out_buf` reuses resident pages instead of faulting a + // fresh mapping (the base witness is borrowed — nothing to reclaim). + // Credit: flock (flock-core) scratch-pool reuse. + let old_f = std::mem::replace(&mut self.f, Witness::Ext(nf)); + let old_b = std::mem::replace(&mut self.combined_basis, nb); + #[cfg(target_arch = "x86_64")] + { + if let Witness::Ext(v) = old_f { + primitives::scratch::give_f192(v); + } + primitives::scratch::give_f192(old_b); + } + #[cfg(not(target_arch = "x86_64"))] + { + drop(old_f); + drop(old_b); + } + self.transcript.push(msg); msg } - /// Introduce a fresh basis poly with claimed sum `h_new`. Sends the + /// Introduce a fresh basis poly with claimed sum `h_new`; sends the /// (u_0, u_2) for `Σ_x f(x) · b_new(x)` at the current dim. - pub fn introduce_new(&mut self, b_new: Vec, h_new: F128) -> SumcheckMessage { - assert_eq!(b_new.len(), self.f.len()); - let msg = round_msg_lsb(&self.f, &b_new); + pub fn introduce_new(&mut self, b_new: Vec, h_new: F192) -> SumcheckMessage { + let msg = match &self.f { + Witness::Base(f) => { + assert_eq!(b_new.len(), f.len()); + round_msg_lsb_base(f, &b_new) + } + Witness::Ext(f) => { + assert_eq!(b_new.len(), f.len()); + round_msg_lsb_ext(f, &b_new) + } + }; + self.transcript.push(msg); self.pending_glue = Some((b_new, h_new)); msg } - /// Like [`Self::introduce_new`] but also returns the claimed sum - /// `h_new = Σ_x f(x)·b_new(x)`, computed in the same pass as the round - /// message. For OOD binding `b_new = build_eq(z)`, so `h_new` is the MLE - /// eval `f̂(z)` — fusing it here removes the separate `mle_eval_inline` - /// fold over `f`. Transcript-identical: the caller observes the returned - /// `h_new` then `(u_0, u_2)`, exactly as the unfused path does. - pub fn introduce_new_with_eval(&mut self, b_new: Vec) -> (SumcheckMessage, F128) { - assert_eq!(b_new.len(), self.f.len()); - let (msg, h_new) = round_msg_and_eval_lsb(&self.f, &b_new); + /// Introduce `b_new` and compute its claimed inner product in the same + /// pass as the round message. OOD claims only occur after the first fold, + /// when the witness has already been lifted from K to E. + pub fn introduce_new_with_eval(&mut self, b_new: Vec) -> (SumcheckMessage, F192) { + let f = match &self.f { + Witness::Ext(f) => f, + Witness::Base(_) => panic!("OOD claim introduced before the first fold"), + }; + assert_eq!(b_new.len(), f.len()); + let (msg, h_new) = round_msg_and_eval_lsb_ext(f, &b_new); + self.transcript.push(msg); self.pending_glue = Some((b_new, h_new)); (msg, h_new) } - /// Combine the introduced basis into `combined_basis` with separation α. - /// `combined_basis[j] += α · b_new[j]` (pointwise), `T_r += α · h_new`. - pub fn glue(&mut self, alpha: F128) { + /// Combine the introduced basis into `combined_basis` with separation + /// `alpha`: `combined_basis[j] += alpha * b_new[j]`, `T_r += alpha * h_new`. + pub fn glue(&mut self, alpha: F192) { use rayon::prelude::*; - let (b_new, h_new) = self - .pending_glue - .take() - .expect("glue without introduce_new"); + let (b_new, h_new) = self.pending_glue.take().expect("glue without introduce_new"); assert_eq!(b_new.len(), self.combined_basis.len()); const PAR_THRESHOLD: usize = 4096; if self.combined_basis.len() < PAR_THRESHOLD { @@ -2251,138 +2116,160 @@ impl SumcheckProver { self.t_r += alpha * h_new; } - pub fn f(&self) -> &[F128] { - &self.f + /// The folded witness (post-first-fold: always E). Panics if called + /// before the first fold (the base phase never reaches a commit). + pub fn f_ext(&self) -> &[F192] { + match &self.f { + Witness::Ext(f) => f, + Witness::Base(_) => panic!("witness still in base phase (no fold yet)"), + } + } + + pub fn transcript(&self) -> &[SumcheckMessage] { + &self.transcript } } // =================================================================== -// Prover / Verifier +// Proof // =================================================================== -// --------------------------------------------------------------------------- -// Per-query opening (no dedup / no sort) — the core `_with_basis` path. -// -// The verification algorithm samples `count` query positions in transcript order -// and verifies each opening INDEPENDENTLY (one Merkle path per query). It does NOT -// dedup or sort — that stays a proof-STORAGE compression (the octopus), expanded -// before verification. This keeps the (recursive) verifier's logic flat, so an -// in-circuit port carries no dedup/sort machinery. -// --------------------------------------------------------------------------- - -/// What the succinct multilevel verifier hands back on accept: the data a -/// recursion harness needs to drive an in-circuit replay, all named and typed -/// (no transcript scraping). -#[derive(Clone, Debug)] -pub struct LigVerifierSummary { - /// Every fold challenge, in order (the full `ris` vector the residual - /// eval_b consumes). - pub ris: Vec, - /// The raw query-sampling squeezes, per level in transcript order (each - /// word packs `128 / depth` positions). - pub query_squeezes: Vec>, +/// L0 opened rows: F64 (the commitment field). +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct InitialProof { + /// One row per query (`num_interleaved` F64 entries), sorted by query + /// position to align with the Merkle multi-proof. + pub opened_rows: Vec>, + pub merkle_proof: Vec, } -/// [`sample_queries_ordered`], also returning the raw squeezed words. -/// Sample `count` query positions in transcript order — no dedup, no sort. -/// `block_len = 2^d`; each squeezed field element yields `⌊128/d⌋` positions — -/// its disjoint d-bit chunks, low bits first. Positions stay uniform and the -/// whole squeeze stays transcript-bound; packing them amortizes one squeeze -/// (and, in the recursive verifier, one 128-bit decomposition) across `128/d` -/// queries. -fn sample_queries_ordered_with_raw( - sponge: &mut Sponge, - block_len: usize, - count: usize, -) -> (Vec, Vec) { - let d = block_len.trailing_zeros() as usize; - let per = 128 / d; - let mut out = Vec::with_capacity(count); - let mut raw = Vec::with_capacity(count.div_ceil(per)); - while out.len() < count { - let v = sponge.sample(); - raw.push(v); - let bits = (v.lo as u128) | ((v.hi as u128) << 64); - for j in 0..per.min(count - out.len()) { - out.push(((bits >> (j * d)) as usize) & (block_len - 1)); - } - } - (out, raw) +/// Deeper-level opened rows: E-valued. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecursiveProof { + pub opened_rows: Vec>, + pub merkle_proof: Vec, } -fn sample_queries_ordered(sponge: &mut Sponge, block_len: usize, count: usize) -> Vec { - let d = block_len.trailing_zeros() as usize; - let per = 128 / d; - let mut out = Vec::with_capacity(count); - while out.len() < count { - let v = sponge.sample(); - let bits = (v.lo as u128) | ((v.hi as u128) << 64); - for j in 0..per.min(count - out.len()) { - out.push(((bits >> (j * d)) as usize) & (block_len - 1)); - } - } - out +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct FinalProof { + /// Remaining polynomial sent in clear at the last recursive step. + pub yr: Vec, + pub opened_rows: Vec>, + pub merkle_proof: Vec, } -/// Verify each query's single Merkle path against `root` (no octopus, no sort). -fn verify_level_opens_perquery( - root: &Hash, - block_len: usize, - queries: &[usize], - opened_rows: &[Vec], - expected_num_interleaved: usize, - paths: &[Hash], -) -> bool { - if queries.len() != opened_rows.len() { - return false; - } - let depth = block_len.trailing_zeros() as usize; - if paths.len() != queries.len() * depth { - return false; - } - for (j, (&q, row)) in queries.iter().zip(opened_rows).enumerate() { - if row.len() != expected_num_interleaved { - return false; +/// The L0 root is the caller's statement, not proof data. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LigeritoProof { + pub initial_proof: InitialProof, + pub recursive_roots: Vec, + pub recursive_proofs: Vec, + pub final_proof: FinalProof, + pub sumcheck_transcript: Vec, + /// Per-level query-phase PoW nonces (0 when the level grinds 0 bits). + pub grinding_nonces: Vec, + /// Claimed multilinear OOD evaluations, flattened in transcript order. + #[serde(default)] + pub ood_values: Vec, + /// Fold-challenge PoW nonces, flattened in transcript order (one per fold + /// challenge at every level with `fold_grinding_bits > 0`). + pub fold_grinding_nonces: Vec, +} + +impl LigeritoProof { + pub fn size_bytes(&self) -> usize { + const EXT: usize = core::mem::size_of::(); + const BASE: usize = core::mem::size_of::(); + let mut total = 0usize; + total += self.recursive_roots.len() * 32; + total += self + .initial_proof + .opened_rows + .iter() + .map(|r| r.len() * BASE) + .sum::() + + self.initial_proof.merkle_proof.len() * 32; + for p in &self.recursive_proofs { + total += p.opened_rows.iter().map(|r| r.len() * EXT).sum::() + p.merkle_proof.len() * 32; } - let bytes: &[u8] = - unsafe { core::slice::from_raw_parts(row.as_ptr() as *const u8, row.len() * core::mem::size_of::()) }; - let leaf = merkle::hash_leaf(bytes); - if !merkle::verify_merkle_proof(root, &leaf, q, &paths[j * depth..(j + 1) * depth]) { - return false; + total += self.final_proof.yr.len() * EXT + + self + .final_proof + .opened_rows + .iter() + .map(|r| r.len() * EXT) + .sum::() + + self.final_proof.merkle_proof.len() * 32; + total += self.sumcheck_transcript.len() * 2 * EXT; + total += self.ood_values.len() * EXT; + total += (self.grinding_nonces.len() + self.fold_grinding_nonces.len()) * 8; + total + } +} + +// =================================================================== +// Prover +// =================================================================== + +/// Sample `count` distinct positions in `[0, block_len)`. Same sponge +/// pattern as the original (`sample().c0 % block_len`). +/// Sample `count` query positions in transcript order — no dedup, no sort. +/// `block_len = 2^d`; each squeezed field element yields `⌊192/d⌋` positions as +/// its disjoint d-bit chunks (low bits first). Mirror of +/// `ligerito::sample_queries_ordered` so the K opener uses the exact +/// recursion-friendly scheme the harness/guest re-derive (fixed `192/d` per +/// squeeze, dup-tolerant — soundness matches the deployed PCS with the same +/// `config.queries`). Duplicates are harmless: a repeated position re-opens the +/// same Merkle-authenticated row. +fn sample_queries_ordered(sponge: &mut Sponge, block_len: usize, count: usize) -> Vec { + let d = block_len.trailing_zeros() as usize; + let per = 192 / d; + let mut out = Vec::with_capacity(count); + while out.len() < count { + let v = sponge.sample(); + for j in 0..per.min(count - out.len()) { + let off = j * d; + let limbs = [v.c0, v.c1, v.c2]; + let (li, sh) = (off / 64, off % 64); + let mut chunk = limbs[li] >> sh; + if sh + d > 64 { + chunk |= limbs[li + 1] << (64 - sh); + } + out.push(chunk as usize & (block_len - 1)); } } - true + out } -// --------------------------------------------------------------------------- -// Storage compression ↔ per-query expansion. -// -// A level's opening is TRANSMITTED compressed (index-deduplicated rows + a single -// octopus Merkle multi-proof) and EXPANDED back to the flat per-query form the -// verifier's authentication + enforced-sum math consume. Queries are sampled -// with replacement in transcript order (`sample_queries_ordered`, cheap to port -// in-circuit); the compression is a pure storage layer that never enters the -// (recursive) verifier's flat per-query logic. -// --------------------------------------------------------------------------- - -/// Sort + dedup a query list into its strictly-ascending distinct positions — -/// the alignment of the stored (compressed) `opened_rows` and the octopus. -fn sorted_unique_queries(queries: &[usize]) -> Vec { - let mut s = queries.to_vec(); - s.sort_unstable(); - s.dedup(); - s +/// [`sample_queries_ordered`] that ALSO returns the raw squeezed words `v` +/// (as native `F192` — the recursion harness reads +/// all three limbs off them to re-derive positions). One raw word per squeeze. +fn sample_queries_ordered_with_raw(sponge: &mut Sponge, block_len: usize, count: usize) -> (Vec, Vec) { + let d = block_len.trailing_zeros() as usize; + let per = 192 / d; + let mut out = Vec::with_capacity(count); + let mut raw = Vec::with_capacity(count.div_ceil(per)); + while out.len() < count { + let v = sponge.sample(); + raw.push(v); + for j in 0..per.min(count - out.len()) { + let off = j * d; + let limbs = [v.c0, v.c1, v.c2]; + let (li, sh) = (off / 64, off % 64); + let mut chunk = limbs[li] >> sh; + if sh + d > 64 { + chunk |= limbs[li + 1] << (64 - sh); + } + out.push(chunk as usize & (block_len - 1)); + } + } + (out, raw) } -/// Fan a level's stored (index-deduplicated, sorted-unique) `opened_rows` back -/// out to the transcript-sampled query order (duplicates included) — the -/// alignment the enforced-sum / induced-basis math indexes by. `queries` is the -/// ordered list re-derived from the transcript. Returns `None` if the stored row -/// count does not match the distinct-query count (malformed proof). -pub fn expand_opened_rows_ordered( - rows_sorted: &[Vec], - queries: &[usize], -) -> Option>> { +/// Fan stored sorted-unique rows back to transcript (ordered, dup-possible) +/// order, so the induce math sees `opened_rows[i]` ↔ `queries[i]`. The rows must +/// already be authenticated (via the octopus check) against the level root. +fn fan_rows_to_ordered(queries: &[usize], rows_sorted: &[Vec]) -> Option>> { let sorted = sorted_unique_queries(queries); if sorted.len() != rows_sorted.len() { return None; @@ -2395,110 +2282,41 @@ pub fn expand_opened_rows_ordered( Some(out) } -/// Expand a level's stored (compressed) opening into the flat per-query form the -/// recursion-friendly verifier consumes: `(rows_ordered, flat_paths)` — one row -/// and one full `⌈log2(block_len)⌉`-deep Merkle path per query, in transcript -/// order (duplicates included). `queries` is the ordered list; `rows_sorted` / -/// `octopus` are the stored `LevelProof` fields. Returns `None` on a -/// malformed proof (wrong row width or unrecoverable octopus). It authenticates -/// nothing itself — the caller re-checks each restored path against the root, so -/// a bad expansion is caught there. Inverse of [`compress_level_opening`]. -pub fn expand_level_opening( - block_len: usize, - queries: &[usize], - rows_sorted: &[Vec], - expected_num_interleaved: usize, - octopus: &[Hash], -) -> Option<(Vec>, Vec)> { - let sorted = sorted_unique_queries(queries); - if sorted.len() != rows_sorted.len() { - return None; - } - let mut leaf_hashes = Vec::with_capacity(rows_sorted.len()); - for row in rows_sorted { - if row.len() != expected_num_interleaved { - return None; - } - let bytes: &[u8] = unsafe { - core::slice::from_raw_parts( - row.as_ptr() as *const u8, - row.len() * core::mem::size_of::(), - ) - }; - leaf_hashes.push(merkle::hash_leaf(bytes)); - } - let flat_paths = merkle::restore_multi_proof(block_len, queries, &leaf_hashes, octopus)?; - let rows_ordered = expand_opened_rows_ordered(rows_sorted, queries)?; - Some((rows_ordered, flat_paths)) -} - -/// Compress a level's opening for STORAGE: index-deduplicate the transcript- -/// ordered `queries` (keep one row per distinct position, in sorted order) and -/// build the shared octopus multi-proof (Merkle path pruning). `row_at` reads the -/// opened row at a position. Returns `(rows_sorted, octopus)` for a -/// `LevelProof`. Inverse of [`expand_level_opening`]. -fn compress_level_opening( - tree: &[Hash], - block_len: usize, - queries: &[usize], - mut row_at: impl FnMut(usize) -> Vec, -) -> (Vec>, Vec) { - let sorted = sorted_unique_queries(queries); - let rows = sorted.iter().map(|&q| row_at(q)).collect(); - let octopus = merkle::merkle_multi_proof(tree, block_len, &sorted); - (rows, octopus) +fn merkle_multi_proof_for(tree: &[Hash], block_len: usize, queries: &[usize]) -> Vec { + merkle::merkle_multi_proof(tree, block_len, queries) } -/// The multilevel Ligerito prover over a generic basis poly + target -/// (typically the combined `Σ γ_k · eq(z_k, ·)` and target produced by -/// `ring_switch::prove_batched_padded_with_precomputed`), against an -/// externally-built L0 commitment (the `pcs::commit` output). +/// Prove `Σ_x witness(x) · b_initial(x) = target` against the L0 commitment +/// produced by [`commit`] (with `log_batch_size = config.initial_k` and +/// `log_inv_rate = config.log_inv_rates[0]`). /// -/// The initial step runs `initial_k` real sumcheck rounds folding `f` and `b` -/// together with FS challenges (a combined basis has no single `z` to -/// partial-evaluate at); the folded `f` becomes the L1 witness and each later -/// level re-commits and folds. -pub fn multilevel_prover_with_basis( - config: &LigeritoConfig, - packed_witness: Vec, - b_initial: Vec, - target: F128, - l0_codeword: &[F128], - l0_tree: &[Hash], - ps: &mut ProverState, -) -> LigeritoProof { - multilevel_prover_with_basis_impl( - config, - packed_witness, - b_initial, - target, - l0_codeword, - l0_tree, - None, - ps, - ) -} - -#[allow(clippy::too_many_arguments)] -fn multilevel_prover_with_basis_impl( - config: &LigeritoConfig, - packed_witness: Vec, - b_initial: Vec, - target: F128, - l0_codeword: &[F128], +/// `witness` is borrowed: it is only READ (round-0 message + the first lane +/// fold, which lifts it into an owned E-vector), so callers with a large +/// committed stack pass the slice directly instead of paying a full copy. +/// +/// Transcript order is identical to the original (target, roots, OOD claims, +/// `(u_0, u_2)` stream, tapered fold grinds, query grinds, queries, alphas, +/// betas, and `yr` in the clear at the end). +pub fn recursive_prover_with_basis( + config: &ProverConfig, + witness: &[F64], + b_initial: Vec, + target: F192, + l0_codeword: &[F64], l0_tree: &[Hash], - first_msg: Option, - ps: &mut ProverState, + sponge: &mut Sponge, ) -> LigeritoProof { - let log_n = packed_witness.len().trailing_zeros() as usize; + let log_n = witness.len().trailing_zeros() as usize; let r = config.level_steps; let initial_k = config.initial_k; - assert_eq!(packed_witness.len(), 1usize << log_n); + assert_eq!(witness.len(), 1usize << log_n); assert_eq!(b_initial.len(), 1usize << log_n); assert_eq!(config.level_ks.len(), r); assert_eq!(config.log_inv_rates.len(), r + 1); assert!(r >= 1); + assert!(initial_k >= 1); + assert_eq!(config.ood_samples.first().copied().unwrap_or(0), 0); let log_inv_rate_0 = config.log_inv_rates[0]; let log_msg_cols_0 = log_n - initial_k; @@ -2507,7 +2325,10 @@ fn multilevel_prover_with_basis_impl( assert_eq!(l0_codeword.len(), block_len_0 * num_interleaved_0); assert_eq!(l0_tree.len(), 2 * block_len_0 - 1); - let trace = std::env::var("LIG_PROVE_TRACE").is_ok(); + // Optional per-phase timing (LIGERITO_TRACE): mirror of the original's + // LIG_PROVE_TRACE. One env lookup per prove; the Instant reads are + // negligible and the accumulation/printing is gated on `trace`. + let trace = std::env::var_os("LIGERITO_TRACE").is_some(); let mut t_init_sumcheck = std::time::Duration::ZERO; let mut t_commits = std::time::Duration::ZERO; let mut t_opens = std::time::Duration::ZERO; @@ -2515,163 +2336,114 @@ fn multilevel_prover_with_basis_impl( let mut t_sumcheck_folds = std::time::Duration::ZERO; let mut t_intro_glue = std::time::Duration::ZERO; let mut t_ood = std::time::Duration::ZERO; - let t_total = std::time::Instant::now(); - ps.observe_scalar(target); + // (No opener domain-label absorb: the extension-field opener has none and the recursion + // guest replays a label-free opening transcript; the observed `target` + + // outer transcript context provide domain separation.) + observe_ext(sponge, target); - // L0 codeword + tree are borrowed (reused from upstream `pcs::commit`). - // wtns_0 access reduces to: root (last tree node), row(q), block_len. + // L0 codeword + tree are borrowed (reused from `commit`). let initial_root: Hash = l0_tree[l0_tree.len() - 1]; - let l0_block_len = block_len_0; - let l0_num_interleaved = num_interleaved_0; - let l0_row = |q: usize| -> &[F128] { - let start = q * l0_num_interleaved; - &l0_codeword[start..start + l0_num_interleaved] + let l0_row = |q: usize| -> &[F64] { + let start = q * num_interleaved_0; + &l0_codeword[start..start + num_interleaved_0] }; - ps.absorb_bytes(&initial_root); - - // L0 takes no explicit OOD samples: it is bound by the opening's own - // evaluation claim (`target` at the post-commit random point behind - // `b_initial`), which plays the OOD role with a union over the list - // instead of over pairs. See `paper_ood_bits`. - assert_eq!( - config.ood_samples.first().copied().unwrap_or(0), - 0, - "L0 must not take explicit OOD samples" - ); - let fold_bits = - |lvl: usize| -> u32 { config.fold_grinding_bits.get(lvl).copied().unwrap_or(0) as u32 }; + observe_root(sponge, &initial_root); + + let mut fold_grinding_nonces: Vec = Vec::new(); + let mut ood_values: Vec = Vec::new(); + let fold_bits = |lvl: usize| -> u32 { config.fold_grinding_bits.get(lvl).copied().unwrap_or(0) as u32 }; let ood_count = |lvl: usize| -> usize { config.ood_samples.get(lvl).copied().unwrap_or(0) }; let _t = std::time::Instant::now(); let sumcheck_span = tracing::info_span!("Sumcheck"); - let initial_log_size = packed_witness.len().trailing_zeros(); - let (mut sc_prover, start_msg) = sumcheck_span.in_scope(|| { - tracing::info_span!( - "Sumcheck round", - round = %pretty_integer(0), - log_size = %pretty_integer(initial_log_size) - ) - .in_scope(|| match first_msg { - Some(msg) => { - SumcheckProver::new_with_first_msg(packed_witness, b_initial, target, msg) - } - None => SumcheckProver::new(packed_witness, b_initial, target), - }) - }); - add_sumcheck_msg(ps, &start_msg); + let (mut sc_prover, start_msg) = sumcheck_span.in_scope(|| SumcheckProver::new(witness, b_initial, target)); + observe_ext(sponge, start_msg.u_0); + observe_ext(sponge, start_msg.u_2); let mut r_lane_fold = Vec::with_capacity(initial_k); for j in 0..initial_k { - // Fold-challenge grinding: the L0 proximity-gap bad event lives on - // each of these lane-fold challenges, so each one is individually - // PoW-guarded (a cheating prover re-rolls a fold challenge by - // varying the preceding sumcheck message; the grind prices every - // such attempt). Tapered per round: round j folds a 2^{ℓ-j}-row word - // whose MCA error carries the factor 2^{ℓ-1-j} (App. C.3 Lemma - // `mca-commutes`), so it needs (fold_bits − j) bits — one fewer per - // round than the worst (j=0) round `fold_grinding_bits` is sized for. - // Derived from fold_grinding_bits + round index; not stored. + // Tapered fold-challenge grinding: round j of the lane fold needs + // (fold_bits - j) bits (worst round j=0 carries the full budget); see + // the original's App. C.3 `mca-commutes` comment. let bits = fold_bits(0).saturating_sub(j as u32); if bits > 0 { - ps.grind(bits); + fold_grinding_nonces.push(sponge.grind_pow(bits)); } - let r = ps.sample(); - let log_size = sc_prover.f().len().trailing_zeros(); - let msg = sumcheck_span.in_scope(|| { - tracing::info_span!( - "Sumcheck round", - round = %pretty_integer(j + 1), - log_size = %pretty_integer(log_size) - ) - .in_scope(|| sc_prover.fold(r)) - }); - add_sumcheck_msg(ps, &msg); - r_lane_fold.push(r); + let r_j = sample_ext(sponge); + let msg = sumcheck_span.in_scope(|| sc_prover.fold(r_j)); + observe_ext(sponge, msg.u_0); + observe_ext(sponge, msg.u_2); + r_lane_fold.push(r_j); } drop(sumcheck_span); if trace { t_init_sumcheck += _t.elapsed(); } - // Commit f^1 = folded packed witness as wtns_1. + // Commit f^1 = folded (now E-valued) witness as wtns_1. let n1 = log_n - initial_k; let log_num_interleaved_1 = config.level_ks[0]; assert!(n1 >= log_num_interleaved_1); let log_msg_cols_1 = n1 - log_num_interleaved_1; let log_inv_rate_1 = config.log_inv_rates[1]; let _t = std::time::Instant::now(); - let ntt_1 = AdditiveNttF128::standard(log_msg_cols_1 + log_inv_rate_1); - let f1 = sc_prover.f().to_vec(); - let wtns_1 = ligero_commit( - &f1, - log_msg_cols_1, - log_num_interleaved_1, - log_inv_rate_1, - &ntt_1, - ); + let ntt_1 = AdditiveNttF64::standard(log_msg_cols_1 + log_inv_rate_1); + let f1 = sc_prover.f_ext().to_vec(); + let wtns_1 = ligero_commit_ext(&f1, log_msg_cols_1, log_num_interleaved_1, log_inv_rate_1, &ntt_1); if trace { t_commits += _t.elapsed(); } - ps.add_scalars(&merkle::hash_to_scalars(&wtns_1.root())); + observe_root(sponge, &wtns_1.root()); - // OOD binding for the L1 commit: each sample evaluates f1's multilinear - // extension at a random transcript point z ∈ F^{n1}, sends the claimed - // value, and folds the claim `Σ_x f1(x)·eq(z,x) = y` into the running - // sumcheck (introduce + glue). Binds the prover to a single codeword of - // the interleaved list before any of L0's queries are drawn. - { - let _t = std::time::Instant::now(); - for _ in 0..ood_count(1) { - let z = ps.sample_vec(n1); - // Build eq(z, ·) once and fuse the MLE eval `y = f̂1(z)` into the - // introduce round message (single pass over f1 + eq_z), instead of - // a separate `mle_eval_inline` fold. - let eq_z = build_eq(&z); - let (intro, y) = sc_prover.introduce_new_with_eval(eq_z); - ps.add_scalar(y); - add_sumcheck_msg(ps, &intro); - let beta = ps.sample(); - sc_prover.glue(beta); - } - if trace { - t_ood += _t.elapsed(); - } + // Bind the L1 Johnson list before drawing L0 queries. Each claimed random + // MLE evaluation is introduced into the running sumcheck. + let _t = std::time::Instant::now(); + for _ in 0..ood_count(1) { + let z = sample_ext_vec(sponge, n1); + let (intro, y) = sc_prover.introduce_new_with_eval(build_eq_table_ext_parallel(&z)); + observe_ext(sponge, y); + ood_values.push(y); + observe_ext(sponge, intro.u_0); + observe_ext(sponge, intro.u_2); + sc_prover.glue(sample_ext(sponge)); + } + if trace { + t_ood += _t.elapsed(); } - // Query-phase PoW grinding for L0: each ground bit substitutes for - // ~1/log₂(1/(1−γ)) queries at this level (this config grinds 18 - // bits here). Verifier mirror checks the nonce; both then proceed to - // sample query positions. (The proximity-gap shortfall is covered - // separately by the fold-challenge grinds above.) - ps.grind(config.grinding_bits[0] as u32); + // Query-phase PoW grinding for L0 (0 bits in the production profile; the + // canonical 0 nonce is still absorbed to keep the transcript in lockstep). + let pow_nonce_0 = sponge.grind_pow(config.grinding_bits[0] as u32); + let mut grinding_nonces: Vec = vec![pow_nonce_0]; // Open L0; lane-fold weights = r_lane_fold. let num_queries_0 = config.queries[0]; - let queries_0 = sample_queries_ordered(ps.sponge_mut(), l0_block_len, num_queries_0); - let alpha_0 = ps.sample_vec(log2_ceil(num_queries_0)); + let queries_0 = sample_queries_ordered(sponge, block_len_0, num_queries_0); + let alpha_0 = sample_ext_vec(sponge, log2_ceil(num_queries_0)); let _t = std::time::Instant::now(); - // `opened_rows_0` stays in transcript (ordered, possibly-duplicate) order for - // the induce-sumcheck math below; the STORED proof compresses it (index dedup - // + octopus path pruning) and the verifier re-expands before its flat checks. - let opened_rows_0: Vec> = queries_0.iter().map(|&q| l0_row(q).to_vec()).collect(); - let (stored_rows_0, merkle_proof_0) = - compress_level_opening(l0_tree, l0_block_len, &queries_0, |q| l0_row(q).to_vec()); + // Ordered (dup-possible) rows for the local induce math ... + let opened_rows_0: Vec> = queries_0.iter().map(|&q| l0_row(q).to_vec()).collect(); + // ... but the stored proof carries the sorted-unique rows + one octopus over + // the sorted-unique positions (the verifier re-fans them to ordered). + let sq_0 = sorted_unique_queries(&queries_0); + let stored_rows_0: Vec> = sq_0.iter().map(|&q| l0_row(q).to_vec()).collect(); + let merkle_proof_0 = merkle_multi_proof_for(l0_tree, block_len_0, &sq_0); if trace { t_opens += _t.elapsed(); } - let initial_proof = LevelProof { + let initial_proof = InitialProof { opened_rows: stored_rows_0, merkle_proof: merkle_proof_0, }; - // Induce basis_0 from wtns_0 opens. L0 dominates the induce phase, where the - // sparse-prefix Fᵀ-NTT path wins; the dispatcher auto-selects it (deeper - // levels stay dense). + // Induce basis_0 from the L0 opens. L0 dominates the induce phase, where + // the sparse-prefix transposed-NTT path wins; the dispatcher auto-selects + // it (deeper levels stay dense), mirroring the original. let sks_vks_n1 = eval_sk_at_vks(n1); let _t = std::time::Instant::now(); - let (basis_0_induced, enforced_sum_0) = induce_sumcheck_poly_auto( + let (basis_0_induced, enforced_sum_0) = induce_sumcheck_poly_auto_base( n1, log_inv_rate_0, &sks_vks_n1, @@ -2687,16 +2459,18 @@ fn multilevel_prover_with_basis_impl( // Introduce + glue basis_0. let _t = std::time::Instant::now(); let intro_msg_0 = sc_prover.introduce_new(basis_0_induced, enforced_sum_0); - add_sumcheck_msg(ps, &intro_msg_0); - let beta_0 = ps.sample(); + observe_ext(sponge, intro_msg_0.u_0); + observe_ext(sponge, intro_msg_0.u_2); + let beta_0 = sample_ext(sponge); sc_prover.glue(beta_0); if trace { t_intro_glue += _t.elapsed(); } - // Recursive levels — same as multilevel_prover_inner from here. + // Recursive levels. let mut wtns_prev = wtns_1; - let mut level_proofs: Vec = Vec::new(); + let mut recursive_roots: Vec = vec![wtns_prev.root()]; + let mut recursive_proofs: Vec = Vec::new(); for i in 0..r { let k_i = config.level_ks[i]; @@ -2704,24 +2478,16 @@ fn multilevel_prover_with_basis_impl( let _t = std::time::Instant::now(); let sumcheck_span = tracing::info_span!("Sumcheck"); for j in 0..k_i { - // These folds fold level i+1's commitment — fold-challenge - // grinding guards its proximity-gap term. Tapered per round: - // round j needs (fold_bits − j) bits (see L0 loop). + // These folds fold level i+1's commitment; tapered grinding as in + // the L0 loop. let bits = fold_bits(i + 1).saturating_sub(j as u32); if bits > 0 { - ps.grind(bits); + fold_grinding_nonces.push(sponge.grind_pow(bits)); } - let ri = ps.sample(); - let log_size = sc_prover.f().len().trailing_zeros(); - let msg = sumcheck_span.in_scope(|| { - tracing::info_span!( - "Sumcheck round", - round = %pretty_integer(j), - log_size = %pretty_integer(log_size) - ) - .in_scope(|| sc_prover.fold(ri)) - }); - add_sumcheck_msg(ps, &msg); + let ri = sample_ext(sponge); + let msg = sumcheck_span.in_scope(|| sc_prover.fold(ri)); + observe_ext(sponge, msg.u_0); + observe_ext(sponge, msg.u_2); level_rs.push(ri); } drop(sumcheck_span); @@ -2730,41 +2496,31 @@ fn multilevel_prover_with_basis_impl( } if i == r - 1 { - ps.add_scalars(sc_prover.f()); + let yr = sc_prover.f_ext().to_vec(); + for v in &yr { + observe_ext(sponge, *v); + } // PoW grinding for the last level before sampling its queries. - ps.grind(config.grinding_bits[i + 1] as u32); + let nonce_last = sponge.grind_pow(config.grinding_bits[i + 1] as u32); + grinding_nonces.push(nonce_last); let num_queries_last = config.queries[i + 1]; - let queries_last = - sample_queries_ordered(ps.sponge_mut(), wtns_prev.block_len, num_queries_last); - // Mirror the verifier's challenge order exactly: the last - // commitment's basis-induction challenge is drawn after `yr` is - // bound and the queries are fixed, so a forged `yr` cannot be - // adapted to it. - let alpha_last = ps.sample_vec(log2_ceil(num_queries_last)); + let queries_last = sample_queries_ordered(sponge, wtns_prev.block_len, num_queries_last); + // The final commitment's basis challenge is drawn only after `yr` + // and its queries are bound, matching the verifier exactly. + let alpha_last = sample_ext_vec(sponge, log2_ceil(num_queries_last)); let _t = std::time::Instant::now(); - // Final level: opened rows are only stored (no induce), so keep just - // the compressed (deduped + octopus) form. - let (opened_rows_last, merkle_proof_last) = compress_level_opening( - &wtns_prev.tree, - wtns_prev.block_len, - &queries_last, - |q| wtns_prev.row(q).to_vec(), - ); - if trace { - t_opens += _t.elapsed(); - } - - // Tie the last commitment into the running claim, then run the - // remaining `n_res` sumcheck rounds instead of leaving the residual - // cube for the verifier to sum over. That is what lets the verifier - // finish with ONE evaluation of its weight rather than `2^n_res` of - // them, which matters whenever the weight does not factor across the - // split (an `eq` weight does; a jagged interval indicator does not). - let rows_last: Vec> = - queries_last.iter().map(|&q| wtns_prev.row(q).to_vec()).collect(); - let enforced_sum_last = - induce_sumcheck_enforced_sum(&rows_last, &level_rs, &queries_last, &alpha_last); - let n_res = sc_prover.f().len().trailing_zeros() as usize; + // Final level: stored (sorted-unique) only — no local induce; the + // verifier fans these to ordered for its last-level induce. + let sq_last = sorted_unique_queries(&queries_last); + let opened_rows_last: Vec> = sq_last.iter().map(|&q| wtns_prev.row(q).to_vec()).collect(); + let merkle_proof_last = merkle_multi_proof_for(&wtns_prev.tree, wtns_prev.block_len, &sq_last); + // Tie the last commitment into the running claim through the same + // intro/glue step as every other level, then finish the remaining + // sumcheck rounds. This closes on one weight evaluation instead of + // a sweep over the residual cube. + let rows_last: Vec> = queries_last.iter().map(|&q| wtns_prev.row(q).to_vec()).collect(); + let enforced_sum_last = induce_sumcheck_enforced_sum_ext(&rows_last, &level_rs, &queries_last, &alpha_last); + let n_res = sc_prover.f_ext().len().trailing_zeros() as usize; let basis_last = induce_sumcheck_evaluate_at_residual( n_res, &eval_sk_at_vks(n_res), @@ -2773,75 +2529,73 @@ fn multilevel_prover_with_basis_impl( &[], n_res, ); - // Same intro/glue shape as every non-final level: the message is - // what lets the verifier fold this basis into its running round - // polynomial, so it travels. Its claimed sum does not, being - // derived from the Merkle-bound opened rows. let intro_msg_last = sc_prover.introduce_new(basis_last, enforced_sum_last); - add_sumcheck_msg(ps, &intro_msg_last); - let beta_last = ps.sample(); - sc_prover.glue(beta_last); + observe_ext(sponge, intro_msg_last.u_0); + observe_ext(sponge, intro_msg_last.u_2); + sc_prover.glue(sample_ext(sponge)); for j in 0..n_res { - let ri = ps.sample(); + let ri = sample_ext(sponge); let msg = sc_prover.fold(ri); - // The last fold leaves a single value; its "next round" message - // would describe a zero-variable round, and the verifier closes - // on the point evaluation instead. if j + 1 < n_res { - add_sumcheck_msg(ps, &msg); + observe_ext(sponge, msg.u_0); + observe_ext(sponge, msg.u_2); } } + let transmitted_sumcheck_len = sc_prover.transcript().len() - usize::from(n_res > 0); if trace { + t_opens += _t.elapsed(); let total = t_total.elapsed(); - eprintln!("[lig-prove] total = {:.2} ms", total.as_secs_f64() * 1e3); + eprintln!("[lig-prove] total = {:.4} s", total.as_secs_f64()); eprintln!( - " initial sumcheck (initial_k folds + SC build): {:.2} ms", - t_init_sumcheck.as_secs_f64() * 1e3 + " initial sumcheck (initial_k folds + SC build): {:.4} s", + t_init_sumcheck.as_secs_f64() ); eprintln!( - " level commits (NTT + merkle): {:.2} ms", - t_commits.as_secs_f64() * 1e3 + " recursive commits (NTT + merkle): {:.4} s", + t_commits.as_secs_f64() ); eprintln!( - " opens (rows + multi-proof): {:.2} ms", - t_opens.as_secs_f64() * 1e3 + " opens (rows + multi-proof, incl. final): {:.4} s", + t_opens.as_secs_f64() ); eprintln!( - " induce_sumcheck_poly: {:.2} ms", - t_induce.as_secs_f64() * 1e3 + " induce_sumcheck_poly: {:.4} s", + t_induce.as_secs_f64() ); eprintln!( - " sumcheck level folds: {:.2} ms", - t_sumcheck_folds.as_secs_f64() * 1e3 + " sumcheck recursive folds: {:.4} s", + t_sumcheck_folds.as_secs_f64() ); eprintln!( - " introduce_new + glue: {:.2} ms", - t_intro_glue.as_secs_f64() * 1e3 - ); - eprintln!( - " OOD samples: MLE evals + glue: {:.2} ms", - t_ood.as_secs_f64() * 1e3 + " introduce_new + glue: {:.4} s", + t_intro_glue.as_secs_f64() ); } return LigeritoProof { initial_proof, - level_proofs, + recursive_roots, + recursive_proofs, final_proof: FinalProof { + yr, opened_rows: opened_rows_last, merkle_proof: merkle_proof_last, }, + sumcheck_transcript: sc_prover.transcript()[..transmitted_sumcheck_len].to_vec(), + grinding_nonces, + ood_values, + fold_grinding_nonces, }; } - let n_next = sc_prover.f().len().trailing_zeros() as usize; + let n_next = sc_prover.f_ext().len().trailing_zeros() as usize; let log_num_interleaved_next = config.level_ks[i + 1]; assert!(n_next >= log_num_interleaved_next); let log_msg_cols_next = n_next - log_num_interleaved_next; let log_inv_rate_next = config.log_inv_rates[i + 2]; let _t = std::time::Instant::now(); - let ntt_next = AdditiveNttF128::standard(log_msg_cols_next + log_inv_rate_next); - let f_evals = sc_prover.f().to_vec(); - let wtns_next = ligero_commit( + let ntt_next = AdditiveNttF64::standard(log_msg_cols_next + log_inv_rate_next); + let f_evals = sc_prover.f_ext().to_vec(); + let wtns_next = ligero_commit_ext( &f_evals, log_msg_cols_next, log_num_interleaved_next, @@ -2851,68 +2605,57 @@ fn multilevel_prover_with_basis_impl( if trace { t_commits += _t.elapsed(); } - ps.add_scalars(&merkle::hash_to_scalars(&wtns_next.root())); + let root_next = wtns_next.root(); + observe_root(sponge, &root_next); + recursive_roots.push(root_next); - // OOD binding for the L_{i+2} commit (same as the L1 block above). - { - let _t = std::time::Instant::now(); - for _ in 0..ood_count(i + 2) { - let z = ps.sample_vec(n_next); - let eq_z = build_eq(&z); - let (intro, y) = sc_prover.introduce_new_with_eval(eq_z); - ps.add_scalar(y); - add_sumcheck_msg(ps, &intro); - let beta = ps.sample(); - sc_prover.glue(beta); - } - if trace { - t_ood += _t.elapsed(); - } + let _t = std::time::Instant::now(); + for _ in 0..ood_count(i + 2) { + let z = sample_ext_vec(sponge, n_next); + let (intro, y) = sc_prover.introduce_new_with_eval(build_eq_table_ext_parallel(&z)); + observe_ext(sponge, y); + ood_values.push(y); + observe_ext(sponge, intro.u_0); + observe_ext(sponge, intro.u_2); + sc_prover.glue(sample_ext(sponge)); + } + if trace { + t_ood += _t.elapsed(); } // PoW grinding for this iteration's query phase. - ps.grind(config.grinding_bits[i + 1] as u32); + let nonce_i = sponge.grind_pow(config.grinding_bits[i + 1] as u32); + grinding_nonces.push(nonce_i); let num_queries_i = config.queries[i + 1]; - let queries_i = sample_queries_ordered(ps.sponge_mut(), wtns_prev.block_len, num_queries_i); - let alpha_i = ps.sample_vec(log2_ceil(num_queries_i)); + let queries_i = sample_queries_ordered(sponge, wtns_prev.block_len, num_queries_i); + let alpha_i = sample_ext_vec(sponge, log2_ceil(num_queries_i)); let _t = std::time::Instant::now(); - // `opened_rows_i` stays ordered for the induce-sumcheck; store compressed. - let opened_rows_i: Vec> = queries_i - .iter() - .map(|&q| wtns_prev.row(q).to_vec()) - .collect(); - let (stored_rows_i, merkle_proof_i) = compress_level_opening( - &wtns_prev.tree, - wtns_prev.block_len, - &queries_i, - |q| wtns_prev.row(q).to_vec(), - ); + // Ordered rows for the local induce; sorted-unique rows + octopus stored. + let opened_rows_i: Vec> = queries_i.iter().map(|&q| wtns_prev.row(q).to_vec()).collect(); + let sq_i = sorted_unique_queries(&queries_i); + let stored_rows_i: Vec> = sq_i.iter().map(|&q| wtns_prev.row(q).to_vec()).collect(); + let merkle_proof_i = merkle_multi_proof_for(&wtns_prev.tree, wtns_prev.block_len, &sq_i); if trace { t_opens += _t.elapsed(); } - level_proofs.push(LevelProof { + recursive_proofs.push(RecursiveProof { opened_rows: stored_rows_i, merkle_proof: merkle_proof_i, }); let sks_vks_i = eval_sk_at_vks(n_next); let _t = std::time::Instant::now(); - let (basis_i_induced, enforced_sum_i) = induce_sumcheck_poly( - n_next, - &sks_vks_i, - &opened_rows_i, - &level_rs, - &queries_i, - &alpha_i, - ); + let (basis_i_induced, enforced_sum_i) = + induce_sumcheck_poly_ext(n_next, &sks_vks_i, &opened_rows_i, &level_rs, &queries_i, &alpha_i); if trace { t_induce += _t.elapsed(); } let _t = std::time::Instant::now(); let intro_msg_i = sc_prover.introduce_new(basis_i_induced, enforced_sum_i); - add_sumcheck_msg(ps, &intro_msg_i); - let beta_i = ps.sample(); + observe_ext(sponge, intro_msg_i.u_0); + observe_ext(sponge, intro_msg_i.u_2); + let beta_i = sample_ext(sponge); sc_prover.glue(beta_i); if trace { t_intro_glue += _t.elapsed(); @@ -2924,49 +2667,642 @@ fn multilevel_prover_with_basis_impl( unreachable!() } -/// Succinct verifier for [`multilevel_prover_with_basis`]: instead of accepting -/// a dense `b_initial: &[F128]` (which would be ~16 MB at m=29), accepts a -/// **closure** `eval_b` that evaluates `b_initial(point)` at any multilinear -/// point. The verifier calls `eval_b` only `yr.len()` times (at the residual) -/// — typically a few dozen times, not 2^L. Use this from -/// `pcs::verify_opening_batch_mixed_ligerito_stacked`, where the closure is -/// built from the `ring_switch::verify_bind` outputs + stacked claim points. -/// -/// `log_n` is the original packed-witness log size (= b_initial's logical dim). -#[allow(clippy::too_many_arguments)] -pub fn multilevel_verifier_with_basis_succinct( - config: &LigeritoConfig, +// =================================================================== +// Dense verifier +// =================================================================== + +/// Verify all opened L0 (F64) rows against one root via a single multi-proof. +fn verify_level_opens_base( + root: &Hash, + block_len: usize, + queries: &[usize], + opened_rows: &[Vec], + expected_num_interleaved: usize, + multi_proof: &[Hash], +) -> bool { + if queries.len() != opened_rows.len() { + return false; + } + let mut leaf_hashes: Vec = Vec::with_capacity(opened_rows.len()); + for row in opened_rows { + if row.len() != expected_num_interleaved { + return false; + } + // SAFETY: F64 is repr(transparent) over u64 (8 bytes, no padding); + // the row's byte image is exactly `row.len() * 8` initialized bytes. + let bytes: &[u8] = + unsafe { core::slice::from_raw_parts(row.as_ptr() as *const u8, row.len() * core::mem::size_of::()) }; + leaf_hashes.push(merkle::hash_leaf(bytes)); + } + merkle::verify_merkle_multi_proof(root, block_len, queries, &leaf_hashes, multi_proof) +} + +/// Verify all opened deeper-level (F192) rows against one root. +fn verify_level_opens_ext( + root: &Hash, + block_len: usize, + queries: &[usize], + opened_rows: &[Vec], + expected_num_interleaved: usize, + multi_proof: &[Hash], +) -> bool { + if queries.len() != opened_rows.len() { + return false; + } + let mut leaf_hashes: Vec = Vec::with_capacity(opened_rows.len()); + for row in opened_rows { + if row.len() != expected_num_interleaved { + return false; + } + // SAFETY: F192 is repr(C) with three u64 limbs (24 bytes, no + // padding); the row's byte image is exactly `row.len() * 24` + // initialized bytes. + let bytes: &[u8] = + unsafe { core::slice::from_raw_parts(row.as_ptr() as *const u8, row.len() * core::mem::size_of::()) }; + leaf_hashes.push(merkle::hash_leaf(bytes)); + } + merkle::verify_merkle_multi_proof(root, block_len, queries, &leaf_hashes, multi_proof) +} + +/// Transcript-order queries with duplicates removed, ascending. The initial opening +/// stores one opened row per distinct query position (sorted); the recursion +/// harness expands back to per-query order below. +fn sorted_unique_queries(queries: &[usize]) -> Vec { + let mut s = queries.to_vec(); + s.sort_unstable(); + s.dedup(); + s +} + +/// Expand a base-level (`F64`, level 0) [`InitialProof`] into the flat per-query +/// form the recursion guest re-hashes: one row and one full Merkle path per +/// query, in transcript order (duplicates included). Mirror of +/// `expand_level_opening` for the K stacked opening's `F64` +/// leaf level. Authenticates nothing itself; the caller re-checks each restored +/// path against the root. +pub fn expand_level_opening_base( + block_len: usize, + queries: &[usize], + rows_sorted: &[Vec], + expected_num_interleaved: usize, + multi_proof: &[Hash], +) -> Option<(Vec>, Vec)> { + let sorted = sorted_unique_queries(queries); + if sorted.len() != rows_sorted.len() { + return None; + } + let mut leaf_hashes = Vec::with_capacity(rows_sorted.len()); + for row in rows_sorted { + if row.len() != expected_num_interleaved { + return None; + } + // SAFETY: F64 is repr(transparent) over u64 (8 bytes, no padding). + let bytes: &[u8] = + unsafe { core::slice::from_raw_parts(row.as_ptr() as *const u8, row.len() * core::mem::size_of::()) }; + leaf_hashes.push(merkle::hash_leaf(bytes)); + } + let flat_paths = merkle::restore_multi_proof(block_len, queries, &leaf_hashes, multi_proof)?; + let mut rows_ordered = Vec::with_capacity(queries.len()); + for &q in queries { + let slot = sorted.binary_search(&q).ok()?; + rows_ordered.push(rows_sorted[slot].clone()); + } + Some((rows_ordered, flat_paths)) +} + +/// Extension-level (`F192`, levels ≥ 1) counterpart of +/// [`expand_level_opening_base`], hashing 24-byte `F192` leaf rows. +pub fn expand_level_opening_ext( + block_len: usize, + queries: &[usize], + rows_sorted: &[Vec], + expected_num_interleaved: usize, + multi_proof: &[Hash], +) -> Option<(Vec>, Vec)> { + let sorted = sorted_unique_queries(queries); + if sorted.len() != rows_sorted.len() { + return None; + } + let mut leaf_hashes = Vec::with_capacity(rows_sorted.len()); + for row in rows_sorted { + if row.len() != expected_num_interleaved { + return None; + } + // SAFETY: F192 is repr(C) with three u64 limbs (24 bytes, no padding). + let bytes: &[u8] = + unsafe { core::slice::from_raw_parts(row.as_ptr() as *const u8, row.len() * core::mem::size_of::()) }; + leaf_hashes.push(merkle::hash_leaf(bytes)); + } + let flat_paths = merkle::restore_multi_proof(block_len, queries, &leaf_hashes, multi_proof)?; + let mut rows_ordered = Vec::with_capacity(queries.len()); + for &q in queries { + let slot = sorted.binary_search(&q).ok()?; + rows_ordered.push(rows_sorted[slot].clone()); + } + Some((rows_ordered, flat_paths)) +} + +/// Dense verifier for [`recursive_prover_with_basis`] (mirror of +/// `ligerito::recursive_verifier_with_basis`): materializes `b_initial` and +/// every induced basis poly, replays the transcript, and checks the residual +/// inner product against the running sum-claim. Production callers should +/// prefer [`recursive_verifier_with_basis_succinct`]; this one exists for +/// correctness testing (dense/succinct agreement) and benchmarking. +#[cfg(test)] +pub fn recursive_verifier_with_basis( + config: &VerifierConfig, + proof: &LigeritoProof, + b_initial: &[F192], + target: F192, + expected_initial_root: &Hash, + sponge: &mut Sponge, +) -> bool { + let log_n = b_initial.len().trailing_zeros() as usize; + let initial_k = config.initial_k; + let r = config.level_steps; + + if r < 1 || config.level_ks.len() != r || config.log_inv_rates.len() != r + 1 { + return false; + } + if b_initial.len() != 1usize << log_n { + return false; + } + if config.ood_samples.first().copied().unwrap_or(0) != 0 { + return false; + } + + // The L0 root is the caller's statement (not proof data): absorb it in the + // prover's slot and check L0 opens against it below. + // (No opener domain-label absorb: the extension-field opener has none and the recursion + // guest replays a label-free opening transcript; the observed `target` + + // outer transcript context provide domain separation.) + observe_ext(sponge, target); + observe_root(sponge, expected_initial_root); + + let log_inv_rate_0 = config.log_inv_rates[0]; + let log_msg_cols_0 = log_n - initial_k; + let block_len_0 = 1usize << (log_msg_cols_0 + log_inv_rate_0); + let num_interleaved_0 = 1usize << initial_k; + + // Replay sumcheck: start msg, then initial_k folds. + let mut t_r = target; + let mut tx_idx = 0usize; + if tx_idx >= proof.sumcheck_transcript.len() { + return false; + } + let start_msg = proof.sumcheck_transcript[tx_idx]; + tx_idx += 1; + observe_ext(sponge, start_msg.u_0); + observe_ext(sponge, start_msg.u_2); + let mut running_quad = RoundQuad::from_msg(start_msg, t_r); + + let fold_bits = |lvl: usize| -> u32 { config.fold_grinding_bits.get(lvl).copied().unwrap_or(0) as u32 }; + let ood_count = |lvl: usize| -> usize { config.ood_samples.get(lvl).copied().unwrap_or(0) }; + let mut fold_nonce_idx = 0usize; + let mut ood_idx = 0usize; + let mut ood_bases: Vec<(Vec, usize, F192)> = Vec::new(); + + let mut r_lane_fold = Vec::with_capacity(initial_k); + for j in 0..initial_k { + // Fold-challenge PoW mirror (L0's lane folds), tapered per round. + let bits = fold_bits(0).saturating_sub(j as u32); + if bits > 0 { + if fold_nonce_idx >= proof.fold_grinding_nonces.len() { + return false; + } + if !sponge.verify_pow(proof.fold_grinding_nonces[fold_nonce_idx], bits) { + return false; + } + fold_nonce_idx += 1; + } + let ri = sample_ext(sponge); + r_lane_fold.push(ri); + t_r = running_quad.eval(ri); + if tx_idx >= proof.sumcheck_transcript.len() { + return false; + } + let msg = proof.sumcheck_transcript[tx_idx]; + tx_idx += 1; + observe_ext(sponge, msg.u_0); + observe_ext(sponge, msg.u_2); + running_quad = RoundQuad::from_msg(msg, t_r); + } + + // Observe wtns_1 root + open wtns_0. + if proof.recursive_roots.is_empty() { + return false; + } + let root_1 = proof.recursive_roots[0]; + observe_root(sponge, &root_1); + + for _ in 0..ood_count(1) { + let z = sample_ext_vec(sponge, log_n - initial_k); + let Some(&y) = proof.ood_values.get(ood_idx) else { + return false; + }; + ood_idx += 1; + observe_ext(sponge, y); + let Some(&intro_msg) = proof.sumcheck_transcript.get(tx_idx) else { + return false; + }; + tx_idx += 1; + observe_ext(sponge, intro_msg.u_0); + observe_ext(sponge, intro_msg.u_2); + let intro_quad = RoundQuad::from_msg(intro_msg, y); + let beta = sample_ext(sponge); + running_quad = RoundQuad::fold(&running_quad, &intro_quad, beta); + t_r += beta * y; + ood_bases.push((build_eq_table_ext(&z), initial_k, beta)); + } + + // PoW grinding check for L0's query phase (no-op at 0 bits but keeps the + // FS state in lockstep with the prover). + let mut nonce_idx = 0usize; + if nonce_idx >= proof.grinding_nonces.len() { + return false; + } + if !sponge.verify_pow(proof.grinding_nonces[nonce_idx], config.grinding_bits[0] as u32) { + return false; + } + nonce_idx += 1; + + let num_queries_0 = config.queries[0]; + let queries_0 = sample_queries_ordered(sponge, block_len_0, num_queries_0); + let alpha_0 = sample_ext_vec(sponge, log2_ceil(num_queries_0)); + let sq_0 = sorted_unique_queries(&queries_0); + if !verify_level_opens_base( + expected_initial_root, + block_len_0, + &sq_0, + &proof.initial_proof.opened_rows, + num_interleaved_0, + &proof.initial_proof.merkle_proof, + ) { + return false; + } + // Fan the authenticated sorted-unique rows back to transcript order for induce. + let ordered_rows_0 = match fan_rows_to_ordered(&queries_0, &proof.initial_proof.opened_rows) { + Some(x) => x, + None => return false, + }; + + // L0 induce with the same auto dispatch as the prover (dense vs sparse + // transposed-NTT; identical outputs either way). + let n1 = log_n - initial_k; + let sks_vks_n1 = eval_sk_at_vks(n1); + let (basis_0_induced, enforced_sum_0) = induce_sumcheck_poly_auto_base( + n1, + log_inv_rate_0, + &sks_vks_n1, + &ordered_rows_0, + &r_lane_fold, + &queries_0, + &alpha_0, + ); + + // Intro + glue. + if tx_idx >= proof.sumcheck_transcript.len() { + return false; + } + let intro_msg_0 = proof.sumcheck_transcript[tx_idx]; + tx_idx += 1; + observe_ext(sponge, intro_msg_0.u_0); + observe_ext(sponge, intro_msg_0.u_2); + let intro_quad_0 = RoundQuad::from_msg(intro_msg_0, enforced_sum_0); + let beta_0 = sample_ext(sponge); + running_quad = RoundQuad::fold(&running_quad, &intro_quad_0, beta_0); + t_r += beta_0 * enforced_sum_0; + + // Basis poly tracking for the residual check. b_initial folds at ALL ris; + // basis_0_induced starts after the lane folds. + let mut basis_polys: Vec> = vec![b_initial.to_vec(), basis_0_induced]; + let mut basis_ris_starts: Vec = vec![0, initial_k]; + let mut basis_separations: Vec = vec![beta_0]; + let mut ris: Vec = r_lane_fold.clone(); + + let mut prev_root = root_1; + let mut prev_log_num_interleaved = config.level_ks[0]; + let mut prev_log_msg_cols = n1 - prev_log_num_interleaved; + let mut prev_log_inv_rate = config.log_inv_rates[1]; + let mut next_root_idx = 1usize; + let mut recursive_proof_idx = 0usize; + let mut n_current = n1; + + for i in 0..r { + let k_i = config.level_ks[i]; + if n_current < k_i { + return false; + } + let mut level_rs = Vec::with_capacity(k_i); + for j in 0..k_i { + // Fold-challenge PoW mirror (level i+1's folds), tapered. + let bits = fold_bits(i + 1).saturating_sub(j as u32); + if bits > 0 { + if fold_nonce_idx >= proof.fold_grinding_nonces.len() { + return false; + } + if !sponge.verify_pow(proof.fold_grinding_nonces[fold_nonce_idx], bits) { + return false; + } + fold_nonce_idx += 1; + } + let ri = sample_ext(sponge); + ris.push(ri); + level_rs.push(ri); + t_r = running_quad.eval(ri); + if tx_idx >= proof.sumcheck_transcript.len() { + return false; + } + let msg = proof.sumcheck_transcript[tx_idx]; + tx_idx += 1; + observe_ext(sponge, msg.u_0); + observe_ext(sponge, msg.u_2); + running_quad = RoundQuad::from_msg(msg, t_r); + } + n_current -= k_i; + + if i == r - 1 { + if ood_idx != proof.ood_values.len() || fold_nonce_idx != proof.fold_grinding_nonces.len() { + return false; + } + let yr = &proof.final_proof.yr; + if yr.len() != 1 << n_current { + return false; + } + for v in yr { + observe_ext(sponge, *v); + } + // PoW grinding check for the last level. + if nonce_idx >= proof.grinding_nonces.len() { + return false; + } + if !sponge.verify_pow(proof.grinding_nonces[nonce_idx], config.grinding_bits[i + 1] as u32) { + return false; + } + // (last nonce: nonce_idx is not advanced past it) + + let prev_block_len = 1usize << (prev_log_msg_cols + prev_log_inv_rate); + let prev_num_interleaved = 1usize << prev_log_num_interleaved; + let num_queries_last = config.queries[i + 1]; + let queries_last = sample_queries_ordered(sponge, prev_block_len, num_queries_last); + // Final-level basis-induction challenge: sampled AFTER `yr` was + // observed and the queries are fixed, so a forged `yr` cannot be + // adapted to it (mirror of the original). + let alpha_last = sample_ext_vec(sponge, log2_ceil(num_queries_last)); + let sq_last = sorted_unique_queries(&queries_last); + if !verify_level_opens_ext( + &prev_root, + prev_block_len, + &sq_last, + &proof.final_proof.opened_rows, + prev_num_interleaved, + &proof.final_proof.merkle_proof, + ) { + return false; + } + let ordered_rows_last = match fan_rows_to_ordered(&queries_last, &proof.final_proof.opened_rows) { + Some(x) => x, + None => return false, + }; + + // Bind the LAST commitment to `yr`: induce its opened rows into + // the sumcheck like every non-final level, batched with a fresh + // `beta_last` (see the original's binding-fix comment). + let sks_vks_last = eval_sk_at_vks(n_current); + let (basis_last_induced, enforced_sum_last) = induce_sumcheck_poly_ext( + n_current, + &sks_vks_last, + &ordered_rows_last, + &level_rs, + &queries_last, + &alpha_last, + ); + let Some(&intro_msg_last) = proof.sumcheck_transcript.get(tx_idx) else { + return false; + }; + tx_idx += 1; + observe_ext(sponge, intro_msg_last.u_0); + observe_ext(sponge, intro_msg_last.u_2); + let intro_quad_last = RoundQuad::from_msg(intro_msg_last, enforced_sum_last); + let beta_last = sample_ext(sponge); + running_quad = RoundQuad::fold(&running_quad, &intro_quad_last, beta_last); + t_r += beta_last * enforced_sum_last; + basis_polys.push(basis_last_induced); + basis_ris_starts.push(ris.len()); + basis_separations.push(beta_last); + + // Finish the residual sumcheck rounds, then evaluate every dense + // basis and the transmitted final message at the one terminal point. + let mut ris_tail = Vec::with_capacity(n_current); + for j in 0..n_current { + let ri = sample_ext(sponge); + t_r = running_quad.eval(ri); + ris_tail.push(ri); + if j + 1 < n_current { + let Some(&msg) = proof.sumcheck_transcript.get(tx_idx) else { + return false; + }; + tx_idx += 1; + observe_ext(sponge, msg.u_0); + observe_ext(sponge, msg.u_2); + running_quad = RoundQuad::from_msg(msg, t_r); + } + } + if tx_idx != proof.sumcheck_transcript.len() { + return false; + } + ris.extend_from_slice(&ris_tail); + let mut weight = F192::ZERO; + for (k, basis) in basis_polys.iter().enumerate() { + let start = basis_ris_starts[k]; + let at = partial_eval_lsb_ext(basis, &ris[start..]); + if at.len() != 1 { + return false; + } + let sep = if k == 0 { F192::ONE } else { basis_separations[k - 1] }; + weight += sep * at[0]; + } + for (basis, start, beta) in &ood_bases { + let at = partial_eval_lsb_ext(basis, &ris[*start..]); + if at.len() != 1 { + return false; + } + weight += *beta * at[0]; + } + return weight * mle_eval_ext(yr, &ris_tail) == t_r; + } + + if next_root_idx >= proof.recursive_roots.len() { + return false; + } + let root_next = proof.recursive_roots[next_root_idx]; + next_root_idx += 1; + observe_root(sponge, &root_next); + + for _ in 0..ood_count(i + 2) { + let z = sample_ext_vec(sponge, n_current); + let Some(&y) = proof.ood_values.get(ood_idx) else { + return false; + }; + ood_idx += 1; + observe_ext(sponge, y); + let Some(&intro_msg) = proof.sumcheck_transcript.get(tx_idx) else { + return false; + }; + tx_idx += 1; + observe_ext(sponge, intro_msg.u_0); + observe_ext(sponge, intro_msg.u_2); + let intro_quad = RoundQuad::from_msg(intro_msg, y); + let beta = sample_ext(sponge); + running_quad = RoundQuad::fold(&running_quad, &intro_quad, beta); + t_r += beta * y; + ood_bases.push((build_eq_table_ext(&z), ris.len(), beta)); + } + + // PoW grinding check for this iteration's query phase. + if nonce_idx >= proof.grinding_nonces.len() { + return false; + } + if !sponge.verify_pow(proof.grinding_nonces[nonce_idx], config.grinding_bits[i + 1] as u32) { + return false; + } + nonce_idx += 1; + + let prev_block_len = 1usize << (prev_log_msg_cols + prev_log_inv_rate); + let prev_num_interleaved = 1usize << prev_log_num_interleaved; + let num_queries_i = config.queries[i + 1]; + let queries_i = sample_queries_ordered(sponge, prev_block_len, num_queries_i); + let sq_i = sorted_unique_queries(&queries_i); + let alpha_i = sample_ext_vec(sponge, log2_ceil(num_queries_i)); + if recursive_proof_idx >= proof.recursive_proofs.len() { + return false; + } + let rp = &proof.recursive_proofs[recursive_proof_idx]; + recursive_proof_idx += 1; + if !verify_level_opens_ext( + &prev_root, + prev_block_len, + &sq_i, + &rp.opened_rows, + prev_num_interleaved, + &rp.merkle_proof, + ) { + return false; + } + let ordered_rows_i = match fan_rows_to_ordered(&queries_i, &rp.opened_rows) { + Some(x) => x, + None => return false, + }; + + let sks_vks_i = eval_sk_at_vks(n_current); + let (basis_i_induced, enforced_sum_i) = + induce_sumcheck_poly_ext(n_current, &sks_vks_i, &ordered_rows_i, &level_rs, &queries_i, &alpha_i); + + if tx_idx >= proof.sumcheck_transcript.len() { + return false; + } + let intro_msg_i = proof.sumcheck_transcript[tx_idx]; + tx_idx += 1; + observe_ext(sponge, intro_msg_i.u_0); + observe_ext(sponge, intro_msg_i.u_2); + let intro_quad_i = RoundQuad::from_msg(intro_msg_i, enforced_sum_i); + let beta_i = sample_ext(sponge); + running_quad = RoundQuad::fold(&running_quad, &intro_quad_i, beta_i); + t_r += beta_i * enforced_sum_i; + basis_polys.push(basis_i_induced); + basis_ris_starts.push(ris.len()); + basis_separations.push(beta_i); + + prev_root = root_next; + let k_next = config.level_ks[i + 1]; + if n_current < k_next { + return false; + } + prev_log_num_interleaved = k_next; + prev_log_msg_cols = n_current - k_next; + prev_log_inv_rate = config.log_inv_rates[i + 2]; + } + + unreachable!() +} + +// =================================================================== +// Succinct verifier +// =================================================================== + +/// Succinct verifier for [`recursive_prover_with_basis`] (mirror of +/// `ligerito::recursive_verifier_with_basis_succinct`): instead of a dense +/// `b_initial` (2^log_n E-values) it takes a closure `eval_b_at` that evaluates +/// b's multilinear extension once, at the final fold point. +/// +/// Per-level induced bases are never materialized: intro time uses the cheap +/// enforced-sum recomputation, and the residual uses the closed-form +/// `induce_sumcheck_evaluate_at_residual`. `log_n` is the committed +/// K-witness log size (b's logical dimension). Transcript replay is +/// byte-identical to the dense verifier (OOD elided; config must take zero +/// OOD samples, as asserted by the prover). +/// Thin wrapper of [`recursive_verifier_with_basis_succinct_with_squeezes`] +/// that discards the query squeezes — the signature every non-recursion caller +/// uses. +pub fn recursive_verifier_with_basis_succinct( + config: &VerifierConfig, + proof: &LigeritoProof, + log_n: usize, + target: F192, + expected_initial_root: &Hash, + eval_b_at: F, + sponge: &mut Sponge, +) -> bool +where + F: Fn(&[F192]) -> F192, +{ + let mut discard = Vec::new(); + recursive_verifier_with_basis_succinct_with_squeezes( + config, + proof, + log_n, + target, + expected_initial_root, + eval_b_at, + sponge, + &mut discard, + ) +} + +/// As [`recursive_verifier_with_basis_succinct`], but on accept fills +/// `query_squeezes_out` with the raw query-sampling squeezes per level in +/// transcript order (the recursion harness reads `.c0/.c1` off them to re-derive +/// query positions). Left partially filled on reject; use it only on `true`. +pub fn recursive_verifier_with_basis_succinct_with_squeezes( + config: &VerifierConfig, proof: &LigeritoProof, log_n: usize, - target: F128, + target: F192, expected_initial_root: &Hash, eval_b_at: F, - vs: &mut VerifierState<'_>, -) -> Option + sponge: &mut Sponge, + query_squeezes_out: &mut Vec>, +) -> bool where - // Called ONCE, at the very end, with the full fold point of length `log_n`. - // The sumcheck now runs to completion, so a single evaluation closes the - // opening; a weight that does not factor across the residual split (a - // jagged interval indicator, say) therefore costs one evaluation, not - // `2^yr_log_n` of them. - F: Fn(&[F128]) -> F128, + // Called once at the terminal check with the full fold point. + F: Fn(&[F192]) -> F192, { - let trace = std::env::var("LIG_VERIFY_TRACE").is_ok(); - let mut t_merkle = std::time::Duration::ZERO; - let mut t_sample_q = std::time::Duration::ZERO; - let mut t_enforced = std::time::Duration::ZERO; - let mut t_residual = std::time::Duration::ZERO; - let mut t_evalb = std::time::Duration::ZERO; - let _t_start = std::time::Instant::now(); - - let mut query_squeezes: Vec> = Vec::new(); let initial_k = config.initial_k; let r = config.level_steps; if r < 1 || config.level_ks.len() != r || config.log_inv_rates.len() != r + 1 { - return None; + return false; + } + if config.ood_samples.first().copied().unwrap_or(0) != 0 { + return false; } - vs.observe_scalar(target); - vs.absorb_bytes(expected_initial_root); + + // The L0 root is the caller's statement (not proof data): absorb it + // exactly where the prover absorbed its own. + // (No opener domain-label absorb: the extension-field opener has none and the recursion + // guest replays a label-free opening transcript; the observed `target` + + // outer transcript context provide domain separation.) + observe_ext(sponge, target); + observe_root(sponge, expected_initial_root); let log_inv_rate_0 = config.log_inv_rates[0]; let log_msg_cols_0 = log_n - initial_k; @@ -2974,50 +3310,74 @@ where let num_interleaved_0 = 1usize << initial_k; let mut t_r = target; - let start_msg = next_sumcheck_msg(vs)?; + let mut tx_idx = 0usize; + if tx_idx >= proof.sumcheck_transcript.len() { + return false; + } + let start_msg = proof.sumcheck_transcript[tx_idx]; + tx_idx += 1; + observe_ext(sponge, start_msg.u_0); + observe_ext(sponge, start_msg.u_2); let mut running_quad = RoundQuad::from_msg(start_msg, t_r); - let fold_bits = - |lvl: usize| -> u32 { config.fold_grinding_bits.get(lvl).copied().unwrap_or(0) as u32 }; + let fold_bits = |lvl: usize| -> u32 { config.fold_grinding_bits.get(lvl).copied().unwrap_or(0) as u32 }; let ood_count = |lvl: usize| -> usize { config.ood_samples.get(lvl).copied().unwrap_or(0) }; - if config.ood_samples.first().copied().unwrap_or(0) != 0 { - return None; // L0 must be bound by the opening's own eval claim - } - // OOD claims glued into the running sumcheck: each contributes - // `beta · Π_b eq(z_b, r_b) · eq(z_tail, ·)` at the residual. + let mut fold_nonce_idx = 0usize; + let mut ood_idx = 0usize; struct OodCtx { - z: Vec, + z: Vec, ris_start: usize, - beta: F128, + beta: F192, } let mut ood_ctxs: Vec = Vec::new(); let mut r_lane_fold = Vec::with_capacity(initial_k); for j in 0..initial_k { - // Fold-challenge PoW mirror (L0's lane folds), tapered per round to - // (fold_bits − j) — see the prover's L0 loop. + // Fold-challenge PoW mirror (L0's lane folds), tapered per round. let bits = fold_bits(0).saturating_sub(j as u32); if bits > 0 { - vs.grind_check(bits).ok()?; + if fold_nonce_idx >= proof.fold_grinding_nonces.len() { + return false; + } + if !sponge.verify_pow(proof.fold_grinding_nonces[fold_nonce_idx], bits) { + return false; + } + fold_nonce_idx += 1; } - let ri = vs.sample(); + let ri = sample_ext(sponge); r_lane_fold.push(ri); t_r = running_quad.eval(ri); - let msg = next_sumcheck_msg(vs)?; + if tx_idx >= proof.sumcheck_transcript.len() { + return false; + } + let msg = proof.sumcheck_transcript[tx_idx]; + tx_idx += 1; + observe_ext(sponge, msg.u_0); + observe_ext(sponge, msg.u_2); running_quad = RoundQuad::from_msg(msg, t_r); } - let root_1 = next_root(vs)?; + if proof.recursive_roots.is_empty() { + return false; + } + let root_1 = proof.recursive_roots[0]; + observe_root(sponge, &root_1); - // OOD binding mirror for the L1 commit: sample z, read the claimed - // evaluation off the stream, and glue the claim into the running - // sumcheck exactly like the prover. for _ in 0..ood_count(1) { - let z = vs.sample_vec(log_n - initial_k); - let y = vs.next_scalar().ok()?; - let intro_msg = next_sumcheck_msg(vs)?; + let z = sample_ext_vec(sponge, log_n - initial_k); + let Some(&y) = proof.ood_values.get(ood_idx) else { + return false; + }; + ood_idx += 1; + observe_ext(sponge, y); + let Some(&intro_msg) = proof.sumcheck_transcript.get(tx_idx) else { + return false; + }; + tx_idx += 1; + observe_ext(sponge, intro_msg.u_0); + observe_ext(sponge, intro_msg.u_2); let intro_quad = RoundQuad::from_msg(intro_msg, y); - let beta = vs.sample(); + let beta = sample_ext(sponge); running_quad = RoundQuad::fold(&running_quad, &intro_quad, beta); t_r += beta * y; ood_ctxs.push(OodCtx { @@ -3027,72 +3387,60 @@ where }); } - // PoW grinding check for L0's query phase. With grinding_bits[0]=0 this - // is a no-op (still absorbs the 0 nonce so the FS state matches the - // prover side). - vs.grind_check(config.grinding_bits[0] as u32).ok()?; + // PoW grinding check for L0's query phase. + let mut nonce_idx = 0usize; + if nonce_idx >= proof.grinding_nonces.len() { + return false; + } + if !sponge.verify_pow(proof.grinding_nonces[nonce_idx], config.grinding_bits[0] as u32) { + return false; + } + nonce_idx += 1; let num_queries_0 = config.queries[0]; - let _t = std::time::Instant::now(); - let (queries_0, raw_0) = sample_queries_ordered_with_raw(vs.sponge_mut(), block_len_0, num_queries_0); - query_squeezes.push(raw_0); - if trace { - t_sample_q += _t.elapsed(); - } - let alpha_0 = vs.sample_vec(log2_ceil(num_queries_0)); - let _t = std::time::Instant::now(); - // Expand the stored (compressed) opening into the flat per-query form: one - // row + one full Merkle path per query in transcript order, then verify each - // path independently. The expansion self-authenticates via these root checks. - let (opened_rows_0, merkle_paths_0) = expand_level_opening( + let (queries_0, raw_0) = sample_queries_ordered_with_raw(sponge, block_len_0, num_queries_0); + query_squeezes_out.push(raw_0); + let alpha_0 = sample_ext_vec(sponge, log2_ceil(num_queries_0)); + let sq_0 = sorted_unique_queries(&queries_0); + if !verify_level_opens_base( + expected_initial_root, block_len_0, - &queries_0, + &sq_0, &proof.initial_proof.opened_rows, num_interleaved_0, &proof.initial_proof.merkle_proof, - )?; - if !verify_level_opens_perquery( - expected_initial_root, - block_len_0, - &queries_0, - &opened_rows_0, - num_interleaved_0, - &merkle_paths_0, ) { - return None; - } - if trace { - t_merkle += _t.elapsed(); + return false; } + let ordered_rows_0 = match fan_rows_to_ordered(&queries_0, &proof.initial_proof.opened_rows) { + Some(x) => x, + None => return false, + }; // Compute enforced_sum cheaply at intro time. The induced basis poly's - // residual evaluations are deferred to the final check (succinct path — - // see `induce_sumcheck_evaluate_at_residual`). + // residual evaluations are deferred to the final closed-form check. let n1 = log_n - initial_k; - let _t = std::time::Instant::now(); - let enforced_sum_0 = induce_sumcheck_enforced_sum( - &opened_rows_0, - &r_lane_fold, - &queries_0, - &alpha_0, - ); - if trace { - t_enforced += _t.elapsed(); - } + let enforced_sum_0 = induce_sumcheck_enforced_sum_base(&ordered_rows_0, &r_lane_fold, &queries_0, &alpha_0); - let intro_msg_0 = next_sumcheck_msg(vs)?; + if tx_idx >= proof.sumcheck_transcript.len() { + return false; + } + let intro_msg_0 = proof.sumcheck_transcript[tx_idx]; + tx_idx += 1; + observe_ext(sponge, intro_msg_0.u_0); + observe_ext(sponge, intro_msg_0.u_2); let intro_quad_0 = RoundQuad::from_msg(intro_msg_0, enforced_sum_0); - let beta_0 = vs.sample(); + let beta_0 = sample_ext(sponge); running_quad = RoundQuad::fold(&running_quad, &intro_quad_0, beta_0); t_r += beta_0 * enforced_sum_0; - // Per-level induced-basis evaluation context — small (no dense vec). + // Per-level induced-basis evaluation context: small (no dense vec). struct LevelCtx { log_msg_cols: usize, queries: Vec, - alpha: Vec, // ⌈log₂ Q⌉ field elements (eq-tensor combination) + alpha: Vec, // ceil(log2 Q) elements (eq-tensor combination) ris_start: usize, - beta: F128, + beta: F192, } let mut level_ctxs: Vec = vec![LevelCtx { log_msg_cols: n1, @@ -3101,101 +3449,104 @@ where ris_start: initial_k, beta: beta_0, }]; - let mut ris: Vec = r_lane_fold.clone(); + let mut ris: Vec = r_lane_fold.clone(); let mut prev_root = root_1; let mut prev_log_num_interleaved = config.level_ks[0]; let mut prev_log_msg_cols = n1 - prev_log_num_interleaved; let mut prev_log_inv_rate = config.log_inv_rates[1]; - let mut level_proof_idx = 0usize; + let mut next_root_idx = 1usize; + let mut recursive_proof_idx = 0usize; let mut n_current = n1; for i in 0..r { let k_i = config.level_ks[i]; if n_current < k_i { - return None; + return false; } let mut level_rs = Vec::with_capacity(k_i); for j in 0..k_i { - // Fold-challenge PoW mirror (level i+1's folds), tapered per round - // to (fold_bits − j) — see the prover's L0 loop. + // Fold-challenge PoW mirror (level i+1's folds), tapered. let bits = fold_bits(i + 1).saturating_sub(j as u32); if bits > 0 { - vs.grind_check(bits).ok()?; + if fold_nonce_idx >= proof.fold_grinding_nonces.len() { + return false; + } + if !sponge.verify_pow(proof.fold_grinding_nonces[fold_nonce_idx], bits) { + return false; + } + fold_nonce_idx += 1; } - let ri = vs.sample(); + let ri = sample_ext(sponge); ris.push(ri); level_rs.push(ri); t_r = running_quad.eval(ri); - let msg = next_sumcheck_msg(vs)?; + if tx_idx >= proof.sumcheck_transcript.len() { + return false; + } + let msg = proof.sumcheck_transcript[tx_idx]; + tx_idx += 1; + observe_ext(sponge, msg.u_0); + observe_ext(sponge, msg.u_2); running_quad = RoundQuad::from_msg(msg, t_r); } n_current -= k_i; if i == r - 1 { - let yr = vs.next_scalars(1 << n_current).ok()?; - // PoW grinding check for last level's query phase. - vs.grind_check(config.grinding_bits[i + 1] as u32).ok()?; + if ood_idx != proof.ood_values.len() || fold_nonce_idx != proof.fold_grinding_nonces.len() { + return false; + } + let yr = &proof.final_proof.yr; + if yr.len() != 1 << n_current { + return false; + } + for v in yr { + observe_ext(sponge, *v); + } + // PoW grinding check for the last level's query phase. + if nonce_idx >= proof.grinding_nonces.len() { + return false; + } + if !sponge.verify_pow(proof.grinding_nonces[nonce_idx], config.grinding_bits[i + 1] as u32) { + return false; + } + // (last nonce: nonce_idx is not advanced past it) let prev_block_len = 1usize << (prev_log_msg_cols + prev_log_inv_rate); let prev_num_interleaved = 1usize << prev_log_num_interleaved; let num_queries_last = config.queries[i + 1]; - let _t = std::time::Instant::now(); - let (queries_last, raw_last) = - sample_queries_ordered_with_raw(vs.sponge_mut(), prev_block_len, num_queries_last); - query_squeezes.push(raw_last); - // Basis-induction challenge for the LAST commitment. Sampled here — - // after `yr` was observed (top of this branch) and the queries are - // fixed — so a forged `yr` cannot be adapted to it. Mirrors `alpha_i` - // at every non-final level (see ~line 3377). - let alpha_last = vs.sample_vec(log2_ceil(num_queries_last)); - if trace { - t_sample_q += _t.elapsed(); - } - let _t = std::time::Instant::now(); - let (opened_rows_last, merkle_paths_last) = expand_level_opening( + let (queries_last, raw_last) = sample_queries_ordered_with_raw(sponge, prev_block_len, num_queries_last); + query_squeezes_out.push(raw_last); + // Basis-induction challenge for the LAST commitment, sampled after + // `yr` was observed and the queries are fixed (mirror of the + // dense verifier, so both stay in lockstep). + let alpha_last = sample_ext_vec(sponge, log2_ceil(num_queries_last)); + let sq_last = sorted_unique_queries(&queries_last); + if !verify_level_opens_ext( + &prev_root, prev_block_len, - &queries_last, + &sq_last, &proof.final_proof.opened_rows, prev_num_interleaved, &proof.final_proof.merkle_proof, - )?; - if !verify_level_opens_perquery( - &prev_root, - prev_block_len, - &queries_last, - &opened_rows_last, - prev_num_interleaved, - &merkle_paths_last, ) { - return None; - } - if trace { - t_merkle += _t.elapsed(); + return false; } + let ordered_rows_last = match fan_rows_to_ordered(&queries_last, &proof.final_proof.opened_rows) { + Some(x) => x, + None => return false, + }; - // Bind the LAST commitment to `yr`. Every non-final level folds its - // opened rows into the running sumcheck via induce_sumcheck; the - // final level used to only Merkle-check its opened rows, leaving `yr` - // (the claimed final message) constrained by a single scalar equation - // — so a malicious prover could solve for a `yr` that opens the - // commitment to an arbitrary value. We add the same proximity tie as - // the other levels: `enforced_sum_last` is the α-weighted lane-fold - // of the (Merkle-bound) opened rows, batched into `t_r` with a fresh - // `beta_last`; its induced basis is already at the residual dimension - // (zero further folds), so it joins `combined` below via this - // LevelCtx. With `alpha_last` drawn after `yr`, the batched check now - // forces `yr` to agree with the committed codeword at every queried - // column (multilinear Schwartz–Zippel), restoring binding. - let enforced_sum_last = induce_sumcheck_enforced_sum( - &opened_rows_last, - &level_rs, - &queries_last, - &alpha_last, - ); - let intro_msg_last = next_sumcheck_msg(vs)?; + let enforced_sum_last = + induce_sumcheck_enforced_sum_ext(&ordered_rows_last, &level_rs, &queries_last, &alpha_last); + let Some(&intro_msg_last) = proof.sumcheck_transcript.get(tx_idx) else { + return false; + }; + tx_idx += 1; + observe_ext(sponge, intro_msg_last.u_0); + observe_ext(sponge, intro_msg_last.u_2); let intro_quad_last = RoundQuad::from_msg(intro_msg_last, enforced_sum_last); - let beta_last = vs.sample(); + let beta_last = sample_ext(sponge); running_quad = RoundQuad::fold(&running_quad, &intro_quad_last, beta_last); t_r += beta_last * enforced_sum_last; level_ctxs.push(LevelCtx { @@ -3206,36 +3557,35 @@ where beta: beta_last, }); - // Finish the sumcheck over the residual cube. The prover has - // already folded its message and combined basis down to a single - // value each, so the check closes on ONE evaluation of the weight - // rather than `2^n_res` of them. + // Finish the sumcheck over the residual cube. Each basis and the + // caller's weight are then evaluated once at `ris ++ ris_tail`. let yr_log_n = n_current; - let mut ris_tail: Vec = Vec::with_capacity(yr_log_n); + let mut ris_tail = Vec::with_capacity(yr_log_n); for j in 0..yr_log_n { - let ri = vs.sample(); + let ri = sample_ext(sponge); t_r = running_quad.eval(ri); ris_tail.push(ri); if j + 1 < yr_log_n { - let msg = next_sumcheck_msg(vs)?; + let Some(&msg) = proof.sumcheck_transcript.get(tx_idx) else { + return false; + }; + tx_idx += 1; + observe_ext(sponge, msg.u_0); + observe_ext(sponge, msg.u_2); running_quad = RoundQuad::from_msg(msg, t_r); } } + if tx_idx != proof.sumcheck_transcript.len() { + return false; + } - // Every basis is now evaluated at the one point `ris ++ ris_tail`. - // `induce_sumcheck_evaluate_at_residual` with a residual width of 0 - // IS a point evaluation, so no new closed form is needed. - let _t = std::time::Instant::now(); - let mut weight = F128::ZERO; - for ctx in level_ctxs.iter() { - if ctx.log_msg_cols < yr_log_n - || ctx.ris_start + (ctx.log_msg_cols - yr_log_n) > ris.len() - { - return None; + let mut weight = F192::ZERO; + for ctx in &level_ctxs { + if ctx.log_msg_cols < yr_log_n || ctx.ris_start + (ctx.log_msg_cols - yr_log_n) > ris.len() { + return false; } let folded = ctx.log_msg_cols - yr_log_n; - let mut point: Vec = - ris[ctx.ris_start..ctx.ris_start + folded].to_vec(); + let mut point = ris[ctx.ris_start..ctx.ris_start + folded].to_vec(); point.extend_from_slice(&ris_tail); let at = induce_sumcheck_evaluate_at_residual( ctx.log_msg_cols, @@ -3245,50 +3595,51 @@ where &point, 0, ); + if at.len() != 1 { + return false; + } weight += ctx.beta * at[0]; } for ctx in &ood_ctxs { if ctx.z.len() < yr_log_n || ctx.ris_start + (ctx.z.len() - yr_log_n) > ris.len() { - return None; + return false; } let folded = ctx.z.len() - yr_log_n; let mut scalar = ctx.beta; for b in 0..folded { - scalar *= F128::ONE + ctx.z[b] + ris[ctx.ris_start + b]; + scalar *= F192::ONE + ctx.z[b] + ris[ctx.ris_start + b]; } weight += scalar * eq_eval(&ctx.z[folded..], &ris_tail); } - if trace { - t_residual += _t.elapsed(); - } - // The caller's weight, once, at the full point. - let mut full_point: Vec = ris.clone(); + let mut full_point = ris.clone(); full_point.extend_from_slice(&ris_tail); - let _te = std::time::Instant::now(); weight += eval_b_at(&full_point); - if trace { - t_evalb += _te.elapsed(); - } - - // `yr` was transmitted in the clear, so its folded value is a - // multilinear evaluation the verifier does itself. - let inner = weight * mle_eval(&yr, &ris_tail); - if inner != t_r { - return None; - } - return Some(LigVerifierSummary { ris, query_squeezes }); + return weight * mle_eval_ext(yr, &ris_tail) == t_r; } - let root_next = next_root(vs)?; + if next_root_idx >= proof.recursive_roots.len() { + return false; + } + let root_next = proof.recursive_roots[next_root_idx]; + next_root_idx += 1; + observe_root(sponge, &root_next); - // OOD binding mirror for the L_{i+2} commit. for _ in 0..ood_count(i + 2) { - let z = vs.sample_vec(n_current); - let y = vs.next_scalar().ok()?; - let intro_msg = next_sumcheck_msg(vs)?; + let z = sample_ext_vec(sponge, n_current); + let Some(&y) = proof.ood_values.get(ood_idx) else { + return false; + }; + ood_idx += 1; + observe_ext(sponge, y); + let Some(&intro_msg) = proof.sumcheck_transcript.get(tx_idx) else { + return false; + }; + tx_idx += 1; + observe_ext(sponge, intro_msg.u_0); + observe_ext(sponge, intro_msg.u_2); let intro_quad = RoundQuad::from_msg(intro_msg, y); - let beta = vs.sample(); + let beta = sample_ext(sponge); running_quad = RoundQuad::fold(&running_quad, &intro_quad, beta); t_r += beta * y; ood_ctxs.push(OodCtx { @@ -3299,55 +3650,52 @@ where } // PoW grinding check for this iteration's query phase. - vs.grind_check(config.grinding_bits[i + 1] as u32).ok()?; + if nonce_idx >= proof.grinding_nonces.len() { + return false; + } + if !sponge.verify_pow(proof.grinding_nonces[nonce_idx], config.grinding_bits[i + 1] as u32) { + return false; + } + nonce_idx += 1; let prev_block_len = 1usize << (prev_log_msg_cols + prev_log_inv_rate); let prev_num_interleaved = 1usize << prev_log_num_interleaved; let num_queries_i = config.queries[i + 1]; - let _t = std::time::Instant::now(); - let (queries_i, raw_i) = sample_queries_ordered_with_raw(vs.sponge_mut(), prev_block_len, num_queries_i); - query_squeezes.push(raw_i); - if trace { - t_sample_q += _t.elapsed(); - } - let alpha_i = vs.sample_vec(log2_ceil(num_queries_i)); - if level_proof_idx >= proof.level_proofs.len() { - return None; + let (queries_i, raw_i) = sample_queries_ordered_with_raw(sponge, prev_block_len, num_queries_i); + query_squeezes_out.push(raw_i); + let sq_i = sorted_unique_queries(&queries_i); + let alpha_i = sample_ext_vec(sponge, log2_ceil(num_queries_i)); + if recursive_proof_idx >= proof.recursive_proofs.len() { + return false; } - let rp = &proof.level_proofs[level_proof_idx]; - level_proof_idx += 1; - let _t = std::time::Instant::now(); - let (opened_rows_i, merkle_paths_i) = expand_level_opening( + let rp = &proof.recursive_proofs[recursive_proof_idx]; + recursive_proof_idx += 1; + if !verify_level_opens_ext( + &prev_root, prev_block_len, - &queries_i, + &sq_i, &rp.opened_rows, prev_num_interleaved, &rp.merkle_proof, - )?; - if !verify_level_opens_perquery( - &prev_root, - prev_block_len, - &queries_i, - &opened_rows_i, - prev_num_interleaved, - &merkle_paths_i, ) { - return None; - } - if trace { - t_merkle += _t.elapsed(); + return false; } + let ordered_rows_i = match fan_rows_to_ordered(&queries_i, &rp.opened_rows) { + Some(x) => x, + None => return false, + }; - let _t = std::time::Instant::now(); - let enforced_sum_i = - induce_sumcheck_enforced_sum(&opened_rows_i, &level_rs, &queries_i, &alpha_i); - if trace { - t_enforced += _t.elapsed(); - } + let enforced_sum_i = induce_sumcheck_enforced_sum_ext(&ordered_rows_i, &level_rs, &queries_i, &alpha_i); - let intro_msg_i = next_sumcheck_msg(vs)?; + if tx_idx >= proof.sumcheck_transcript.len() { + return false; + } + let intro_msg_i = proof.sumcheck_transcript[tx_idx]; + tx_idx += 1; + observe_ext(sponge, intro_msg_i.u_0); + observe_ext(sponge, intro_msg_i.u_2); let intro_quad_i = RoundQuad::from_msg(intro_msg_i, enforced_sum_i); - let beta_i = vs.sample(); + let beta_i = sample_ext(sponge); running_quad = RoundQuad::fold(&running_quad, &intro_quad_i, beta_i); t_r += beta_i * enforced_sum_i; level_ctxs.push(LevelCtx { @@ -3361,7 +3709,7 @@ where prev_root = root_next; let k_next = config.level_ks[i + 1]; if n_current < k_next { - return None; + return false; } prev_log_num_interleaved = k_next; prev_log_msg_cols = n_current - k_next; @@ -3371,925 +3719,434 @@ where unreachable!() } +// =================================================================== +// Tests +// =================================================================== + #[cfg(test)] mod tests { use super::*; + use crate::ligerito::{QUERY_GRINDING_BITS, default_config, default_verifier_config}; + + fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } - /// `validate()` rejects a config whose declared `expected_eps_pg_bits` - /// disagrees with what Theorem 1.5 predicts for the level's - /// `(eta, log_inv_rate, log_msg_cols)`. Enforces that the per-level - /// diagnostics weren't hand-waved. - #[test] - fn ligerito_security_config_rejects_paper_inconsistent_eps_pg() { - let mut cfg = blake3_m29_udr_example(); - cfg.levels[0].expected_eps_pg_bits = 50.0; // very wrong - let err = cfg.validate().unwrap_err(); - assert!( - err.contains("doesn't match") && err.contains("prediction"), - "expected paper-mismatch error, got: {err}" - ); + fn rand_ext(s: &mut u64) -> F192 { + F192::new(splitmix64(s), splitmix64(s), splitmix64(s)) } - /// Same enforcement on the query side. + #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] #[test] - fn ligerito_security_config_rejects_paper_inconsistent_eps_query() { - let mut cfg = blake3_m29_udr_example(); - // Bump query bits by 5 — far outside tolerance. - cfg.levels[0].expected_eps_query_bits += 5.0; - let err = cfg.validate().unwrap_err(); - assert!( - err.contains("doesn't match") && err.contains("prediction"), - "expected paper-mismatch error, got: {err}" - ); + fn fused_ext_butterfly_avx512_matches_scalar() { + let mut s = 0x56c8_1b92_d4a7_30efu64; + for _ in 0..100 { + let mut a: [F192; 8] = std::array::from_fn(|_| rand_ext(&mut s)); + let mut b: [F192; 8] = std::array::from_fn(|_| rand_ext(&mut s)); + let mut c: [F192; 8] = std::array::from_fn(|_| rand_ext(&mut s)); + let mut d: [F192; 8] = std::array::from_fn(|_| rand_ext(&mut s)); + let (mut want_a, mut want_b, mut want_c, mut want_d) = (a, b, c, d); + let t_outer = F64(splitmix64(&mut s)); + let t_inner_a = F64(splitmix64(&mut s)); + let t_inner_b = F64(splitmix64(&mut s)); + + for lane in 0..8 { + let new_a = want_a[lane] + want_c[lane].mul_base(t_outer); + want_c[lane] += new_a; + want_a[lane] = new_a; + let new_b = want_b[lane] + want_d[lane].mul_base(t_outer); + want_d[lane] += new_b; + want_b[lane] = new_b; + let new_a = want_a[lane] + want_b[lane].mul_base(t_inner_a); + want_b[lane] += new_a; + want_a[lane] = new_a; + let new_c = want_c[lane] + want_d[lane].mul_base(t_inner_b); + want_d[lane] += new_c; + want_c[lane] = new_c; + } + + butterfly_ext_fused_lanes(&mut a, &mut b, &mut c, &mut d, t_outer, t_inner_a, t_inner_b); + assert_eq!(a, want_a); + assert_eq!(b, want_b); + assert_eq!(c, want_c); + assert_eq!(d, want_d); + } } - /// UDR-regime m=29 example (the shipped configuration), the base config the - /// validation tests mutate. - fn blake3_m29_udr_example() -> LigeritoSecurityConfig { - LigeritoSecurityConfig::derive_config(29).expect("derive m29") + /// Configs for a K-witness of `2^log_n` elements. Prefers the strict + /// Secure-profile derivation (the production path, [`configs_for`]); + /// its ladder needs L0 block_len >= ~300 queries, i.e. log_n >= 14, so + /// smaller test sizes fall back to the ad-hoc `default_config` shape + /// (test-only; same fallback the main crate uses for small instances). + fn test_configs_for(log_n: usize) -> (ProverConfig, VerifierConfig) { + match super::configs_for(log_n) { + Ok(pv) => pv, + Err(_) => { + let pc = default_config(log_n, 5, 1).unwrap(); + let vc = default_verifier_config(log_n, 5, 1).unwrap(); + (pc, vc) + } + } } - /// Schema validates the worked example end to end. - #[test] - fn ligerito_security_config_validates() { - let cfg = blake3_m29_udr_example(); - cfg.validate() - .unwrap_or_else(|e| panic!("validate failed: {e}")); + struct Instance { + vc: VerifierConfig, + log_n: usize, + /// The eq-point behind `b_initial` (for the succinct closure). + point: Vec, + b_initial: Vec, + target: F192, + root: Hash, + proof: LigeritoProof, } - /// Lowering a level's expected_eps_query_bits below the required - /// (target − grinding) is caught by validation. - #[test] - fn ligerito_security_config_rejects_insufficient_queries() { - let mut cfg = blake3_m29_udr_example(); - cfg.levels[0].expected_eps_query_bits = 50.0; // < target 100 (grinding 0) - let err = cfg.validate().unwrap_err(); - assert!(err.contains("expected_eps_query_bits"), "err = {err}"); + fn prove_instance(log_n: usize, seed: u64) -> Instance { + let (pc, vc) = test_configs_for(log_n); + let mut s = seed; + let witness: Vec = (0..1usize << log_n).map(|_| F64(splitmix64(&mut s))).collect(); + let (cm, pd) = commit(&witness, pc.initial_k, pc.log_inv_rates[0]); + let point: Vec = (0..log_n).map(|_| rand_ext(&mut s)).collect(); + let b_initial = build_eq_table_ext(&point); + let target = inner_product_base_ext(&witness, &b_initial); + let mut ch = Sponge::new(b"ligerito-test", &[]); + let proof = recursive_prover_with_basis( + &pc, + &witness, + b_initial.clone(), + target, + &pd.codeword, + &pd.merkle_tree, + &mut ch, + ); + Instance { + vc, + log_n, + point, + b_initial, + target, + root: cm.root, + proof, + } } - /// UDR regime must not carry an `eta` value. - #[test] - fn ligerito_security_config_rejects_udr_with_eta() { - let mut cfg = blake3_m29_udr_example(); - cfg.levels[0].eta = Some(0.02); // eta is Johnson-only — should fail - let err = cfg.validate().unwrap_err(); - assert!(err.contains("udr") && err.contains("eta"), "err = {err}"); + fn verify_instance(inst: &Instance, proof: &LigeritoProof) -> bool { + let mut ch = Sponge::new(b"ligerito-test", &[]); + recursive_verifier_with_basis(&inst.vc, proof, &inst.b_initial, inst.target, &inst.root, &mut ch) } - /// UDR regime requires `proximity_loss` to be set, not `eta`. - #[test] - fn ligerito_security_config_rejects_udr_without_proximity_loss() { - let mut cfg = blake3_m29_udr_example(); - cfg.levels[0].proximity_loss = None; // missing! - let err = cfg.validate().unwrap_err(); - assert!( - err.contains("udr") && err.contains("proximity_loss"), - "err = {err}" - ); + /// Succinct verify with the eq weight evaluated at the terminal fold point. + fn verify_succinct_instance(inst: &Instance, proof: &LigeritoProof) -> bool { + let mut ch = Sponge::new(b"ligerito-test", &[]); + let point = &inst.point; + recursive_verifier_with_basis_succinct( + &inst.vc, + proof, + inst.log_n, + inst.target, + &inst.root, + |fold_point| eq_eval(point, fold_point), + &mut ch, + ) } - /// `proximity_loss` is only valid for the UDR regime. - #[test] - fn ligerito_security_config_rejects_johnson_with_proximity_loss() { - let mut cfg = blake3_m29_udr_example(); - // JohnsonOod regime with proximity_loss set — should fail. - cfg.levels[0].regime = SoundnessRegime::JohnsonOod; - cfg.levels[0].eta = Some(0.02); - cfg.levels[0].proximity_loss = Some(0.01); - let err = cfg.validate().unwrap_err(); - assert!( - err.contains("proximity_loss") && err.contains("udr"), - "err = {err}" - ); + /// Both verifiers on the same proof, asserting they agree; returns the + /// shared verdict. + fn verify_both_agree(inst: &Instance, proof: &LigeritoProof, what: &str) -> bool { + let dense = verify_instance(inst, proof); + let succinct = verify_succinct_instance(inst, proof); + assert_eq!(dense, succinct, "dense/succinct verdict split on {what}"); + dense } - /// End-to-end: a hand-built UDR-regime level validates against the - /// paper's Thm `ca-udr` bound (a = γ·n + 1) and the per-query/UDR formula. - #[test] - fn ligerito_security_config_udr_regime_validates() { - let mut cfg = blake3_m29_udr_example(); - // Convert L0 to UDR at the maximal radius γ = δ/2 − 3/(δ·n) − ε* - // (ε* = 0 → top of C.3's valid range). δ = 1 − ρ; per-query soundness - // is log₂(1/(1−γ)) and Q is sized so Q·per_q ≥ 100 bits. - let eps_star = 0.0f64; - let rho = 0.5f64; - let delta = 1.0 - rho; - let n = ((cfg.levels[0].log_msg_cols + cfg.levels[0].log_inv_rate) as f64).exp2(); - let gamma = delta / 2.0 - 3.0 / (delta * n) - eps_star; - let per_q = (1.0 / (1.0 - gamma)).log2(); - let target_bits = cfg.target_security_bits as f64; - let queries = (target_bits / per_q).ceil() as usize; - // a = γ·n + 1; ε_pg = 128 − log₂ a with NO row-union penalty in the - // unique-decoding regime (list size 1; Diamond and Gruen). Any - // shortfall below the target is covered by fold-grinding. - let log_a_base = (gamma * n + 1.0).log2(); - let eps_pg = 128.0 - log_a_base; - cfg.levels[0].regime = SoundnessRegime::Udr; - cfg.levels[0].eta = None; - cfg.levels[0].proximity_loss = Some(eps_star); - cfg.levels[0].queries = queries; - cfg.levels[0].grinding_bits = 0; - cfg.levels[0].fold_grinding_bits = (target_bits - eps_pg).ceil().max(0.0) as usize; - cfg.levels[0].expected_eps_pg_bits = (eps_pg * 10.0).round() / 10.0; - cfg.levels[0].expected_eps_query_bits = ((queries as f64 * per_q) * 10.0).round() / 10.0; - cfg.validate() - .unwrap_or_else(|e| panic!("UDR config failed to validate: {e}")); - } - - /// End-to-end sumcheck on a single basis poly: prove `Σ_x f(x)·b(x) = h`. - /// Stops one round early (yr length 2 sent in clear, à la Ligerito). - /// Verifier replays each round message, checks `q(0)+q(1)=T_r`, applies - /// the challenge, and confirms the residual inner product matches. + /// Pin the production 128-bit Johnson/OOD profile rather than the small- + /// size test fallback. #[test] - fn stateful_sumcheck_single_basis_roundtrip() { - - let n = 5; - let len = 1usize << n; - let f: Vec = (0..len) - .map(|i| { - F128::new( - (i as u64).wrapping_mul(0x1234_5678_9ABC_DEF0), - 0x55AA ^ i as u64, - ) - }) - .collect(); - let b: Vec = (0..len) - .map(|i| { - F128::new( - (i as u64).wrapping_mul(0xFEDC_BA98_7654_3210), - 0xAA55 ^ i as u64, - ) - }) - .collect(); - let h: F128 = f - .iter() - .zip(b.iter()) - .map(|(&fi, &bi)| fi * bi) - .fold(F128::ZERO, |a, v| a + v); - - // Prover: 1 start message + (n-1) folds, leaving a length-2 residual. - let (mut prover, first) = SumcheckProver::new(f.clone(), b.clone(), h); - let mut ch = crate::VerifierState::detached(&(0xC0FFEEu64).to_le_bytes(), &[]); - let mut ris: Vec = Vec::new(); - let mut msgs = vec![first]; - for _ in 0..(n - 1) { - let r = ch.sample(); - ris.push(r); - msgs.push(prover.fold(r)); - } - assert_eq!(prover.f().len(), 2); - assert_eq!(prover.combined_basis.len(), 2); - - // Verifier replay: n messages (start + n-1 folds), n-1 prover-folds challenges - // (r_0..r_{n-2}) already in ris, plus one new r_last for the final residual. - assert_eq!(msgs.len(), n); - let r_last = ch.sample(); - let mut t_r = h; - for (i, msg) in msgs.iter().enumerate() { - let quad = RoundQuad::from_msg(*msg, t_r); - assert_eq!( - quad.eval(F128::ZERO) + quad.eval(F128::ONE), - t_r, - "round {i}: q(0)+q(1) != T_r" - ); - let r_i = if i < n - 1 { ris[i] } else { r_last }; - t_r = quad.eval(r_i); - } - let one_plus_r = F128::ONE + r_last; - let f_resid = prover.f()[0] * one_plus_r + prover.f()[1] * r_last; - let b_resid = prover.combined_basis[0] * one_plus_r + prover.combined_basis[1] * r_last; - assert_eq!(f_resid * b_resid, t_r, "residual inner product != t_r"); + fn configs_johnson_profile_shape() { + let (pc, vc) = configs_for(16).expect("Johnson profile feasible at log_n = 16"); + assert_eq!(pc.initial_k, 6); + assert!(pc.level_steps >= 1); + assert_eq!(vc.initial_k, pc.initial_k); + assert_eq!(pc.ood_samples[0], 0); + assert!(pc.ood_samples.iter().skip(1).all(|&s| s >= 1)); + assert!(pc.grinding_bits.iter().all(|&b| b == QUERY_GRINDING_BITS)); + assert!(pc.fold_grinding_bits.iter().all(|&b| b == 0)); + // And log_n = 12 is below the production ladder's feasibility floor, so + // the tests there use the default_config fallback. + assert!(configs_for(12).is_err()); } - /// Multi-basis sumcheck: introduce_new + glue mid-protocol. Verifier replays. + /// The parallel eq builder must be byte-identical to the serial one, and + /// the seeded variant must equal the gamma-scaled table, at sizes on both + /// sides of the internal parallel level floor (2^12 halves, so n = 15 + /// exercises parallel levels; n = 6 stays fully serial). #[test] - fn stateful_sumcheck_introduce_glue() { - - let n = 5; - let len = 1usize << n; - let mk = |seed: u64| -> Vec { - (0..len) - .map(|i| F128::new(seed.wrapping_mul(i as u64 + 1), seed ^ (i as u64) << 7)) - .collect() - }; - let f = mk(0xC1); - let b1 = mk(0xB1); - let b2 = mk(0xB2); - let h1: F128 = f - .iter() - .zip(b1.iter()) - .map(|(&x, &y)| x * y) - .fold(F128::ZERO, |a, v| a + v); - - let (mut prover, first) = SumcheckProver::new(f.clone(), b1.clone(), h1); - let mut ch = crate::VerifierState::detached(&(0xBEEFu64).to_le_bytes(), &[]); - let mut msgs = vec![first]; - - // Fold once before introducing b2 (must fold at the same dim as the introduced poly). - let r0 = ch.sample(); - msgs.push(prover.fold(r0)); - // Partial-eval b2 too so it matches the prover's current f dim. - let mut b2_folded = b2.clone(); - partial_eval_lsb_one(&mut b2_folded, r0); - // The h for b2 at the folded dim is Σ b2_folded · f_folded — but the verifier - // also gets to recompute this from the same shared inputs. For the test we - // pass it explicitly. - let h2_folded: F128 = b2_folded - .iter() - .zip(prover.f().iter()) - .map(|(&x, &y)| x * y) - .fold(F128::ZERO, |a, v| a + v); - msgs.push(prover.introduce_new(b2_folded.clone(), h2_folded)); - let alpha = ch.sample(); - prover.glue(alpha); - - // Continue folding to length 2 residual: n total fold-vars used, but - // we've already used 1 (r0). One more r_last is the verifier's final. - let mut ris = vec![r0]; - for _ in 0..(n - 2) { - let r = ch.sample(); - ris.push(r); - msgs.push(prover.fold(r)); - } - let r_last = ch.sample(); - ris.push(r_last); - assert_eq!(prover.f().len(), 2); - - // Verifier replays: 1 start, 1 fold, 1 introduce_new (no T_r update), 1 glue - // (combine running quad with introduced, update T_r), then (n-2) folds. - // start (idx 0) + fold(r0) → idx 1 + introduce_new → idx 2 + later folds - // Note: glue doesn't add a message; it just combines internal state. - assert_eq!(msgs.len(), 1 + 1 + 1 + (n - 2)); - - let mut t_r = h1; - // start - let q0 = RoundQuad::from_msg(msgs[0], t_r); - assert_eq!(q0.eval(F128::ZERO) + q0.eval(F128::ONE), t_r); - t_r = q0.eval(r0); // fold(r0) - // fold msg (idx 1) - let q1 = RoundQuad::from_msg(msgs[1], t_r); - assert_eq!(q1.eval(F128::ZERO) + q1.eval(F128::ONE), t_r); - // introduce_new msg (idx 2): claim is h2_folded, not T_r - let q_intro = RoundQuad::from_msg(msgs[2], h2_folded); - assert_eq!( - q_intro.eval(F128::ZERO) + q_intro.eval(F128::ONE), - h2_folded - ); - // glue: running := q1 + alpha · q_intro; T_r := T_r + alpha · h2_folded - let combined = RoundQuad::fold(&q1, &q_intro, alpha); - t_r += alpha * h2_folded; - // The combined quad must satisfy sumcheck identity against the new T_r - assert_eq!(combined.eval(F128::ZERO) + combined.eval(F128::ONE), t_r); - // Apply the rest of the folds; each subsequent msg supersedes `combined` after eval. - // After glue, the next fold uses challenge ris[1]. msgs[3] is from fold(ris[1]). - let mut running = combined; - // Remaining prover folds: ris[1..n-1] correspond to msgs[3..n+1]. - // Total prover-fold messages after start = (n-1) (single basis) ... but here we - // have 1 start + 1 fold + 1 intro + (n-2) more folds = n+1 messages. - assert_eq!(msgs.len(), n + 1); - for (k, &r) in ris.iter().enumerate().skip(1).take(n - 2) { - t_r = running.eval(r); - let msg = msgs[2 + k]; // idx 3, 4, ... - running = RoundQuad::from_msg(msg, t_r); + fn eq_table_parallel_and_seeded_match_serial() { + let mut s = 21u64; + for n in [0usize, 1, 6, 13, 15] { + let point: Vec = (0..n).map(|_| rand_ext(&mut s)).collect(); + let serial = build_eq_table_ext(&point); assert_eq!( - running.eval(F128::ZERO) + running.eval(F128::ONE), - t_r, - "post-glue round k={k}" + build_eq_table_ext_parallel(&point), + serial, + "parallel mismatch at n={n}" ); + let g = rand_ext(&mut s); + let mut seeded = vec![F192::ZERO; 1 << n]; + build_eq_table_ext_seeded_into(&point, g, &mut seeded); + let scaled: Vec = serial.iter().map(|&e| g * e).collect(); + assert_eq!(seeded, scaled, "seeded mismatch at n={n}"); } - // Final: apply r_last to the LAST message's quad - t_r = running.eval(r_last); - - let one_plus_r = F128::ONE + r_last; - let f_resid = prover.f()[0] * one_plus_r + prover.f()[1] * r_last; - // With the collapsed-basis design, combined_basis already holds - // eq + α·b2 at the residual dim. - let combined_resid = - prover.combined_basis[0] * one_plus_r + prover.combined_basis[1] * r_last; - assert_eq!( - f_resid * combined_resid, - t_r, - "residual inner product != t_r" - ); } - /// `induce_sumcheck_poly` is consistent with the codeword: - /// 1. `enforced_sum` equals `Σ_i α^i · c[q_i]` computed directly, - /// 2. `Σ_j msg[j] · basis_poly[j]` equals `enforced_sum` (the sumcheck - /// claim that the verifier reduces to a residual eval). #[test] - fn induce_sumcheck_poly_consistent_with_codeword() { - - let log_msg = 4; - let log_inv_rate = 1; - let msg_cols = 1usize << log_msg; - let block_len = msg_cols << log_inv_rate; - - // Single-lane (num_interleaved = 1, no v_challenges). - let mut ch = crate::VerifierState::detached(&(0xF00DCAFEu64).to_le_bytes(), &[]); - let msg: Vec = (0..msg_cols).map(|_| ch.sample()).collect(); - - // Encode via Flock's NTT (zero-pad to block_len). - let ntt = AdditiveNttF128::standard(log_msg + log_inv_rate); - let mut codeword = vec![F128::ZERO; block_len]; - codeword[..msg_cols].copy_from_slice(&msg); - ntt.forward_transform(&mut codeword); - - // Pick random distinct query positions. - let num_queries = 6; - let mut queries: Vec = Vec::new(); - while queries.len() < num_queries { - let q = (ch.sample().lo as usize) % block_len; - if !queries.contains(&q) { - queries.push(q); - } - } - let opened_rows: Vec> = queries.iter().map(|&q| vec![codeword[q]]).collect(); - let alpha = ch.sample_vec(log2_ceil(queries.len())); - let sks_vks = eval_sk_at_vks(log_msg); - - let (basis_poly, enforced_sum) = - induce_sumcheck_poly(log_msg, &sks_vks, &opened_rows, &[], &queries, &alpha); - assert_eq!(basis_poly.len(), msg_cols); - - // Check 1: enforced_sum = Σ_i eq(α, i_bin) · c[q_i] - let alpha_weights: Vec = primitives::multilinear::build_eq(&alpha) - .into_iter() - .take(queries.len()) - .collect(); - let expected: F128 = queries - .iter() - .zip(alpha_weights.iter()) - .map(|(&q, &w)| w * codeword[q]) - .fold(F128::ZERO, |a, v| a + v); - assert_eq!(enforced_sum, expected, "enforced_sum != eq(α)-batched c[q]"); - - // Check 2: Σ_j msg[j] · basis_poly[j] = enforced_sum. - // This is the LCH novel-basis identity: c[q] = Σ_j msg[j] · Ŵ_j(q_field), - // so Σ_i α^i · c[q_i] = Σ_j msg[j] · Σ_i α^i · Ŵ_j(q_i_field) = Σ_j msg[j] · basis_poly[j]. - let inner: F128 = msg - .iter() - .zip(basis_poly.iter()) - .map(|(&m, &b)| m * b) - .fold(F128::ZERO, |a, v| a + v); - assert_eq!(inner, enforced_sum, "msg · basis_poly != enforced_sum"); + fn roundtrip_log_n_12() { + let inst = prove_instance(12, 1); + assert!(verify_instance(&inst, &inst.proof), "honest proof rejected"); } - /// `induce_sumcheck_poly_via_ntt` must be byte-identical to dense across - /// shapes incl. the real m30_fast level dims. #[test] - fn induce_sumcheck_poly_via_ntt_matches_dense() { - - let shapes = [ - (4usize, 1usize, 0usize, 6usize), - (3, 1, 2, 5), - (6, 2, 3, 30), - (10, 1, 6, 218), - (8, 3, 3, 71), - (5, 5, 3, 43), - (0, 2, 1, 3), - ]; - for (si, &(log_msg, log_inv_rate, log_int, n_queries)) in shapes.iter().enumerate() { - let block_len = 1usize << (log_msg + log_inv_rate); - let num_interleaved = 1usize << log_int; - let mut ch = crate::VerifierState::detached(&(0xA11CEu64 ^ si as u64).to_le_bytes(), &[]); - let mut queries: Vec = Vec::new(); - while queries.len() < n_queries.min(block_len) { - let q = (ch.sample().lo as usize) % block_len; - if !queries.contains(&q) { - queries.push(q); - } - } - let nq = queries.len(); - let opened_rows: Vec> = (0..nq) - .map(|_| ch.sample_vec(num_interleaved)) - .collect(); - let v_challenges = ch.sample_vec(log_int); - let alpha = ch.sample_vec(log2_ceil(nq.max(1))); - let sks_vks = eval_sk_at_vks(log_msg); - - let dense = induce_sumcheck_poly( - log_msg, - &sks_vks, - &opened_rows, - &v_challenges, - &queries, - &alpha, - ); - let ntt = induce_sumcheck_poly_via_ntt( - log_msg, - log_inv_rate, - &opened_rows, - &v_challenges, - &queries, - &alpha, - ); - assert_eq!(ntt.1, dense.1, "shape {si}: enforced_sum"); - assert_eq!(ntt.0, dense.0, "shape {si}: basis_poly"); - } + fn roundtrip_log_n_16() { + let inst = prove_instance(16, 2); + assert!(verify_instance(&inst, &inst.proof), "honest proof rejected"); } - /// The sparse-prefix transpose must equal the baseline dense transpose on - /// the same scattered input, across sizes (incl. > and < the k=8 prefix gate). + /// At log_n = 18 the production profile's L0 has log_msg_cols = 12 and + /// enough queries to select the sparse transposed-NTT dispatch in both + /// queries, which trips the sparse transposed-NTT dispatch in BOTH the + /// prover and the dense verifier; pin the heuristic, then roundtrip. #[test] - fn transpose_sparse_matches_dense() { - - for &log_d in &[6usize, 11, 12, 14, 16, 18] { - for &nq in &[1usize, 5, 43, 218] { - let n = 1usize << log_d; - let nq = nq.min(n); - let mut ch = - crate::VerifierState::detached(&(0xC0DEu64 ^ (log_d * 131 + nq) as u64).to_le_bytes(), &[]); - let ntt = AdditiveNttF128::standard(log_d); - let mut positions: Vec = Vec::new(); - let mut values: Vec = Vec::new(); - while positions.len() < nq { - let p = (ch.sample().lo as usize) % n; - if !positions.contains(&p) { - positions.push(p); - values.push(ch.sample()); - } - } - // Baseline: scatter then dense transpose. - let mut dense = vec![F128::ZERO; n]; - for (&p, &v) in positions.iter().zip(&values) { - dense[p] += v; - } - transpose_forward_ntt(&ntt, &mut dense, log_d); - let sparse = transpose_forward_ntt_sparse(&ntt, &positions, &values, log_d); - assert_eq!(sparse, dense, "log_d={log_d}, nq={nq}"); - } - } + fn roundtrip_log_n_18_sparse_induce() { + let (pc, _) = configs_for(18).expect("Johnson profile feasible at log_n = 18"); + assert!( + induce_use_ntt_heuristic(18 - pc.initial_k, pc.log_inv_rates[0], pc.queries[0]), + "shape must select the sparse transposed-NTT induce at L0" + ); + // And the smaller roundtrips stay on the dense path (cols < 12). + let (pc16, _) = configs_for(16).unwrap(); + assert!(!induce_use_ntt_heuristic( + 16 - pc16.initial_k, + pc16.log_inv_rates[0], + pc16.queries[0] + )); + let inst = prove_instance(18, 8); + assert!(verify_instance(&inst, &inst.proof), "honest proof rejected"); } - /// As above, with num_interleaved > 1 and non-empty v_challenges (the - /// partial-eval challenges used to fold lanes). + /// The succinct verifier accepts the same honest proofs the dense one + /// does, closing through one eq-weight evaluation at the fold point. #[test] - fn induce_sumcheck_poly_with_interleaving_and_v_challenges() { - - let log_msg = 3; // msg_cols = 8 - let log_interleaved = 2; // num_interleaved = 4 - let log_inv_rate = 1; // block_len = 16 - let msg_cols = 1usize << log_msg; - let num_interleaved = 1usize << log_interleaved; - let block_len = msg_cols << log_inv_rate; - let poly_len = msg_cols * num_interleaved; - - let mut ch = crate::VerifierState::detached(&(0xDEAD_BEEFu64).to_le_bytes(), &[]); - // poly[lane * msg_cols + col] convention (matches ligero_commit input). - let poly: Vec = (0..poly_len).map(|_| ch.sample()).collect(); - - // v_challenges fold the lanes after commit. Under the LSB-lane layout, - // f_folded is just partial_eval_lsb of the poly at v_challenges. - let v_challenges: Vec = (0..log_interleaved).map(|_| ch.sample()).collect(); - let f_folded = partial_eval_lsb(&poly, &v_challenges); - assert_eq!(f_folded.len(), msg_cols); - - // Encode via ligero_commit (so we use the same matrix layout). - let ntt = AdditiveNttF128::standard(log_msg + log_inv_rate); - let w = ligero_commit(&poly, log_msg, log_interleaved, log_inv_rate, &ntt); - assert_eq!(w.block_len, block_len); - - let num_queries = 5; - let mut queries: Vec = Vec::new(); - while queries.len() < num_queries { - let q = (ch.sample().lo as usize) % block_len; - if !queries.contains(&q) { - queries.push(q); - } + fn succinct_roundtrips() { + for (log_n, seed) in [(16usize, 2u64), (18, 8)] { + let inst = prove_instance(log_n, seed); + assert!( + verify_succinct_instance(&inst, &inst.proof), + "succinct verifier rejected an honest proof at log_n={log_n}" + ); } - let opened_rows: Vec> = queries.iter().map(|&q| w.row(q).to_vec()).collect(); - - let alpha = ch.sample_vec(log2_ceil(queries.len())); - let sks_vks = eval_sk_at_vks(log_msg); - let (basis_poly, enforced_sum) = induce_sumcheck_poly( - log_msg, - &sks_vks, - &opened_rows, - &v_challenges, - &queries, - &alpha, - ); - - // The folded polynomial f_folded should satisfy Σ_j f_folded[j] · basis_poly[j] = enforced_sum. - let inner: F128 = f_folded - .iter() - .zip(basis_poly.iter()) - .map(|(&m, &b)| m * b) - .fold(F128::ZERO, |a, v| a + v); - assert_eq!( - inner, enforced_sum, - "folded-msg · basis_poly != enforced_sum (interleaved + v_challenges path)" - ); } - /// `induce_sumcheck_evaluate_at_residual` matches dense - /// `induce_sumcheck_poly` + `partial_eval_lsb`. + /// The succinct verifier rejects the same tamper cases the dense one does. #[test] - fn induce_sumcheck_evaluate_at_residual_matches_dense() { - - let log_msg_cols = 6; - let yr_log_n = 2; - let prefix_len = log_msg_cols - yr_log_n; - let num_interleaved = 4; - let log_num_interleaved = 2; - let num_queries = 5; - - let mut rng = fiat_shamir::sponge::Sponge::new(&(0x2017_5052u64).to_le_bytes(), &[]); - let queries: Vec = (0..num_queries).map(|i| (i * 7 + 3) % (1 << 8)).collect(); - let opened_rows: Vec> = (0..num_queries) - .map(|_| (0..num_interleaved).map(|_| rng.sample()).collect()) - .collect(); - let v_challenges: Vec = (0..log_num_interleaved) - .map(|_| rng.sample()) - .collect(); - let alpha: Vec = (0..log2_ceil(num_queries)) - .map(|_| rng.sample()) - .collect(); - let ris_for_basis: Vec = (0..prefix_len).map(|_| rng.sample()).collect(); - let sks_vks = eval_sk_at_vks(log_msg_cols); - - // Dense path - let (basis_dense, dense_enforced_sum) = induce_sumcheck_poly( - log_msg_cols, - &sks_vks, - &opened_rows, - &v_challenges, - &queries, - &alpha, - ); - let dense_residual = partial_eval_lsb(&basis_dense, &ris_for_basis); - - // Succinct path - let succinct_enforced_sum = - induce_sumcheck_enforced_sum(&opened_rows, &v_challenges, &queries, &alpha); - let succinct_residual = induce_sumcheck_evaluate_at_residual( - log_msg_cols, - &sks_vks, - &queries, - &alpha, - &ris_for_basis, - yr_log_n, + fn succinct_rejects_tampered() { + let inst = prove_instance(12, 3); + let mut bad = inst.proof.clone(); + bad.initial_proof.opened_rows[0][0].0 ^= 1; + assert!( + !verify_succinct_instance(&inst, &bad), + "bit-flipped L0 opened row must be rejected" ); - - assert_eq!( - succinct_enforced_sum, dense_enforced_sum, - "enforced_sum mismatch" + let mut bad2 = inst.proof.clone(); + bad2.final_proof.opened_rows[0][0].c0 ^= 1; + assert!( + !verify_succinct_instance(&inst, &bad2), + "bit-flipped final-level opened row must be rejected" ); - assert_eq!( - succinct_residual.len(), - dense_residual.len(), - "residual length mismatch" + let mut bad3 = inst.proof.clone(); + bad3.sumcheck_transcript[0].u_0.c0 ^= 1; + assert!( + !verify_succinct_instance(&inst, &bad3), + "bit-flipped sumcheck u_0 must be rejected" ); - for (i, (s, d)) in succinct_residual - .iter() - .zip(dense_residual.iter()) - .enumerate() - { - assert_eq!(s, d, "residual mismatch at y={i}"); - } } - /// Regression for the final-level proximity binding (the Ligerito - /// soundness fix). Every non-final fold level folds its opened rows - /// into the running sumcheck via `induce_sumcheck`; the final level used to - /// only Merkle-check its opened rows, leaving `yr` (the claimed final - /// message) constrained by a single scalar equation — so a malicious prover - /// could solve for a `yr` that opens the commitment to an arbitrary value. - /// - /// The fixed verifier ties `yr` to the committed codeword by checking - /// `enforced_sum_last == ⟨yr, induced_basis_last⟩`, exactly as every other - /// level does. This test pins that identity against a *real* `ligero_commit` - /// codeword: the honest `yr` (the committed message) satisfies it, and any - /// perturbed `yr` violates it. If `ligero_commit`'s additive-NTT encoding - /// and the verifier's LCH novel-basis (`induce_sumcheck_evaluate_at_residual`) - /// ever diverged, the honest assertion here would fail. + /// Dense and succinct must return the same verdict on every proof: + /// honest plus a spread of randomized single-bit tampers, at both a + /// fallback-config shape (log_n = 12) and the Johnson/OOD production + /// shape (log_n = 16). #[test] - fn final_level_binding_pins_yr_to_committed_codeword() { - - let log_msg_cols = 5; // yr has 32 entries (within the shipped yr_log_n range) - let log_inv_rate = 1; - let num_queries = 20; - let msg_cols = 1usize << log_msg_cols; - let block_len = msg_cols << log_inv_rate; - - let mut rng = fiat_shamir::sponge::Sponge::new(&(0xB19D_1235u64).to_le_bytes(), &[]); - // num_interleaved = 1 ⇒ no lane fold (level_rs empty) ⇒ yr == the message. - let yr: Vec = (0..msg_cols).map(|_| rng.sample()).collect(); - let ntt = AdditiveNttF128::standard(log_msg_cols + log_inv_rate); - let wtns = ligero_commit(&yr, log_msg_cols, 0, log_inv_rate, &ntt); - - // Distinct query positions (the protocol always samples distinct ones). - let mut queries: Vec = Vec::new(); - let mut q = 1usize; - while queries.len() < num_queries { - q = (q * 73 + 41) % block_len; - if !queries.contains(&q) { - queries.push(q); + fn dense_and_succinct_agree() { + for (log_n, seed) in [(12usize, 11u64), (16, 12)] { + let inst = prove_instance(log_n, seed); + assert!(verify_both_agree(&inst, &inst.proof, "honest proof")); + + let mut s = seed ^ 0xABCD; + type Tamper = fn(&mut LigeritoProof, u64); + let tampers: &[(&str, Tamper)] = &[ + ("L0 opened row", |p, r| { + let row = (r as usize) % p.initial_proof.opened_rows.len(); + p.initial_proof.opened_rows[row][0].0 ^= 1; + }), + ("sumcheck u_2", |p, r| { + let idx = (r as usize) % p.sumcheck_transcript.len(); + p.sumcheck_transcript[idx].u_2.c1 ^= 1; + }), + ("yr value", |p, r| { + let idx = (r as usize) % p.final_proof.yr.len(); + p.final_proof.yr[idx].c0 ^= 1; + }), + ("recursive root", |p, _| { + p.recursive_roots[0][0] ^= 1; + }), + ("merkle proof node", |p, r| { + let idx = (r as usize) % p.initial_proof.merkle_proof.len(); + p.initial_proof.merkle_proof[idx][0] ^= 1; + }), + ("grinding nonce", |p, _| { + p.grinding_nonces[0] ^= 1; + }), + ]; + for (what, tamper) in tampers { + let mut bad = inst.proof.clone(); + tamper(&mut bad, splitmix64(&mut s)); + assert!( + !verify_both_agree(&inst, &bad, what), + "tampered {what} accepted at log_n={log_n}" + ); + } + // Fold-grinding nonce tamper (present only under the Secure + // profile's nonzero L0 fold grinding, i.e. log_n = 16 here). + if !inst.proof.fold_grinding_nonces.is_empty() { + let mut bad = inst.proof.clone(); + bad.fold_grinding_nonces[0] ^= 1; + assert!( + !verify_both_agree(&inst, &bad, "fold-grinding nonce"), + "tampered fold-grinding nonce accepted at log_n={log_n}" + ); + } + if !inst.proof.ood_values.is_empty() { + let mut bad = inst.proof.clone(); + bad.ood_values[0] += F192::ONE; + assert!( + !verify_both_agree(&inst, &bad, "OOD value"), + "tampered OOD value accepted at log_n={log_n}" + ); } } - let opened_rows: Vec> = queries.iter().map(|&p| wtns.row(p).to_vec()).collect(); - - let level_rs: Vec = Vec::new(); // num_interleaved = 1 - let alpha: Vec = (0..log2_ceil(num_queries)) - .map(|_| rng.sample()) - .collect(); - - // The two quantities the fixed verifier batches into the final check. - let enforced_sum = induce_sumcheck_enforced_sum(&opened_rows, &level_rs, &queries, &alpha); - let sks_vks = eval_sk_at_vks(log_msg_cols); - let induced_basis = induce_sumcheck_evaluate_at_residual( - log_msg_cols, - &sks_vks, - &queries, - &alpha, - &[], - log_msg_cols, - ); - let inner = |v: &[F128]| -> F128 { - v.iter() - .zip(induced_basis.iter()) - .map(|(&a, &b)| a * b) - .fold(F128::ZERO, |s, x| s + x) - }; - - // Honest yr (the committed message) satisfies the proximity tie. - assert_eq!( - inner(&yr), - enforced_sum, - "honest yr must satisfy ⟨yr, induced_basis⟩ == enforced_sum" - ); - - // A forged yr violates it: perturb a coordinate with nonzero basis weight, - // so the change to the inner product is provably nonzero. - let jnz = induced_basis - .iter() - .position(|b| !b.is_zero()) - .expect("induced basis must not be identically zero"); - let mut yr_bad = yr.clone(); - yr_bad[jnz] += F128::ONE; - assert_ne!( - inner(&yr_bad), - enforced_sum, - "a forged yr must break the final-level proximity tie" - ); } - /// Build a config with explicit OOD samples and fold-challenge grinding - /// for the OOD-path tests below. - /// Shape: L0 (initial_k) → r fold levels of `k`; small query counts - /// and grind bits keep the test fast while still exercising every path. - fn ood_test_config( - log_n: usize, - initial_k: usize, - ks: &[usize], - ood_samples: Vec, - fold_grinding_bits: Vec, - ) -> LigeritoConfig { - let r = ks.len(); - let log_inv_rates: Vec = (0..=r).map(|i| 1 + i).collect(); - let mut level_log_msg_cols = Vec::new(); - let mut dim = log_n - initial_k; - for &k in ks { - level_log_msg_cols.push(dim - k); - dim -= k; - } - let queries = vec![20usize; r + 1]; - let grinding_bits = vec![0usize; r + 1]; - LigeritoConfig { - log_inv_rates, - level_steps: r, - initial_log_msg_cols: log_n - initial_k, - initial_log_num_interleaved: initial_k, - initial_k, - level_log_msg_cols, - level_ks: ks.to_vec(), - queries, - grinding_bits, - fold_grinding_bits, - ood_samples, - } + #[test] + fn proving_is_deterministic() { + let a = prove_instance(12, 7); + let b = prove_instance(12, 7); + assert_eq!(a.proof, b.proof, "same inputs must yield identical proofs"); } - /// End-to-end OOD binding + fold-challenge grinding: a JohnsonOod-shaped - /// config (explicit OOD samples at L1/L2, a few fold-grind bits at every - /// level) round-trips through BOTH the dense and succinct verifiers, and - /// tampering with either an OOD value or a fold-grinding nonce makes both - /// reject. Exercises every new prover/verifier code path. #[test] - fn ligerito_ood_and_fold_grinding_roundtrip_and_tamper() { - - let log_n = 12; - let initial_k = 2; - let ks = [2usize, 2]; - // OOD at L1 and L2 (L0 must be 0); 3 fold-grind bits at each level. - let cfg = ood_test_config(log_n, initial_k, &ks, vec![0, 2, 2], vec![3, 3, 3]); - - let mut rng = fiat_shamir::sponge::Sponge::new(&(0x00D_7E57u64).to_le_bytes(), &[]); - let poly: Vec = (0..(1usize << log_n)).map(|_| rng.sample()).collect(); - let z: Vec = (0..log_n).map(|_| rng.sample()).collect(); - let b = build_eq(&z); - let target: F128 = poly - .iter() - .zip(b.iter()) - .map(|(&a, &c)| a * c) - .fold(F128::ZERO, |a, x| a + x); - - let log_msg_cols_0 = log_n - initial_k; - let ntt_0 = AdditiveNttF128::standard(log_msg_cols_0 + 1); - let wtns_0 = ligero_commit(&poly, log_msg_cols_0, initial_k, 1, &ntt_0); - let initial_root = wtns_0.root(); - - let mut p_ch = crate::ProverState::new(b"ood-test", &[]); - let proof = multilevel_prover_with_basis( - &cfg, - poly.clone(), - b.clone(), - target, - &wtns_0.mat, - &wtns_0.tree, - &mut p_ch, + fn tampered_opened_row_rejects() { + let inst = prove_instance(12, 3); + let mut bad = inst.proof.clone(); + bad.initial_proof.opened_rows[0][0].0 ^= 1; + assert!( + !verify_instance(&inst, &bad), + "bit-flipped L0 opened row must be rejected" + ); + // Also flip a deeper (E-valued) row for good measure. + let mut bad2 = inst.proof.clone(); + bad2.final_proof.opened_rows[0][0].c0 ^= 1; + assert!( + !verify_instance(&inst, &bad2), + "bit-flipped final-level opened row must be rejected" ); - - let bundle = p_ch.into_proof(); - - let eval_b_at = { - let z = z.clone(); - move |point: &[F128]| -> F128 { primitives::multilinear::eq_eval(&z, point) } - }; - let succinct = |bundle: &fiat_shamir::transcript::Proof| { - let mut ch = crate::VerifierState::new(b"ood-test", bundle, &[]); - multilevel_verifier_with_basis_succinct( - &cfg, - &proof, - log_n, - target, - &initial_root, - &eval_b_at, - &mut ch, - ) - .is_some() - }; - - assert!(succinct(&bundle), "verifier must accept OOD proof"); - - // Stream layout of the head: [u_0, u_2] start message, then per L0 - // fold j: one raw fold-grind nonce (bits = 3−j > 0) + [u_0, u_2], - // then the L1 root (2 scalars), then per OOD sample: y + [u_0, u_2]. - let fold_nonce_0_idx = 2; - let first_ood_idx = 2 + initial_k * 3 + 2; - - // Tamper the first OOD value → reject. - let mut bad_ood = bundle.clone(); - bad_ood.stream[first_ood_idx] += F128::ONE; - assert!(!succinct(&bad_ood), "must reject tampered OOD value"); - - // Tamper a fold-grinding nonce → reject (PoW fails; the nonce is raw - // transport, already bound by the grind itself). - let mut bad_nonce = bundle.clone(); - bad_nonce.stream[fold_nonce_0_idx] += F128::new(0xDEAD_BEEF, 0); - assert!(!succinct(&bad_nonce), "must reject tampered fold nonce"); } - /// Multi-claim batched basis: `b = γ_1·eq(z_1, ·) + γ_2·eq(z_2, ·)`, - /// `target = γ_1·poly(z_1) + γ_2·poly(z_2)`. This is the shape ring_switch - /// produces. #[test] - fn multilevel_prover_with_basis_roundtrip_batched_claims() { - - let log_n = 14; - let initial_k = 3; - let k_0 = 2; - let log_inv_rate = 1; - - let mut rng = fiat_shamir::sponge::Sponge::new(&(0xBA51_BA51u64).to_le_bytes(), &[]); - let poly: Vec = (0..(1usize << log_n)).map(|_| rng.sample()).collect(); - let z1: Vec = (0..log_n).map(|_| rng.sample()).collect(); - let z2: Vec = (0..log_n).map(|_| rng.sample()).collect(); - let g1 = rng.sample(); - let g2 = rng.sample(); - let b1 = build_eq(&z1); - let b2 = build_eq(&z2); - let b: Vec = b1 - .iter() - .zip(b2.iter()) - .map(|(&a, &c)| g1 * a + g2 * c) - .collect(); - let v1: F128 = poly - .iter() - .zip(b1.iter()) - .map(|(&a, &c)| a * c) - .fold(F128::ZERO, |a, x| a + x); - let v2: F128 = poly - .iter() - .zip(b2.iter()) - .map(|(&a, &c)| a * c) - .fold(F128::ZERO, |a, x| a + x); - let target = g1 * v1 + g2 * v2; - - let log_inv_rates = vec![log_inv_rate, log_inv_rate]; - let cfg = LigeritoConfig { - log_inv_rates: log_inv_rates.clone(), - level_steps: 1, - initial_log_msg_cols: log_n - initial_k, - initial_log_num_interleaved: initial_k, - initial_k, - level_log_msg_cols: vec![log_n - initial_k - k_0], - level_ks: vec![k_0], - queries: log_inv_rates.iter().map(|&r| udr_queries(r)).collect(), - grinding_bits: vec![0; log_inv_rates.len()], - fold_grinding_bits: vec![0; 2], - ood_samples: vec![0; 2], - }; - - let log_msg_cols_0 = log_n - initial_k; - let ntt_0 = AdditiveNttF128::standard(log_msg_cols_0 + log_inv_rate); - let wtns_0 = ligero_commit(&poly, log_msg_cols_0, initial_k, log_inv_rate, &ntt_0); - let initial_root = wtns_0.root(); - - let mut p_ch = crate::ProverState::new(b"batched", &[]); - let proof = multilevel_prover_with_basis( - &cfg, - poly.clone(), - b.clone(), - target, - &wtns_0.mat, - &wtns_0.tree, - &mut p_ch, + fn tampered_sumcheck_u0_rejects() { + let inst = prove_instance(12, 4); + let mut bad = inst.proof.clone(); + bad.sumcheck_transcript[0].u_0.c0 ^= 1; + assert!( + !verify_instance(&inst, &bad), + "bit-flipped sumcheck u_0 must be rejected" ); - - // Succinct verify: the closure evaluates the batched basis - // `γ₁·eq(z₁,·) + γ₂·eq(z₂,·)` at the single fold point. - let eval_b_at = move |point: &[F128]| -> F128 { - g1 * primitives::multilinear::eq_eval(&z1, point) - + g2 * primitives::multilinear::eq_eval(&z2, point) - }; - let bundle = p_ch.into_proof(); - let mut v_ch = crate::VerifierState::new(b"batched", &bundle, &[]); - let ok = multilevel_verifier_with_basis_succinct( - &cfg, - &proof, - log_n, - target, - &initial_root, - &eval_b_at, - &mut v_ch, - ) - .is_some(); - assert!(ok, "batched-basis verifier rejected valid proof"); } + /// The E-valued interleaved NTT with K-twiddles must act lane-wise on the + /// tower coordinates: transforming (c0, c1) packed as F192 equals two + /// independent F64 transforms of the c0 and c1 lanes. #[test] - fn ligero_commit_encoding_roundtrips_via_inv_ntt() { - let log_msg = 4; // msg_cols = 16 - let log_interleaved = 3; // num_interleaved = 8 - let log_inv_rate = 1; // block_len = 32 - let msg_cols = 1 << log_msg; - let num_interleaved = 1 << log_interleaved; - let block_len = msg_cols << log_inv_rate; - - // Deterministic dummy polynomial. - let poly: Vec = (0..num_interleaved * msg_cols) - .map(|i| { - F128::new( - (i as u64).wrapping_mul(0x9E3779B97F4A7C15), - 0x1234 ^ i as u64, - ) - }) - .collect(); - - let ntt = AdditiveNttF128::standard(log_msg + log_inv_rate); - let w = ligero_commit(&poly, log_msg, log_interleaved, log_inv_rate, &ntt); - assert_eq!(w.block_len, block_len); - assert_eq!(w.num_interleaved, num_interleaved); - assert_eq!(w.mat.len(), block_len * num_interleaved); - - // Per-lane inv-NTT should recover the padded message. Under the LSB-lane - // layout, lane `lane`'s col `col` message lives at `poly[col * num_interleaved + lane]`. - for lane in 0..num_interleaved { - let mut col: Vec = (0..block_len) - .map(|pos| w.mat[pos * num_interleaved + lane]) - .collect(); - ntt.inverse_transform(&mut col); - for col_idx in 0..msg_cols { - assert_eq!( - col[col_idx], - poly[col_idx * num_interleaved + lane], - "lane {lane} col_idx {col_idx} mismatch", - ); + fn ext_ntt_matches_two_base_ntts() { + let mut s = 5u64; + for (log_d, lanes, start_layer) in [(6usize, 4usize, 0usize), (9, 2, 2), (10, 1, 1)] { + let ntt = AdditiveNttF64::standard(log_d); + let n = (1usize << log_d) * lanes; + let ext: Vec = (0..n).map(|_| rand_ext(&mut s)).collect(); + let mut c0: Vec = ext.iter().map(|e| F64(e.c0)).collect(); + let mut c1: Vec = ext.iter().map(|e| F64(e.c1)).collect(); + let mut c2: Vec = ext.iter().map(|e| F64(e.c2)).collect(); + let mut ext_t = ext.clone(); + forward_transform_interleaved_ext_from_layer(&ntt, &mut ext_t, lanes, start_layer); + ntt.forward_transform_interleaved_from_layer(&mut c0, lanes, start_layer); + ntt.forward_transform_interleaved_from_layer(&mut c1, lanes, start_layer); + ntt.forward_transform_interleaved_from_layer(&mut c2, lanes, start_layer); + for i in 0..n { + assert_eq!(ext_t[i], F192::new(c0[i].0, c1[i].0, c2[i].0), "mismatch at {i}"); } - for col_idx in msg_cols..block_len { - assert_eq!( - col[col_idx], - F128::ZERO, - "lane {lane} pad position {col_idx} not zero", - ); + } + } + + /// The sparse transposed-NTT induce must be byte-identical to the dense + /// LCH-expansion induce (same guarantee the original pins). Covers both + /// the windowed sparse-prefix path (log_block >= 12, k = 8) and the + /// scatter + full-dense-transpose path (log_block < 12, k = 0). + #[test] + fn induce_via_ntt_matches_dense() { + let mut s = 9u64; + for (log_msg_cols, log_inv_rate, lanes_log, n_queries) in [(12usize, 1usize, 5usize, 130usize), (6, 2, 3, 40)] { + let block_len = 1usize << (log_msg_cols + log_inv_rate); + let lanes = 1usize << lanes_log; + // Distinct sorted query positions plus one aligned random row each. + let mut qs: Vec = Vec::new(); + let mut seen = std::collections::HashSet::new(); + while qs.len() < n_queries { + let q = (splitmix64(&mut s) as usize) % block_len; + if seen.insert(q) { + qs.push(q); + } } + qs.sort_unstable(); + let rows: Vec> = (0..n_queries) + .map(|_| (0..lanes).map(|_| F64(splitmix64(&mut s))).collect()) + .collect(); + let v_challenges: Vec = (0..lanes_log).map(|_| rand_ext(&mut s)).collect(); + let alpha: Vec = (0..log2_ceil(n_queries)).map(|_| rand_ext(&mut s)).collect(); + + let sks_vks = eval_sk_at_vks(log_msg_cols); + let dense = induce_sumcheck_poly_base(log_msg_cols, &sks_vks, &rows, &v_challenges, &qs, &alpha); + let via_ntt = + induce_sumcheck_poly_via_ntt_base(log_msg_cols, log_inv_rate, &rows, &v_challenges, &qs, &alpha); + assert_eq!(dense.1, via_ntt.1, "enforced_sum mismatch"); + assert_eq!(dense.0, via_ntt.0, "basis_poly mismatch"); } + } - // Merkle root is deterministic: re-running the same commit yields the - // same root. - let w2 = ligero_commit(&poly, log_msg, log_interleaved, log_inv_rate, &ntt); - assert_eq!(w.root(), w2.root()); + /// The scalar and parallel ext transforms agree (parallel path is only + /// taken for larger inputs; force both on the same data). + #[test] + fn ext_ntt_scalar_matches_parallel() { + let mut s = 6u64; + let log_d = 13; + let lanes = 2; + let ntt = AdditiveNttF64::standard(log_d); + let n = (1usize << log_d) * lanes; + let orig: Vec = (0..n).map(|_| rand_ext(&mut s)).collect(); + let mut a = orig.clone(); + let mut b = orig; + forward_transform_interleaved_ext_scalar_from_layer(&ntt, &mut a, lanes, 1); + forward_transform_interleaved_ext_parallel_from_layer(&ntt, &mut b, lanes, 1); + assert_eq!(a, b); } } diff --git a/crates/pcs/src/ligerito_config.rs b/crates/pcs/src/ligerito_config.rs new file mode 100644 index 00000000..475e5ce0 --- /dev/null +++ b/crates/pcs/src/ligerito_config.rs @@ -0,0 +1,1303 @@ +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/bcc-research/bolt-rs, MIT. +// Copyright (c) 2026 Bain Capital Crypto, LP and Ron Rothblum +// Modifications copyright 2026 Succinct Labs, Benedikt Bunz, William Wang +// SPDX-License-Identifier: Apache-2.0 OR MIT +// +// Ported from bolt-rs (https://github.com/bcc-research/bolt-rs, +// `ligerito_recursive.rs`). + +//! Field-independent configuration and soundness analysis for Ligerito. +//! +//! Source of truth: `misc/pcs.tex` ("A note on WHIR/Ligerito over binary +//! fields"), Theorem `thm:rbr`. Its per-verifier-message error table maps +//! onto the per-level checks in [`LigeritoSecurityConfig::validate`]: +//! +//! - batching challenges -> [`johnson_algebraic_bits`] (this implementation +//! batches with an eq-vector challenge plus scalar glue challenges instead +//! of the doc's powers of a single alpha; see that function), +//! - fold challenge `s_j` -> `2 L/|F| + 2^(l-j) eps`: the MCA part via +//! [`paper_johnson_log_a`] (worst round `j = 1`), the `2 L/|F|` part +//! under [`johnson_algebraic_bits`], +//! - OOD challenge -> [`paper_ood_bits`], +//! - query message -> `(1 - gamma)^t`, plus [`QUERY_GRINDING_BITS`]. +//! +//! Round-by-round (RBR) soundness means every entry individually clears +//! [`SECURITY_BITS`]: the Fiat--Shamir error per random-oracle query is the +//! MAX of the entries, not their sum. +//! +use serde::{Deserialize, Serialize}; + +// =================================================================== +// Config +// =================================================================== + +// The production Ligerito configuration: rate-1/2 Johnson list decoding with +// OOD binding and 128-bit round-by-round soundness over F192. + +/// Round-by-round soundness target (bits): every verifier-challenge transition +/// must have conditional failure probability at most `2^-SECURITY_BITS`. +pub const SECURITY_BITS: usize = 128; + +/// L0 code rate index: `rho_0 = 2^-LOG_INV_RATE_0` (rate 1/2). +pub const LOG_INV_RATE_0: usize = 1; + +/// CLI-selectable L0 rates are `2^-r` for `r = 1, 2, 3, 4`. +pub const MIN_LOG_INV_RATE: usize = 1; +pub const MAX_LOG_INV_RATE: usize = 4; + +/// Validate a production Ligerito inverse-rate logarithm. +pub fn validate_log_inv_rate(log_inv_rate: usize) -> Result<(), String> { + if !(MIN_LOG_INV_RATE..=MAX_LOG_INV_RATE).contains(&log_inv_rate) { + return Err(format!( + "log_inv_rate must be in {MIN_LOG_INV_RATE}..={MAX_LOG_INV_RATE}, got {log_inv_rate}" + )); + } + Ok(()) +} + +/// Per-level query-phase proof-of-work budget. These bits are ground after the +/// level commitment and before its query positions are sampled, so the query +/// count only needs to close the remaining `SECURITY_BITS - 17` bits. +pub const QUERY_GRINDING_BITS: usize = 17; + +/// Maximum BCHKS25 integer parameter considered by the per-level eta search. +/// Production configurations hit the proximity-gap boundary far below this; +/// the generous cap makes the optimizer deterministic even if sizes expand. +const JOHNSON_ETA_SEARCH_MAX_M: usize = 4096; + +pub const INITIAL_FOLDING_FACTOR: usize = 6; +pub const SUBSEQUENT_FOLDING_FACTOR: usize = 3; + +/// Logarithmic reduction of the total Reed--Solomon domain after the initial +/// fold. With the production six-variable initial fold, `3` changes the +/// inverse-rate logarithm by `6 - 3 = 3` at the first recursive level. +pub const RS_DOMAIN_INITIAL_REDUCTION_FACTOR: usize = 3; + +/// After each subsequent fold, shrink the total Reed--Solomon domain by one +/// bit. This mirrors WHIR's recursive-domain schedule; unlike the initial +/// reduction, it is deliberately fixed rather than a tuning parameter. +const RS_DOMAIN_SUBSEQUENT_REDUCTION_FACTOR: usize = 1; + +const _: () = assert!(RS_DOMAIN_INITIAL_REDUCTION_FACTOR <= INITIAL_FOLDING_FACTOR); +const _: () = assert!(RS_DOMAIN_SUBSEQUENT_REDUCTION_FACTOR <= SUBSEQUENT_FOLDING_FACTOR); + +/// Folding stops once at most this many variables remain: the residual +/// polynomial (`yr`, at most `2^RESIDUAL_MAX_LOG` coefficients) is sent in +/// clear instead of committed and folded further. +pub const RESIDUAL_MAX_LOG: usize = 5; + +#[derive(Clone, Debug)] +pub struct ProverConfig { + pub log_inv_rates: Vec, + pub level_steps: usize, + pub initial_log_msg_cols: usize, + pub initial_log_num_interleaved: usize, + pub initial_k: usize, + pub level_log_msg_cols: Vec, + pub level_ks: Vec, + /// Per-level query counts (L0, L1, ..., L_r). Length = level_steps + 1. + /// [`LigeritoSecurityConfig::derive_config`] fills these from the + /// per-level soundness analysis. + pub queries: Vec, + /// Per-level **query-phase** PoW grinding bits (L0, L1, ..., L_r), ground + /// post-commit/pre-queries. Length = level_steps + 1. Each bit here + /// substitutes for ~1/log₂(1/(1−γ)) queries at that level. + pub grinding_bits: Vec, + /// Per-level **fold-challenge** PoW grinding bits (L0, ..., L_r), ground + /// immediately before EACH of the level's fold challenges (so a level + /// with `k` folds does `k` grinds of this many bits). Boosts the + /// proximity-gap term, which lives on the fold challenges. Length = + /// level_steps + 1. + pub fold_grinding_bits: Vec, + /// Per-commit-level out-of-domain samples (L0, ..., L_r), taken right + /// after the level's Merkle root enters the transcript. `[0]` must be 0: + /// L0 is bound by the opening's own (post-commit, random-point) + /// evaluation claim. Length = level_steps + 1. + pub ood_samples: Vec, +} + +/// The per-level shape table a [`VerifierConfig`] implies for a +/// `log_n`-variable opening — the numbers every consumer of the multilevel +/// protocol (the verifier itself, recursion harnesses) otherwise re-derives. +#[derive(Clone, Debug)] +pub struct LevelShapes { + /// Level count (`level_steps + 1`). + pub levels: usize, + /// Fold count per level: `initial_k` then `level_ks`. + pub ks: Vec, + /// Log message columns entering each level's fold (`log_n - initial_k`, + /// then descending by each level's `k`). + pub log_msg_cols: Vec, + /// Committed block length per level (`msg_cols * inv_rate`). + pub block_len: Vec, + /// The residual cube dimension left after every fold. + pub yr_log_n: usize, +} + +#[derive(Clone, Debug)] +pub struct VerifierConfig { + pub log_inv_rates: Vec, + pub level_steps: usize, + pub initial_log_msg_cols: usize, + pub initial_log_num_interleaved: usize, + pub initial_k: usize, + pub level_log_msg_cols: Vec, + pub level_ks: Vec, + /// Per-level query counts. Length = level_steps + 1. + pub queries: Vec, + /// Per-level query-phase PoW grinding bits. Length = level_steps + 1. + pub grinding_bits: Vec, + /// Per-level fold-challenge PoW grinding bits (one grind per fold + /// challenge of the level). Length = level_steps + 1. + pub fold_grinding_bits: Vec, + /// Per-commit-level OOD samples. Length = level_steps + 1. + pub ood_samples: Vec, +} + +impl VerifierConfig { + /// See [`LevelShapes`]. + pub fn level_shapes(&self, log_n: usize) -> LevelShapes { + let r = self.level_steps; + let ks: Vec = std::iter::once(self.initial_k) + .chain(self.level_ks.iter().copied()) + .collect(); + let mut log_msg_cols = vec![log_n - self.initial_k]; + for i in 0..r { + log_msg_cols.push(log_msg_cols[i] - self.level_ks[i]); + } + let mut block_len = vec![1usize << (self.initial_log_msg_cols + self.log_inv_rates[0])]; + for i in 0..r { + block_len.push(1usize << (self.level_log_msg_cols[i] + self.log_inv_rates[i + 1])); + } + LevelShapes { + levels: r + 1, + ks, + yr_log_n: *log_msg_cols.last().unwrap(), + log_msg_cols, + block_len, + } + } +} + +/// Soundness (in bits) the query phase must close on its own at every level +/// (the "100 bits from queries always" policy). +#[cfg(test)] +const UDR_TARGET_BITS: f64 = 100.0; + +/// Number of queries for 100-bit soundness in the **unique-decoding regime** +/// at rate `2^(-log_inv_rate)`: `γ = δ/2 = (1−ρ)/2`, per-query soundness +/// `log₂(1/(1−γ))` (see [`udr_per_query_bits`]). Within the unique decoding +/// radius the prover is pinned to a single codeword, so there is no list and +/// no union-bound term — queries close the full target by themselves. +/// Per-query soundness saturates below 1 bit (`γ < 1/2`), so slimmer codes +/// bottom out near `UDR_TARGET_BITS` queries: 243 at rate 1/2, 148 at 1/4, +/// 121 at 1/8, 110 at 1/16, 105 at 1/32. +#[cfg(test)] +pub fn udr_queries(log_inv_rate: usize) -> usize { + assert!(log_inv_rate > 0, "log_inv_rate=0 (rate 1) has no soundness"); + let per_q = udr_per_query_bits_asymptotic(log_inv_rate); + (UDR_TARGET_BITS / per_q).ceil() as usize +} + +/// Build an ad-hoc Ligerito config from the raw PCS shape, WITHOUT the +/// per-level soundness derivation of [`LigeritoSecurityConfig::derive_config`]. +/// `log_n` is the packed-witness log size (= `m - LOG_PACKING`). +/// +/// Strategy: 3-bit recursive folds (`k_i = 3`) with **decreasing rate** (one +/// rate step per level) until the residual is small (`≤ 5` bits), asserting +/// `block_len ≥ udr_queries(rate)` at every level. Returns `Err` when no +/// feasible config exists (e.g. `log_n` too small for the chosen rate). +/// +/// Test-support only: the small F64 PCS tests exercise sizes below `derive_config`'s +/// feasibility floor, where they fall back to this shape. Production callers +/// use `derive_config` (the audited, per-level-sound path). +#[cfg(test)] +pub fn default_config(log_n: usize, log_batch_size: usize, log_inv_rate: usize) -> Result { + let initial_k = log_batch_size; + if log_n <= initial_k { + return Err("log_n must be > initial_k"); + } + + let mut log_inv_rates = vec![log_inv_rate]; + let mut level_ks = Vec::new(); + let mut level_log_msg_cols = Vec::new(); + + let mut n_running = log_n - initial_k; + let mut rate_running = log_inv_rate; + + // L0 feasibility check. + { + let block_len_log = n_running + rate_running; + let qs = udr_queries(rate_running); + if (1usize << block_len_log) < qs { + return Err("L0 block_len < udr_queries — log_n too small for chosen rate"); + } + } + + while n_running > 5 { + let k = 3.min(n_running); + let log_msg_cols_next = n_running - k; + // Pick the smallest rate ≥ rate_running+1 such that block_len ≥ queries. + let mut next_rate = rate_running + 1; + loop { + let bl = 1usize << (log_msg_cols_next + next_rate); + let qs = udr_queries(next_rate); + if bl >= qs { + break; + } + next_rate += 1; + if next_rate > 20 { + return Err("could not find feasible recursive rate (level too deep)"); + } + } + level_log_msg_cols.push(log_msg_cols_next); + level_ks.push(k); + log_inv_rates.push(next_rate); + n_running -= k; + rate_running = next_rate; + } + + if level_ks.is_empty() { + return Err("log_n too small — no recursive levels needed (use BaseFold directly)"); + } + + let queries: Vec = log_inv_rates.iter().map(|&r| udr_queries(r)).collect(); + let n_levels = log_inv_rates.len(); + let grinding_bits = vec![0usize; n_levels]; + + Ok(ProverConfig { + log_inv_rates: log_inv_rates.clone(), + level_steps: level_ks.len(), + initial_log_msg_cols: log_n - initial_k, + initial_log_num_interleaved: initial_k, + initial_k, + level_log_msg_cols, + level_ks, + queries, + grinding_bits, + fold_grinding_bits: vec![0usize; n_levels], + ood_samples: vec![0usize; n_levels], + }) +} + +/// The [`VerifierConfig`] matching [`default_config`] (test-support only). +#[cfg(test)] +pub fn default_verifier_config( + log_n: usize, + log_batch_size: usize, + log_inv_rate: usize, +) -> Result { + let p = default_config(log_n, log_batch_size, log_inv_rate)?; + Ok(VerifierConfig { + log_inv_rates: p.log_inv_rates, + level_steps: p.level_steps, + initial_log_msg_cols: p.initial_log_msg_cols, + initial_log_num_interleaved: p.initial_log_num_interleaved, + initial_k: p.initial_k, + level_log_msg_cols: p.level_log_msg_cols, + level_ks: p.level_ks, + queries: p.queries, + grinding_bits: p.grinding_bits, + fold_grinding_bits: p.fold_grinding_bits, + ood_samples: p.ood_samples, + }) +} + +/// Level-ladder shape: per-level dims (index 0 = L0) plus the residual. +struct LadderShape { + log_inv_rates: Vec, + log_msg_cols: Vec, + log_num_interleaved: Vec, + k_levels: Vec, + yr_log_n: usize, +} + +/// Shared shape derivation behind [`LigeritoSecurityConfig::derive_config`]. +/// The total RS domain loses [`RS_DOMAIN_INITIAL_REDUCTION_FACTOR`] bits after +/// the initial fold, then exactly one bit per subsequent fold. Consequently a +/// fold of `k` variables raises the inverse-rate logarithm by `k - reduction`. +fn derive_ladder_shape(log_n: usize, initial_k: usize, log_inv_rate: usize) -> Result { + if log_n <= initial_k { + return Err("log_n must be > initial_k".into()); + } + let mut shape = LadderShape { + log_inv_rates: vec![log_inv_rate], + log_msg_cols: vec![log_n - initial_k], + log_num_interleaved: vec![initial_k], + k_levels: vec![initial_k], + yr_log_n: 0, + }; + let mut n_running = log_n - initial_k; + let mut rate_running = log_inv_rate; + let mut fold_running = initial_k; + let mut domain_reduction = RS_DOMAIN_INITIAL_REDUCTION_FACTOR; + while n_running > RESIDUAL_MAX_LOG { + let k = SUBSEQUENT_FOLDING_FACTOR.min(n_running); + let log_msg_cols_next = n_running - k; + let rate_increase = fold_running.checked_sub(domain_reduction).ok_or_else(|| { + format!("folding factor {fold_running} is smaller than RS domain reduction {domain_reduction}") + })?; + let next_rate = rate_running + rate_increase; + shape.log_inv_rates.push(next_rate); + shape.log_msg_cols.push(log_msg_cols_next); + shape.log_num_interleaved.push(k); + shape.k_levels.push(k); + n_running -= k; + rate_running = next_rate; + fold_running = k; + domain_reduction = RS_DOMAIN_SUBSEQUENT_REDUCTION_FACTOR; + } + if shape.k_levels.len() < 2 { + return Err("log_n too small: needs at least 2 fold levels".into()); + } + shape.yr_log_n = n_running; + Ok(shape) +} + +// =================================================================== +// Security configuration schema +// =================================================================== +// +// Auditable, per-level spec for a Ligerito instance: query count, grinding +// bits, slack-from-Johnson, and the proximity-gap analysis the parameters +// were derived under. Designed to be (de)serializable so it can live in a +// TOML/JSON file alongside the prover/verifier code. + +/// Which proximity-gap analysis a level's parameters were derived under. +/// Single-variant by design: it self-documents the analysis in serialized +/// configs and rejects configs claiming an analysis this code cannot check. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SoundnessRegime { + /// Johnson radius with explicit slack `η` (γ = (1 − √ρ) − η) **with + /// out-of-domain binding** (`misc/pcs.tex`, Thm `thm:rbr`). The MCA + /// theorem (`thm:mca-johnson` = BCHKS25 Thm 4.6) gives the proximity-gap + /// exceptional set `a = O_ρ(n / η^5)`; the level's `fold_grinding_bits` + /// should be ≥ (target_bits − log₂(q/a)). + /// Binding to a single codeword of the (Johnson-bounded) interleaved list + /// is via `ood_samples` explicit multilinear OOD evaluations — except at + /// L0, where the opening's own post-commit random evaluation claim plays + /// the OOD role (union over the list, `L·μ/q`), so `ood_samples = 0`. + /// + /// Note there is deliberately no plain `Johnson` variant: without OOD + /// binding the query phase pays a union bound over the interleaved list + /// (≈ 19–52 bits here), which our query counts do not include. A config + /// claiming Johnson soundness without OOD accounting would be unsound. + JohnsonOod, +} + +/// Where in a level's Fiat-Shamir transcript the grinding step lands. +/// Currently only one choice; reserved for future protocol variants. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GrindingStep { + /// Grind happens after the level's Merkle root is observed but before + /// query positions are sampled. Standard FRI/STARK pattern. + PostCommitPreQueries, +} + +/// Parameters for a single level in the multilevel Ligerito ladder. +/// L0 = the upstream `pcs::commit` output (reused, not re-committed); +/// L1 .. L_{r−1} are the level commits; the final residual `yr` block +/// is described separately in [`FinalBlockConfig`]. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LigeritoLevelConfig { + /// PCS rate at this level: codeword expansion factor = 2^log_inv_rate. + pub log_inv_rate: usize, + /// Message dimension at this level (log of the number of field columns in + /// the codeword). `log_msg_cols + log_inv_rate = log_2(block_len)`. + pub log_msg_cols: usize, + /// Log of lane width per Merkle leaf at this level. For L0 = `initial_k`; + /// for L_i (i ≥ 1) = the previous level's `k`. + pub log_num_interleaved: usize, + /// Number of sumcheck folds taken at this level. For L0 = `initial_k` + /// (the lane fold); for L_i (i ≥ 1) = the level fold k_{i−1}. + pub k: usize, + /// Which proximity-gap analysis the (eta, queries, grinding_bits) + /// tuple was derived under. Determines the formulas the implementation + /// validates against. + pub regime: SoundnessRegime, + /// Slack from the Johnson radius: γ = (1 − √ρ) − η. + pub eta: f64, + /// Number of codeword position queries opened at this level (the FRI + /// query phase). Bounds the per-query soundness term `(1−γ)^Q`. + pub queries: usize, + /// **Query-phase** PoW grinding bits, ground post-commit/pre-queries + /// (see [`GrindingStep`]). Each bit substitutes for + /// ~1/log₂(1/(1−γ)) queries at this level. + pub grinding_bits: usize, + /// **Fold-challenge** PoW grinding bits, ground immediately before EACH + /// of this level's `k` fold challenges. Boosts the + /// proximity-gap term (which lives on the fold challenges): + /// `eps_pg + fold_grinding_bits ≥ target`. + #[serde(default)] + pub fold_grinding_bits: usize, + /// Out-of-domain samples taken right after this level's commit enters + /// the transcript. Each binds the prover to a single codeword of the + /// interleaved list via a multilinear evaluation claim. + /// Must be 0 at L0 (bound by the opening's own post-commit evaluation + /// claim) and ≥ 1 at deeper levels. + #[serde(default)] + pub ood_samples: usize, + /// Security target this level guarantees, post-grinding. + pub target_security_bits: usize, + /// Diagnostic — `log₂(q/a)` under the chosen regime. The implementation + /// should assert this matches the formula at startup, modulo rounding. + pub expected_eps_pg_bits: f64, + /// Diagnostic — `Q · log₂(1/(1−γ))`. Should be ≥ + /// `target_security_bits − grinding_bits`. + pub expected_eps_query_bits: f64, + /// Diagnostic — OOD binding bits: + /// `s·(192 − log₂μ) − (2·log₂L − 1)` for explicit samples, or + /// `192 − log₂L − log₂μ` for the implicit L0 binding, where `L` is the + /// Johnson interleaved list size and `μ` the level's variable count. + pub expected_eps_ood_bits: f64, +} + +/// Descriptor for the final-residual block (`yr`) sent in the clear at the +/// end of the last fold level. It has no commit and no queries, so the +/// only meaningful parameter is its dimension. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct FinalBlockConfig { + /// `log_2(|yr|)` — number of extension-field values sent in the clear. The last + /// fold level's sumcheck stops at this dim instead of folding to 1. + pub yr_log_n: usize, +} + +/// Complete security spec for one Ligerito instance, covering a single +/// `(hash, m)` pair. Designed to round-trip cleanly via serde (TOML/JSON). +/// +/// **Validation invariants** (checked by [`Self::validate`]): +/// 1. `initial_k + Σ levels[1..].k + final_block.yr_log_n == log_n`. +/// 2. Each level's `expected_eps_pg_bits` is consistent with the declared +/// regime and `eta` (within tolerance). +/// 3. Each level's `expected_eps_query_bits ≥ target_security_bits − +/// grinding_bits` (queries cover what grinding doesn't). +/// 4. `eta` is finite and inside the Johnson range for the level's rate. +/// 5. `log_msg_cols`, `log_num_interleaved`, `k` match the +/// level-shape constraint (each level's input dim equals the +/// previous level's `log_msg_cols`). +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LigeritoSecurityConfig { + /// Block-encoder log size: m = log₂(witness bit count). + pub m: usize, + /// Committed-witness log dimension. + pub log_n: usize, + /// L0 lane fold. Must equal the upstream `PcsParams::log_batch_size` so + /// the L0 commit can be reused without re-committing. + pub initial_k: usize, + /// Round-by-round security target (bits): `validate()` asserts that every + /// error term associated with a verifier challenge clears at least this + /// much. This is an RBR target, not a claim that the sum of all interactive + /// failure probabilities is bounded by `2^-target_security_bits`. + pub target_security_bits: usize, + /// Identifier of the proximity-gap analysis used. Self-documents which + /// theorem the per-level parameters were derived from. Example: + /// `"ben_sasson_2025_thm_4_6"`. + pub analysis_version: String, + /// Field of the protocol. Example: `"f192"`. + pub field: String, + /// Hash function used by Merkle + FS sponge. Example: `"blake3"`. + pub hash: String, + /// Where in the per-level FS transcript grinding is placed. + pub grinding_step: GrindingStep, + /// Per-level parameters, in order L0, L1, L2, .... + pub levels: Vec, + /// Final residual block descriptor. + pub final_block: FinalBlockConfig, +} + +/// Extension-field size used for soundness analysis: `q = 2^192`. +const ANALYSIS_LOG_Q: f64 = 192.0; + +/// BCHKS25 parameter `rho = k/n` for an RS code of dimension `k + 1`. +/// Our message has `2^log_msg_cols` coefficients (degree strictly below that +/// value), so `k = 2^log_msg_cols - 1`. This differs perceptibly from the +/// nominal code rate at the small recursive levels. +fn reduced_rate(log_inv_rate: usize, log_msg_cols: usize) -> f64 { + let dimension = (log_msg_cols as f64).exp2(); + (dimension - 1.0) / ((log_msg_cols + log_inv_rate) as f64).exp2() +} + +/// Round a float to one decimal place. Used to round paper-predicted +/// soundness diagnostics so the generated TOMLs stay readable. +fn round1(x: f64) -> f64 { + (x * 10.0).round() / 10.0 +} + +/// Bit-level tolerance when comparing declared diagnostics +/// (`expected_eps_pg_bits` / `expected_eps_query_bits`) against the value +/// computed from the regime's formulas. Set generously enough that rounding +/// in the TOML doesn't cause spurious failures, but tightly enough that an +/// incorrect declaration of η, Q, or grinding can't slip through. +const PAPER_COMPAT_TOL_BITS: f64 = 0.6; + +/// Proximity-gap exceptional set for the list-decoding (Johnson) regime, per +/// `misc/pcs.tex` Thm `thm:mca-johnson` = BCHKS25 Theorem 4.6 (list +/// correlated agreement). For a Reed–Solomon code of (slightly reduced) rate +/// `ρ`, codeword length `n`, and Johnson slack `η` (proximity radius +/// `γ = 1 − √ρ − η`), the MCA error is `a/|F|` with +/// +/// `a = [2(m+½)^5 + 3(m+½)·γ·ρ] / (3·ρ^{3/2}) · n + (m+½)/√ρ`, +/// +/// where `η = 1 − √ρ − γ` and `m = max(⌈√ρ/η⌉, 3)`. Returns `log₂ a`. +/// +/// This is the per-fold-step MCA error, stated for a two-row interleaved word +/// (`C ∈ F^{2×n}`). The ℓ-round lane fold of a `2^ℓ`-interleaved word adds a +/// row-union factor via `pcs.tex` Lemma `lem:fold-list`; see +/// [`paper_johnson_log_a`]. +fn paper_thm_ca_johnson_log_a(log_inv_rate: usize, eta: f64, log_msg_cols: usize) -> f64 { + let rho = reduced_rate(log_inv_rate, log_msg_cols); + let sqrt_rho = rho.sqrt(); + let gamma = 1.0 - sqrt_rho - eta; + // BCHKS25 Thm 4.6: m = ⌈√ρ/(1−√ρ−γ)⌉ = ⌈√ρ/η⌉, floored at 3. + let m_param = johnson_m_param(log_inv_rate, log_msg_cols, eta); + let half = m_param + 0.5; + let half5 = half.powi(5); + let numerator = 2.0 * half5 + 3.0 * half * gamma * rho; + let denominator = 3.0 * rho.powf(1.5); + let n = ((log_msg_cols + log_inv_rate) as f64).exp2(); + let a = (numerator / denominator) * n + half / sqrt_rho; + a.log2() +} + +/// Integer parameter `m = max(⌈√ρ/η⌉, 3)` of BCHKS25 Thm 4.6 (list +/// correlated agreement), represented as `f64` for the bound. Beware: the +/// plain, non-list Thm 1.5 has the factor-two-smaller `⌈√ρ/(2η)⌉`, and +/// Flock's Thm 8 quotes Thm 4.6 with that non-list parameter; the list form +/// costs a factor 2 of slack (see the footnote in `pcs.tex` +/// Thm `thm:mca-johnson`). +fn johnson_m_param(log_inv_rate: usize, log_msg_cols: usize, eta: f64) -> f64 { + let sqrt_rho = reduced_rate(log_inv_rate, log_msg_cols).sqrt(); + ((sqrt_rho / eta).ceil() as usize).max(3) as f64 +} + +/// Johnson-regime proximity-gap `log₂ a` for a level, including the row-union +/// factor from `pcs.tex` Lemma `lem:fold-list` ("Folding preserves lists"). +/// +/// The base MCA error `ε = a_RLC/|F|` from [`paper_thm_ca_johnson_log_a`] is +/// stated for a two-row interleaved word (one fold step). Folding a +/// `2^ℓ`-interleaved word (ℓ = `log_num_interleaved`) over its ℓ lane-fold +/// rounds pays a row union: `thm:rbr`'s fold row is `2L/|F| + 2^{ℓ-j}·ε` at +/// round `j`, so the worst round (`j = 1`) pays the factor `2^{ℓ-1}` = +/// (interleaving factor)/2 (the `2L/|F|` part is checked separately, under +/// [`johnson_algebraic_bits`]). We bind the per-level grinding to that worst +/// round, returning `log₂(2^{ℓ-1}·a_RLC) = log₂ a_RLC + (ℓ-1)`. +/// +/// `ℓ ≤ 1` (`L ≤ 2`) means no row union; the `(ℓ-1)` penalty clamps to 0. +fn paper_johnson_log_a(log_inv_rate: usize, eta: f64, log_msg_cols: usize, log_num_interleaved: usize) -> f64 { + let base = paper_thm_ca_johnson_log_a(log_inv_rate, eta, log_msg_cols); + // Row-union factor 2^{ℓ-1} (worst round i=1 of the ℓ-round lane fold), + // ℓ = log_num_interleaved. In bits: (ℓ-1), clamped ≥ 0. + let row_union_penalty = (log_num_interleaved as f64 - 1.0).max(0.0); + base + row_union_penalty +} + +/// Per-query log₂(1/(1−γ)) under the Johnson regime: each query closes +/// `log_2(1/(1-γ))` bits of soundness against a γ-far adversary. +fn paper_per_query_bits(log_inv_rate: usize, log_msg_cols: usize, eta: f64) -> f64 { + let rho = reduced_rate(log_inv_rate, log_msg_cols); + let gamma = 1.0 - rho.sqrt() - eta; + (1.0 / (1.0 - gamma)).log2() +} + +/// Unique-decoding-regime per-query soundness at `γ = δ/2` (`δ = 1 − ρ`). +/// Test-support only, backing [`udr_queries`] and the ad-hoc +/// [`default_config`] shape used by small F64 PCS tests. +#[cfg(test)] +fn udr_per_query_bits_asymptotic(log_inv_rate: usize) -> f64 { + let rho = (-(log_inv_rate as f64)).exp2(); + let gamma = (1.0 - rho) / 2.0; + (1.0 / (1.0 - gamma)).log2() +} + +/// Johnson-bound list size of the *interleaved* RS code at radius +/// `θ = 1 − √ρ − η`, in log₂. Independent of the interleaving factor. +/// +/// Interleaving preserves relative distance — `V^{⊙m}` has the base code's +/// distance `δ = 1 − ρ` — and only enlarges the alphabet (to `q^m`). The +/// Johnson bound depends solely on (distance, radius, alphabet size), so the +/// interleaved list size at any radius *below* the Johnson radius `1 − √ρ` +/// is bounded by the very same single-code Johnson list size +/// +/// `L_int ≤ L_base ≤ 1/(2·η·√ρ)`, +/// +/// with no dependence on `m` and, crucially, no `L_base^r` blow-up. +/// +/// The general GGR (Gopalan–Guruswami–Raghavendra, Thm 2.5) interleaved bound +/// `L_int ≤ C(b+r, r)·L_base^r` is only needed to push the list-decoding +/// radius *past* the Johnson bound toward `δ`. Ligerito deliberately sits at +/// `θ = 1 − √ρ − η`, strictly below the Johnson radius by slack `η > 0`, so +/// that regime never applies and the plain Johnson bound is both correct and +/// far tighter (it dominates GGR throughout the regime RS can reach). +fn johnson_interleaved_list_log2(log_inv_rate: usize, log_msg_cols: usize, eta: f64) -> f64 { + debug_assert!(eta > 0.0, "η must be > 0 to stay strictly below the Johnson radius"); + let rho = reduced_rate(log_inv_rate, log_msg_cols); + let sqrt_rho = rho.sqrt(); + let l_base = 1.0 / (2.0 * eta * sqrt_rho); + l_base.log2() +} + +/// Worst algebraic verifier-challenge transition in the production opening — +/// the implementation's counterpart of `thm:rbr`'s batch row (`(T−1)·L/|F|` +/// for powers-of-alpha batching) and of the `2L/|F|` part of its fold row. +/// This codebase batches differently from the doc: the per-level query +/// consistency claims are combined with a multilinear eq-vector challenge, +/// and each new claim (OOD, induced) is glued into the single running +/// sumcheck with a fresh scalar challenge. A degree-`d` identity test +/// unioned over a Johnson list of size `L` fails with probability at most +/// `dL/|F|`. The relevant degrees are: +/// +/// - the total degree of the GF64-to-GF192 ring-switch batching map (L0 only, +/// but included at every level so the bound also dominates the eq-vector +/// batch entering the next level's list, whatever its query count); +/// - `ceil(log2(queries))` for the multilinear query-row batching; and +/// - 2 for quadratic sumcheck (glue challenges have degree 1). +fn johnson_algebraic_bits_for(log_inv_rate: usize, log_msg_cols: usize, eta: f64, queries: usize) -> f64 { + let log2_l = johnson_interleaved_list_log2(log_inv_rate, log_msg_cols, eta); + let degree = crate::ring_switch::RING_SWITCH_SOUNDNESS_DEGREE + .max(log2_ceil(queries)) + .max(2); + ANALYSIS_LOG_Q - (degree as f64).log2() - log2_l +} + +fn johnson_algebraic_bits(level: &LigeritoLevelConfig) -> f64 { + johnson_algebraic_bits_for(level.log_inv_rate, level.log_msg_cols, level.eta, level.queries) +} + +/// OOD binding bits for a level. `mu_vars` is the level's multilinear +/// variable count (`log_msg_cols + log_num_interleaved`). +/// +/// - `ood_samples ≥ 1` (explicit samples): `pcs.tex` Lemma `lem:ood` / +/// `thm:rbr`'s OOD row `binom(L,2)·μ/|F|`, generalized to `s` samples: the +/// bad event is two distinct list elements agreeing on all `s` random +/// points of `F^μ` (Schwartz–Zippel, total degree ≤ μ), union over pairs: +/// `bits = s·(192 − log₂ μ) − (2·log₂ L_int − 1)`. +/// - `ood_samples = 0` (L0): the protocol takes no OOD sample at commitment, +/// so the PCS itself is only list binding (`pcs.tex`, abstract). What this +/// term materializes is the OUTER protocol's binding: the opening's own +/// evaluation claim sits at a post-commit random point, so at most one +/// list member matches it except with `L·μ/|F|` (union over the list, not +/// pairs): `bits = 192 − log₂ L_int − log₂ μ`. +fn paper_ood_bits(log_inv_rate: usize, log_msg_cols: usize, eta: f64, mu_vars: usize, ood_samples: usize) -> f64 { + let log2_l = johnson_interleaved_list_log2(log_inv_rate, log_msg_cols, eta); + let log2_mu = (mu_vars as f64).log2(); + if ood_samples == 0 { + ANALYSIS_LOG_Q - log2_l - log2_mu + } else { + ood_samples as f64 * (ANALYSIS_LOG_Q - log2_mu) - (2.0 * log2_l - 1.0) + } +} + +/// Result of the WHIR-style per-level Johnson-slack search. The search +/// minimizes queries; ties keep the smallest theorem parameter `m`, which has +/// the largest eta and therefore the smallest list bound. +struct OptimizedJohnsonLevel { + eta: f64, + queries: usize, + ood_samples: usize, + eps_pg: f64, + eps_query: f64, + eps_ood: f64, +} + +/// Eta at the lower boundary for a fixed BCHKS25 theorem parameter +/// `m = ceil(sqrt(rho) / eta)`. Moving eta lower would increase `m` and worsen +/// the proximity-gap bound; this boundary maximizes query soundness for the +/// given `m`. Step upward by an ulp if floating-point division lands just +/// below the intended ceil boundary. +fn johnson_eta_for_m(log_inv_rate: usize, log_msg_cols: usize, m: usize) -> f64 { + debug_assert!(m >= 3); + let sqrt_rho = reduced_rate(log_inv_rate, log_msg_cols).sqrt(); + let mut eta = sqrt_rho / m as f64; + while johnson_m_param(log_inv_rate, log_msg_cols, eta) > m as f64 { + eta = f64::from_bits(eta.to_bits() + 1); + } + debug_assert_eq!(johnson_m_param(log_inv_rate, log_msg_cols, eta), m as f64); + eta +} + +/// Choose eta independently for one recursive level, following leanVM's +/// discrete `m` search but using this implementation's exact reduced rate and +/// corrected BCHKS25 parameter. Candidates must satisfy every non-grindable +/// 128-bit term and the proximity-gap target without fold grinding. +fn optimize_johnson_level( + level: usize, + log_inv_rate: usize, + log_msg_cols: usize, + log_num_interleaved: usize, + target_bits: usize, + query_grinding_bits: usize, +) -> Result { + let target = target_bits as f64; + let query_target = target_bits.saturating_sub(query_grinding_bits).max(1) as f64; + let mu = log_msg_cols + log_num_interleaved; + let block_len = 1usize << (log_msg_cols + log_inv_rate); + let mut best: Option = None; + + for m in 3..=JOHNSON_ETA_SEARCH_MAX_M { + let eta = johnson_eta_for_m(log_inv_rate, log_msg_cols, m); + let max_eta = 1.0 - reduced_rate(log_inv_rate, log_msg_cols).sqrt(); + if eta >= max_eta { + continue; + } + + let eps_pg = ANALYSIS_LOG_Q - paper_johnson_log_a(log_inv_rate, eta, log_msg_cols, log_num_interleaved); + // At the theorem-parameter boundaries a grows monotonically with m; + // no later candidate can recover once the proximity-gap target fails. + if eps_pg + 1e-12 < target { + break; + } + + let per_q = paper_per_query_bits(log_inv_rate, log_msg_cols, eta); + if !per_q.is_finite() || per_q <= 0.0 { + continue; + } + let queries = (query_target / per_q).ceil() as usize; + if queries > block_len { + continue; + } + let eps_query = queries as f64 * per_q; + + let ood_samples = if level == 0 { + 0 + } else { + match (1..=8usize).find(|&s| paper_ood_bits(log_inv_rate, log_msg_cols, eta, mu, s) + 1e-12 >= target) { + Some(samples) => samples, + None => continue, + } + }; + let eps_ood = paper_ood_bits(log_inv_rate, log_msg_cols, eta, mu, ood_samples); + if eps_ood + 1e-12 < target + || johnson_algebraic_bits_for(log_inv_rate, log_msg_cols, eta, queries) + 1e-12 < target + { + continue; + } + + let candidate = OptimizedJohnsonLevel { + eta, + queries, + ood_samples, + eps_pg, + eps_query, + eps_ood, + }; + if best.as_ref().is_none_or(|current| candidate.queries < current.queries) { + best = Some(candidate); + } + } + + best.ok_or_else(|| { + format!( + "L{level}: no eta candidate satisfies {target_bits}-bit Johnson/OOD soundness at rate 1/2^{log_inv_rate}" + ) + }) +} + +impl LigeritoLevelConfig { + /// Compute the proximity-gap and per-query soundness bits this level is + /// expected to deliver under its declared regime. Returns + /// `(eps_pg_bits, eps_query_bits)` where: + /// eps_pg_bits = log₂(q/a) under the regime's threshold-a formula + /// eps_query_bits = Q · log₂(1/(1−γ)) + /// + /// Used by [`LigeritoSecurityConfig::validate`] to assert the declared + /// `expected_*_bits` diagnostics are consistent with the regime's + /// canonical formulas (i.e., the config is compatible with the paper). + pub fn paper_predicted_bits(&self) -> (f64, f64) { + // Fold row of `thm:rbr`, MCA part: the ℓ-round fold of a + // 2^ℓ-interleaved word (ℓ = log_num_interleaved) pays a row-union + // factor 2^{ℓ-j} at round j (`lem:fold-list`); the worst round (j=1) + // gives 2^{ℓ-1}, on top of the base Thm 4.6 MCA error. + let log_a = paper_johnson_log_a(self.log_inv_rate, self.eta, self.log_msg_cols, self.log_num_interleaved); + let eps_pg = ANALYSIS_LOG_Q - log_a; + // Per-query soundness WITHOUT a list union bound — the OOD + // binding (see `paper_ood_bits`) pins the prover to a single + // codeword of the interleaved list before queries are drawn. + let per_q = paper_per_query_bits(self.log_inv_rate, self.log_msg_cols, self.eta); + let eps_query = self.queries as f64 * per_q; + (eps_pg, eps_query) + } + + /// OOD binding bits this level is expected to deliver. + /// See `paper_ood_bits`. + pub fn paper_predicted_ood_bits(&self) -> f64 { + let mu = self.log_msg_cols + self.log_num_interleaved; + paper_ood_bits(self.log_inv_rate, self.log_msg_cols, self.eta, mu, self.ood_samples) + } +} + +impl LigeritoSecurityConfig { + /// Validate that the config is internally consistent and matches the + /// declared analysis. Returns the first violation found, if any. + pub fn validate(&self) -> Result<(), String> { + if self.log_n + crate::LOG_PACKING != self.m { + return Err(format!( + "log_n ({}) + LOG_PACKING ({}) != m ({})", + self.log_n, + crate::LOG_PACKING, + self.m + )); + } + + // Level shape: initial_k + Σ k (L1+) + yr_log_n = log_n. + let levels_level_k_sum: usize = self.levels.iter().skip(1).map(|lv| lv.k).sum(); + let yr_log_n = self.final_block.yr_log_n; + if self.initial_k + levels_level_k_sum + yr_log_n != self.log_n { + return Err(format!( + "shape mismatch: initial_k ({}) + Σ k ({}) + yr_log_n ({}) = {} ≠ log_n ({})", + self.initial_k, + levels_level_k_sum, + yr_log_n, + self.initial_k + levels_level_k_sum + yr_log_n, + self.log_n, + )); + } + + // L0 must have k = initial_k and log_num_interleaved = initial_k. + let l0 = self.levels.first().ok_or_else(|| "empty levels".to_string())?; + if l0.k != self.initial_k { + return Err(format!("L0.k ({}) must equal initial_k ({})", l0.k, self.initial_k)); + } + if l0.log_num_interleaved != self.initial_k { + return Err(format!( + "L0.log_num_interleaved ({}) must equal initial_k ({})", + l0.log_num_interleaved, self.initial_k + )); + } + + // Per-level checks. + let mut dim_in = self.log_n; + for (i, lv) in self.levels.iter().enumerate() { + if lv.log_inv_rate == 0 { + return Err(format!("L{i}: log_inv_rate=0 gives a rate-one code")); + } + if lv.log_msg_cols == 0 { + return Err(format!("L{i}: log_msg_cols must be positive")); + } + + // Shape: log_msg_cols + log_num_interleaved = dim_in. + if lv.log_msg_cols + lv.log_num_interleaved != dim_in { + return Err(format!( + "L{i}: log_msg_cols ({}) + log_num_interleaved ({}) ≠ input dim ({dim_in})", + lv.log_msg_cols, lv.log_num_interleaved + )); + } + + // Folding `lv.k` variables changes the next level's total RS + // domain logarithm from `dim_in + rate_i` to + // `dim_in - lv.k + rate_{i+1}`. Pin that difference to the public + // initial reduction and to one bit at every later transition. + if let Some(next) = self.levels.get(i + 1) { + let domain_reduction = if i == 0 { + RS_DOMAIN_INITIAL_REDUCTION_FACTOR + } else { + RS_DOMAIN_SUBSEQUENT_REDUCTION_FACTOR + }; + let expected_next_rate = lv + .log_inv_rate + .checked_add(lv.k) + .and_then(|r| r.checked_sub(domain_reduction)) + .ok_or_else(|| format!("L{i}: invalid RS domain reduction {domain_reduction}"))?; + if next.log_inv_rate != expected_next_rate { + return Err(format!( + "L{}: log_inv_rate ({}) does not reduce the preceding RS domain by {} bit(s); expected {}", + i + 1, + next.log_inv_rate, + domain_reduction, + expected_next_rate, + )); + } + } + + // eta within the Johnson range for this level's (reduced) rate. + let max_eta = 1.0 - reduced_rate(lv.log_inv_rate, lv.log_msg_cols).sqrt(); + if !lv.eta.is_finite() || lv.eta <= 0.0 || lv.eta >= max_eta { + return Err(format!( + "L{i}: Johnson eta must be finite and in (0, {max_eta}), got {}", + lv.eta + )); + } + + // OOD samples: every level past L0 needs explicit samples, while + // L0 is bound by the opening's own post-commit evaluation claim. + if i == 0 && lv.ood_samples != 0 { + return Err(format!( + "L0: ood_samples={} but L0 is bound by the opening's \ + own evaluation claim (must be 0)", + lv.ood_samples + )); + } + if i > 0 && lv.ood_samples == 0 { + return Err(format!( + "L{i}: ood_samples ≥ 1 required past L0 (the query \ + counts assume single-codeword binding)" + )); + } + + // OOD diagnostic matches the formula and clears the target. + let declared = lv.expected_eps_ood_bits; + if !declared.is_finite() { + return Err(format!("L{i}: expected_eps_ood_bits must be finite, got {declared}")); + } + let ood_pred = lv.paper_predicted_ood_bits(); + if (declared - ood_pred).abs() > PAPER_COMPAT_TOL_BITS { + return Err(format!( + "L{i}: expected_eps_ood_bits ({declared:.2}) doesn't \ + match prediction ({ood_pred:.2}); tolerance ±{:.2} bits.", + PAPER_COMPAT_TOL_BITS + )); + } + if ood_pred + 1e-12 < lv.target_security_bits as f64 { + return Err(format!( + "L{i}: OOD binding ({ood_pred:.2} bits) < target ({})", + lv.target_security_bits + )); + } + + // Paper-compatibility: the declared expected_*_bits must agree + // with what the regime's formula predicts (within tolerance). + // Asserts the config was actually derived from the paper, not + // hand-waved into compliance. + let (pg_pred, q_pred) = lv.paper_predicted_bits(); + if !lv.expected_eps_pg_bits.is_finite() || !lv.expected_eps_query_bits.is_finite() { + return Err(format!("L{i}: expected soundness diagnostics must be finite")); + } + if (lv.expected_eps_pg_bits - pg_pred).abs() > PAPER_COMPAT_TOL_BITS { + return Err(format!( + "L{i}: expected_eps_pg_bits ({:.2}) doesn't match \ + {analysis} prediction ({:.2}); tolerance ±{:.2} bits. \ + Re-derive Q, eta, or grinding so the declared diagnostic \ + matches the formula.", + lv.expected_eps_pg_bits, + pg_pred, + PAPER_COMPAT_TOL_BITS, + analysis = self.analysis_version, + )); + } + if (lv.expected_eps_query_bits - q_pred).abs() > PAPER_COMPAT_TOL_BITS { + return Err(format!( + "L{i}: expected_eps_query_bits ({:.2}) doesn't match \ + {analysis} prediction ({:.2}); tolerance ±{:.2} bits.", + lv.expected_eps_query_bits, + q_pred, + PAPER_COMPAT_TOL_BITS, + analysis = self.analysis_version, + )); + } + + // Security: queries cover the gap left by grinding. + if lv.target_security_bits > lv.grinding_bits + && q_pred + 1e-12 < (lv.target_security_bits - lv.grinding_bits) as f64 + { + return Err(format!( + "L{i}: query soundness ({q_pred:.2} bits) < target ({}) - grinding ({}) = {}", + lv.target_security_bits, + lv.grinding_bits, + lv.target_security_bits - lv.grinding_bits + )); + } + + // Per-application proximity gap + fold-challenge grinding must + // reach target. (The pg bad event lives on the fold challenges, + // so only the fold grind — done before each fold challenge — + // boosts it; the query-phase grind does not.) + if pg_pred + lv.fold_grinding_bits as f64 + 1e-12 < lv.target_security_bits as f64 { + return Err(format!( + "L{i}: proximity-gap soundness ({pg_pred:.2} bits) + fold_grinding ({}) < target ({})", + lv.fold_grinding_bits, lv.target_security_bits + )); + } + + // The largest list-unioned algebraic identity test (currently the + // composed ring-switch batching map) is not grindable and must + // clear the target. + let algebraic = johnson_algebraic_bits(lv); + if algebraic + 1e-12 < lv.target_security_bits as f64 { + return Err(format!( + "L{i}: list-unioned algebraic soundness ({algebraic:.2} bits) < target ({})", + lv.target_security_bits + )); + } + + if lv.target_security_bits < self.target_security_bits { + return Err(format!( + "L{i}: target_security_bits ({}) < global target ({})", + lv.target_security_bits, self.target_security_bits + )); + } + + // Advance dim_in for next level: subtract k (the folds at this level). + dim_in -= lv.k; + } + + if dim_in != yr_log_n { + return Err(format!( + "after consuming all levels, dim_in ({dim_in}) ≠ yr_log_n ({yr_log_n})" + )); + } + + // Round-by-round soundness (misc/pcs.tex, Thm `thm:rbr`): each + // verifier-challenge transition is checked against + // `target_security_bits` in the per-level loop above, so the + // Fiat--Shamir error per random-oracle query is their MAX; ordinary + // interactive soundness may additionally union-bound over transitions. + Ok(()) + } + + /// Derive the production security config at witness size `m`: Johnson + /// list decoding with OOD binding, rate `2^-LOG_INV_RATE_0`, and + /// [`SECURITY_BITS`] bits per round under + /// **round-by-round soundness** — every verifier-challenge error term (pg + /// + fold grinding, query + query grinding, OOD, and algebraic checks) + /// clears the target individually. + pub fn derive_config(m: usize) -> Result { + Self::derive_config_with_log_inv_rate(m, LOG_INV_RATE_0) + } + + /// Derive a configuration for an explicit L0 rate `2^-log_inv_rate`. + /// This side-effect-free entry point is used by parameter tooling and + /// tests and production callers that accept a transcript-bound rate. + pub fn derive_config_with_log_inv_rate(m: usize, log_inv_rate: usize) -> Result { + validate_log_inv_rate(log_inv_rate)?; + let target_bits = SECURITY_BITS; + let query_grind: usize = QUERY_GRINDING_BITS; + let log_n = m + .checked_sub(crate::LOG_PACKING) + .ok_or_else(|| format!("m ({m}) < LOG_PACKING ({})", crate::LOG_PACKING))?; + let initial_k = INITIAL_FOLDING_FACTOR; + + // The ladder geometry is independent of eta. Exact block-length + // feasibility is checked below by the same per-level optimizer that + // supplies the production eta and query count. + let shape = derive_ladder_shape(log_n, initial_k, log_inv_rate)?; + let n_levels = shape.log_inv_rates.len(); + + // Round-by-round target: every verifier-challenge error term (pg, + // query, OOD, and algebraic checks) must individually clear + // `target_bits`. We do not add a whole-transcript union-bound margin: + // this configuration targets 128-bit RBR soundness, as required by the + // Fiat--Shamir analysis, rather than 128-bit interactive soundness after + // summing every transition probability. + let mut levels = Vec::with_capacity(n_levels); + for i in 0..n_levels { + let rate = shape.log_inv_rates[i]; + let cols = shape.log_msg_cols[i]; + let ilv = shape.log_num_interleaved[i]; + let optimized = optimize_johnson_level(i, rate, cols, ilv, target_bits, query_grind)?; + + levels.push(LigeritoLevelConfig { + log_inv_rate: rate, + log_msg_cols: cols, + log_num_interleaved: ilv, + k: shape.k_levels[i], + regime: SoundnessRegime::JohnsonOod, + eta: optimized.eta, + queries: optimized.queries, + grinding_bits: query_grind, + fold_grinding_bits: 0, + ood_samples: optimized.ood_samples, + target_security_bits: target_bits, + expected_eps_pg_bits: round1(optimized.eps_pg), + expected_eps_query_bits: round1(optimized.eps_query), + expected_eps_ood_bits: round1(optimized.eps_ood), + }); + } + + let analysis_version = "bchks25_thm_4_6_exact_reduced_rate_row_union_optimized_eta"; + let cfg = Self { + m, + log_n, + initial_k, + target_security_bits: target_bits, + analysis_version: analysis_version.into(), + field: "f192".into(), + hash: "blake3".into(), + grinding_step: GrindingStep::PostCommitPreQueries, + levels, + final_block: FinalBlockConfig { + yr_log_n: shape.yr_log_n, + }, + }; + cfg.validate()?; + Ok(cfg) + } + + /// Build a `(ProverConfig, VerifierConfig)` pair from this security config. + /// Drops the security-only fields (eta, queries, grinding, expected_*) but + /// preserves the level shape so the existing prover/verifier code path + /// works unchanged. + pub fn to_prover_verifier_configs(&self) -> Result<(ProverConfig, VerifierConfig), String> { + self.validate()?; + let log_inv_rates: Vec = self.levels.iter().map(|lv| lv.log_inv_rate).collect(); + let level_ks: Vec = self.levels.iter().skip(1).map(|lv| lv.k).collect(); + let level_log_msg_cols: Vec = self.levels.iter().skip(1).map(|lv| lv.log_msg_cols).collect(); + let queries: Vec = self.levels.iter().map(|lv| lv.queries).collect(); + let grinding_bits: Vec = self.levels.iter().map(|lv| lv.grinding_bits).collect(); + let fold_grinding_bits: Vec = self.levels.iter().map(|lv| lv.fold_grinding_bits).collect(); + let ood_samples: Vec = self.levels.iter().map(|lv| lv.ood_samples).collect(); + let prover = ProverConfig { + log_inv_rates: log_inv_rates.clone(), + level_steps: level_ks.len(), + initial_log_msg_cols: self.levels[0].log_msg_cols, + initial_log_num_interleaved: self.initial_k, + initial_k: self.initial_k, + level_log_msg_cols: level_log_msg_cols.clone(), + level_ks: level_ks.clone(), + queries: queries.clone(), + grinding_bits: grinding_bits.clone(), + fold_grinding_bits: fold_grinding_bits.clone(), + ood_samples: ood_samples.clone(), + }; + let verifier = VerifierConfig { + log_inv_rates: log_inv_rates.clone(), + level_steps: level_ks.len(), + initial_log_msg_cols: self.levels[0].log_msg_cols, + initial_log_num_interleaved: self.initial_k, + initial_k: self.initial_k, + level_log_msg_cols, + level_ks, + queries, + grinding_bits, + fold_grinding_bits, + ood_samples, + }; + Ok((prover, verifier)) + } +} + +/// `ceil(log2(n))`, used to size per-query batching challenges. +#[inline] +pub fn log2_ceil(n: usize) -> usize { + if n <= 1 { 0 } else { (n - 1).ilog2() as usize + 1 } +} + +#[cfg(test)] +mod tests { + use super::*; + use primitives::pretty_integer; + + #[test] + fn johnson_bound_uses_theorem_parameter_and_reduced_rate() { + // BCHKS25 Thm 4.6 (list correlated agreement) uses + // m = ceil(sqrt(rho) / eta). The factor-two-smaller ceil(sqrt(rho) / (2 eta)) + // belongs to the plain, non-list Thm 1.5; Flock's Thm 8 quotes Thm 4.6 + // with that non-list parameter, which would overstate eps_pg by ~5 bits. + assert_eq!(johnson_m_param(1, 16, 0.02), 36.0); + + // A message of dimension 16 has maximum degree 15, so the theorem's + // reduced rate at block length 512 is 15/512, not the nominal 1/32. + assert_eq!(reduced_rate(5, 4), 15.0 / 512.0); + } + + #[test] + fn production_profile_is_128_bit_johnson_with_query_grinding() { + let mut min_pg_bits = f64::INFINITY; + for log_inv_rate in MIN_LOG_INV_RATE..=MAX_LOG_INV_RATE { + for m in 22 + crate::LOG_PACKING..=28 + crate::LOG_PACKING { + let cfg = LigeritoSecurityConfig::derive_config_with_log_inv_rate(m, log_inv_rate).unwrap(); + assert_eq!(cfg.target_security_bits, 128); + assert_eq!(cfg.levels[0].log_inv_rate, log_inv_rate); + assert_eq!(cfg.levels[0].ood_samples, 0); + for (i, level) in cfg.levels.iter().enumerate() { + let (pg_bits, query_bits) = level.paper_predicted_bits(); + let ood_bits = level.paper_predicted_ood_bits(); + let algebraic_bits = johnson_algebraic_bits(level); + min_pg_bits = min_pg_bits.min(pg_bits); + assert_eq!(level.grinding_bits, QUERY_GRINDING_BITS); + assert_eq!(level.fold_grinding_bits, 0); + assert!(query_bits + level.grinding_bits as f64 >= 128.0); + assert!(pg_bits >= 128.0); + assert!(ood_bits >= 128.0); + assert!(algebraic_bits >= 128.0); + if i > 0 { + assert_eq!(level.ood_samples, 1); + } + } + } + } + assert!( + (128.0..129.0).contains(&min_pg_bits), + "eta search should use, but not exceed, the one-bit PG margin: {min_pg_bits}" + ); + } + + #[test] + fn optimized_eta_query_and_rate_profile_is_stable() { + let cfg = LigeritoSecurityConfig::derive_config_with_log_inv_rate(22 + crate::LOG_PACKING, 1).unwrap(); + assert_eq!( + cfg.levels.iter().map(|level| level.log_inv_rate).collect::>(), + [1, 4, 6, 8, 10] + ); + assert_eq!( + cfg.levels.iter().map(|level| level.queries).collect::>(), + [225, 56, 38, 28, 23] + ); + assert_eq!( + cfg.levels + .iter() + .map(|level| { johnson_m_param(level.log_inv_rate, level.log_msg_cols, level.eta,) as usize }) + .collect::>(), + [216, 80, 18, 35, 7] + ); + } + + #[test] + fn recursive_rs_domain_reduction_schedule_is_stable() { + for starting_rate in MIN_LOG_INV_RATE..=MAX_LOG_INV_RATE { + let cfg = LigeritoSecurityConfig::derive_config_with_log_inv_rate(27 + crate::LOG_PACKING, starting_rate) + .unwrap(); + let mut dim_in = cfg.log_n; + let mut previous_domain_log = dim_in + cfg.levels[0].log_inv_rate; + for (i, level) in cfg.levels.iter().enumerate() { + dim_in -= level.k; + if let Some(next) = cfg.levels.get(i + 1) { + let next_domain_log = dim_in + next.log_inv_rate; + let expected_reduction = if i == 0 { RS_DOMAIN_INITIAL_REDUCTION_FACTOR } else { 1 }; + assert_eq!( + previous_domain_log - next_domain_log, + expected_reduction, + "transition L{i} -> L{} at starting rate 1/{}", + i + 1, + 1usize << starting_rate, + ); + previous_domain_log = next_domain_log; + } + } + } + } + + /// Parameter-report helper: + /// `LIGERITO_LOG_INV_RATE=2 LIGERITO_NUM_VARS=22 cargo test --release -p pcs print_ligerito_query_counts -- --ignored --nocapture` + #[test] + #[ignore = "manual parameter report; configure it through environment variables"] + fn print_ligerito_query_counts() { + let env_usize = |name: &str| { + std::env::var(name) + .unwrap_or_else(|_| panic!("missing {name}")) + .parse::() + .unwrap_or_else(|_| panic!("{name} must be a non-negative integer")) + }; + let log_inv_rate = env_usize("LIGERITO_LOG_INV_RATE"); + let num_vars = env_usize("LIGERITO_NUM_VARS"); + let cfg = LigeritoSecurityConfig::derive_config_with_log_inv_rate(num_vars + crate::LOG_PACKING, log_inv_rate) + .unwrap(); + + println!( + "num_vars={}, rate=1/{}", + pretty_integer(num_vars), + pretty_integer(1usize << log_inv_rate) + ); + for (level, params) in cfg.levels.iter().enumerate() { + let eta = params.eta; + println!( + "L{}: rate=1/{}, queries={}, eta={eta:.12e}, m={}", + pretty_integer(level), + pretty_integer(1usize << params.log_inv_rate), + pretty_integer(params.queries), + pretty_integer(johnson_m_param(params.log_inv_rate, params.log_msg_cols, eta) as usize), + ); + } + } +} diff --git a/crates/pcs/src/merkle.rs b/crates/pcs/src/merkle.rs index 8705b998..84d0ea72 100644 --- a/crates/pcs/src/merkle.rs +++ b/crates/pcs/src/merkle.rs @@ -1,4 +1,4 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. //! Binary Merkle tree with BLAKE3, SIMD-batching independent hashes across //! leaves and internal levels through BLAKE3's multi-input backend. //! @@ -22,7 +22,7 @@ mod blake3_neon8; use primitives::epool::{SyncPtr, run_hetero_chunks}; -use primitives::{field::F128, pretty_integer}; +use primitives::{field::F192, pretty_integer}; use rayon::prelude::*; pub type Hash = [u8; 32]; @@ -49,20 +49,25 @@ const HASH_GROUP: usize = 1024; /// Encode a Merkle hash as the two little-endian field words used by transcripts. #[inline] -pub fn hash_to_scalars(hash: &Hash) -> [F128; 2] { - [ - F128::from_le_bytes(hash[..16].try_into().unwrap()), - F128::from_le_bytes(hash[16..].try_into().unwrap()), - ] +pub fn hash_to_scalars(hash: &Hash) -> [F192; 2] { + let w = |o: usize| u64::from_le_bytes(hash[o..o + 8].try_into().unwrap()); + [F192::new(w(0), w(8), w(16)), F192::new(w(24), 0, 0)] } /// Decode the two field words used by transcripts back into a Merkle hash. #[inline] -pub fn scalars_to_hash(scalars: &[F128]) -> Hash { +pub fn scalars_to_hash(scalars: &[F192]) -> Hash { assert_eq!(scalars.len(), 2, "a Merkle hash is exactly two field words"); let mut hash = [0u8; 32]; - hash[..16].copy_from_slice(&scalars[0].to_le_bytes()); - hash[16..].copy_from_slice(&scalars[1].to_le_bytes()); + hash[0..8].copy_from_slice(&scalars[0].c0.to_le_bytes()); + hash[8..16].copy_from_slice(&scalars[0].c1.to_le_bytes()); + assert_eq!( + (scalars[1].c1, scalars[1].c2), + (0, 0), + "packed Merkle hash tail must be K-valued" + ); + hash[16..24].copy_from_slice(&scalars[0].c2.to_le_bytes()); + hash[24..32].copy_from_slice(&scalars[1].c0.to_le_bytes()); hash } @@ -83,7 +88,7 @@ pub fn hash_leaf(data: &[u8]) -> Hash { *blake3::hash(data).as_bytes() } -/// Hash a pair of children into a parent node (64 B → 32 B): one [`compress`], +/// Hash a pair of children into a parent node (64 B → 32 B): one compression, /// which is already exactly the VM opcode. #[inline] pub fn hash_pair(left: &Hash, right: &Hash) -> Hash { @@ -94,24 +99,39 @@ pub fn hash_pair(left: &Hash, right: &Hash) -> Hash { /// inputs. A whole-block input no longer than 1024 bytes is exactly one chunk; /// counter zero, CHUNK_START on its first block, and CHUNK_END|ROOT on its last /// reproduce `blake3::hash` byte-for-byte. -fn hash_many_oneshot( +#[cfg(test)] +fn hash_many_oneshot(platform: blake3::platform::Platform, data: &[u8], out: &mut [Hash]) { + const { + assert!(N > 0 && N.is_multiple_of(64) && N <= 1024); + } + debug_assert_eq!(data.len(), out.len() * N); + let inputs: Vec<&[u8; N]> = data.chunks_exact(N).map(|input| input.try_into().unwrap()).collect(); + // Hash is [u8; 32] with no padding; expose the contiguous output storage + // expected by hash_many, which writes exactly 32 bytes per input. + let out_bytes = unsafe { core::slice::from_raw_parts_mut(out.as_mut_ptr().cast::(), out.len() * 32) }; + platform.hash_many::( + &inputs, + &B3_IV, + 0, + blake3::IncrementCounter::No, + 0, + B3_CHUNK_START, + B3_CHUNK_END | B3_ROOT, + out_bytes, + ); +} + +fn hash_many_oneshot_uninit( platform: blake3::platform::Platform, data: &[u8], - out: &mut [Hash], + out: &mut [std::mem::MaybeUninit], ) { const { - assert!(N > 0 && N % 64 == 0 && N <= 1024); + assert!(N > 0 && N.is_multiple_of(64) && N <= 1024); } debug_assert_eq!(data.len(), out.len() * N); - let inputs: Vec<&[u8; N]> = data - .chunks_exact(N) - .map(|input| input.try_into().unwrap()) - .collect(); - // Hash is [u8; 32] with no padding; expose the contiguous output storage - // expected by hash_many, which writes exactly 32 bytes per input. - let out_bytes = unsafe { - core::slice::from_raw_parts_mut(out.as_mut_ptr().cast::(), out.len() * 32) - }; + let inputs: Vec<&[u8; N]> = data.chunks_exact(N).map(|input| input.try_into().unwrap()).collect(); + let out_bytes = unsafe { core::slice::from_raw_parts_mut(out.as_mut_ptr().cast::(), out.len() * 32) }; platform.hash_many::( &inputs, &B3_IV, @@ -124,23 +144,21 @@ fn hash_many_oneshot( ); } -/// Hash equal-size leaves in parallel with standard BLAKE3, using cross-leaf -/// SIMD for the PCS's whole-block, single-chunk geometries. -fn hash_leaves_batched( +fn hash_leaves_batched_uninit( platform: blake3::platform::Platform, data: &[u8], leaf_size: usize, - out: &mut [Hash], + out: &mut [std::mem::MaybeUninit], ) { fn batched( platform: blake3::platform::Platform, data: &[u8], - out: &mut [Hash], + out: &mut [std::mem::MaybeUninit], ) { - // Leaf hashing is the prover's purest embarrassingly parallel phase: + // Leaf hashing is the purest embarrassingly parallel phase here: // fixed-size independent groups, no cross-group dependency, one join - // at the end. That makes it the right place to spend the idle - // efficiency cores — see `primitives::epool` for why the shared + // at the end. That makes it the right place to spend the otherwise + // idle efficiency cores — see `primitives::epool` for why a shared // atomic chunk queue is safe here and widening the main pool is not. let n_groups = out.len().div_ceil(HASH_GROUP); let out_base = SyncPtr(out.as_mut_ptr()); @@ -152,67 +170,53 @@ fn hash_leaves_batched( // group `g` writes only `out[lo .. lo+len]`, so the mutable // ranges are pairwise disjoint and in bounds. let outputs = unsafe { core::slice::from_raw_parts_mut(out_base.ptr().add(lo), len) }; - hash_many_oneshot::(platform, &data[lo * N..(lo + len) * N], outputs); + let inputs = &data[lo * N..(lo + len) * N]; + // Eight-wide NEON for the complete groups; upstream `hash_many` + // for whatever remains (a chunk's leaf count need not be a + // multiple of 8). + #[cfg(target_arch = "aarch64")] + { + let done = blake3_neon8::hash_complete_groups::(inputs, outputs); + if done < len { + hash_many_oneshot_uninit::(platform, &inputs[done * N..], &mut outputs[done..]); + } + } + #[cfg(not(target_arch = "aarch64"))] + hash_many_oneshot_uninit::(platform, inputs, outputs); }); } match leaf_size { - // 1 KiB leaves (the PCS's L0 geometry at `log_batch_size = 6`) go - // through the eight-leaf NEON kernel, which keeps two independent - // 4-lane BLAKE3 states in flight instead of upstream's one. - #[cfg(target_arch = "aarch64")] - 1024 => batched_1024(data, out), 64 => batched::<64>(platform, data, out), 128 => batched::<128>(platform, data, out), 256 => batched::<256>(platform, data, out), 512 => batched::<512>(platform, data, out), - #[cfg(not(target_arch = "aarch64"))] + // The Ligerito recursion levels commit F192 rows, so their leaves are + // `num_interleaved * 24` bytes — a multiple of 64 but not a power of + // two, which used to miss every batched arm and fall through to the + // one-leaf-at-a-time path with no cross-leaf SIMD at all. + 192 => batched::<192>(platform, data, out), + 384 => batched::<384>(platform, data, out), + 768 => batched::<768>(platform, data, out), 1024 => batched::<1024>(platform, data, out), _ => out .par_iter_mut() .zip(data.par_chunks(leaf_size)) - .for_each(|(slot, leaf)| *slot = hash_leaf(leaf)), + .for_each(|(slot, leaf)| { + slot.write(hash_leaf(leaf)); + }), } } -/// 1 KiB leaves: eight-wide NEON for the complete groups, upstream `hash_many` -/// for whatever remains (a chunk's leaf count need not be a multiple of 8). -#[cfg(target_arch = "aarch64")] -fn batched_1024(data: &[u8], out: &mut [Hash]) { - let n_groups = out.len().div_ceil(HASH_GROUP); - let out_base = SyncPtr(out.as_mut_ptr()); - let out_len = out.len(); - run_hetero_chunks(n_groups, |g| { - let lo = g * HASH_GROUP; - let len = HASH_GROUP.min(out_len - lo); - // SAFETY: as in `hash_leaves_batched` — one worker per `g`, and group - // `g` writes only `out[lo .. lo+len]`. - let outputs = unsafe { core::slice::from_raw_parts_mut(out_base.ptr().add(lo), len) }; - let inputs = &data[lo * 1024..(lo + len) * 1024]; - let done = blake3_neon8::hash_complete_groups_1024(inputs, outputs); - if done < len { - hash_many_oneshot::<1024>( - blake3::platform::Platform::detect(), - &inputs[done * 1024..], - &mut outputs[done..], - ); - } - }); -} - -/// Hash one internal level. Child hashes are already contiguous, so each pair -/// is a zero-copy 64-byte input to the same SIMD-batched one-shot primitive. -fn hash_pairs_level( +fn hash_pairs_level_uninit( platform: blake3::platform::Platform, read: &[Hash], - write: &mut [Hash], + write: &mut [std::mem::MaybeUninit], ) { debug_assert_eq!(read.len(), 2 * write.len()); - let read_bytes = unsafe { - core::slice::from_raw_parts(read.as_ptr().cast::(), read.len() * 32) - }; + let read_bytes = unsafe { core::slice::from_raw_parts(read.as_ptr().cast::(), read.len() * 32) }; const SERIAL_LEVEL_NODES: usize = 1024; if write.len() <= SERIAL_LEVEL_NODES { - hash_many_oneshot::<64>(platform, read_bytes, write); + hash_many_oneshot_uninit::<64>(platform, read_bytes, write); } else { let n_groups = write.len().div_ceil(HASH_GROUP); let write_base = SyncPtr(write.as_mut_ptr()); @@ -220,10 +224,10 @@ fn hash_pairs_level( run_hetero_chunks(n_groups, |g| { let lo = g * HASH_GROUP; let len = HASH_GROUP.min(write_len - lo); - // SAFETY: as in `hash_leaves_batched` — one worker per `g`, and - // group `g` writes only `write[lo .. lo+len]`. + // SAFETY: as in `batched` — one worker per `g`, and group `g` + // writes only `write[lo .. lo+len]`. let outputs = unsafe { core::slice::from_raw_parts_mut(write_base.ptr().add(lo), len) }; - hash_many_oneshot::<64>(platform, &read_bytes[lo * 64..(lo + len) * 64], outputs); + hash_many_oneshot_uninit::<64>(platform, &read_bytes[lo * 64..(lo + len) * 64], outputs); }); } } @@ -251,14 +255,11 @@ pub fn merkle_tree(data: &[u8], num_leaves: usize) -> Vec { let leaf_size = data.len() / num_leaves; let total_nodes = 2 * num_leaves - 1; - // Uninit alloc — every node is written exactly once before being read: - // leaves at step 1, then each internal level reads the level below (which - // was just written) and writes itself. - let mut tree: Vec = primitives::alloc_uninit_vec(total_nodes); + let mut tree = primitives::alloc_uninit(total_nodes); let platform = blake3::platform::Platform::detect(); // 1. Leaves — independent standard BLAKE3 hashes. - hash_leaves_batched(platform, data, leaf_size, &mut tree[..num_leaves]); + hash_leaves_batched_uninit(platform, data, leaf_size, &mut tree[..num_leaves]); // 2. Internal levels — parallel within a level, sequential across levels. // Small upper levels can't fill the cores, so a rayon dispatch per level @@ -268,18 +269,16 @@ pub fn merkle_tree(data: &[u8], num_leaves: usize) -> Vec { let mut read_len = num_leaves; while read_len > 1 { let next_len = read_len >> 1; - // Split the buffer at the end of the current level so we get two - // non-overlapping mutable slices: `read` (input) and `write` (output). - let (read, rest) = tree[read_start..].split_at_mut(read_len); - let write = &mut rest[..next_len]; - - hash_pairs_level(platform, read, write); + let read = unsafe { std::slice::from_raw_parts(tree.as_ptr().add(read_start).cast::(), read_len) }; + let write_start = read_start + read_len; + hash_pairs_level_uninit(platform, read, &mut tree[write_start..write_start + next_len]); read_start += read_len; read_len = next_len; } - tree + // SAFETY: leaves and each successive internal level initialize the full tree. + unsafe { primitives::assume_init(tree) } } // --------------------------------------------------------------------------- @@ -293,11 +292,7 @@ pub fn verify_merkle_proof(root: &Hash, leaf_hash: &Hash, index: usize, proof: & let mut idx = index; for sibling in proof { // If idx is even, our node is the LEFT child; sibling is on the RIGHT. - let (left, right) = if idx & 1 == 0 { - (acc, *sibling) - } else { - (*sibling, acc) - }; + let (left, right) = if idx & 1 == 0 { (acc, *sibling) } else { (*sibling, acc) }; acc = hash_pair(&left, &right); idx >>= 1; } @@ -365,6 +360,86 @@ pub fn merkle_multi_proof(tree: &[Hash], num_leaves: usize, positions: &[usize]) proof } +/// Verify a Merkle multi-proof produced by [`merkle_multi_proof`]. +/// +/// `sorted_unique_positions` and `leaf_hashes` must be aligned and sorted: +/// `leaf_hashes[i]` is the hash of the leaf at `sorted_unique_positions[i]`, +/// and the position list is strictly ascending. Returns true iff the +/// reconstructed root equals `root` and the proof is consumed exactly. +pub fn verify_merkle_multi_proof( + root: &Hash, + num_leaves: usize, + sorted_unique_positions: &[usize], + leaf_hashes: &[Hash], + proof: &[Hash], +) -> bool { + if !num_leaves.is_power_of_two() || num_leaves == 0 { + return false; + } + if sorted_unique_positions.len() != leaf_hashes.len() { + return false; + } + if sorted_unique_positions.is_empty() { + // Vacuous; nothing to verify. Treat as "ok" iff the proof is empty. + return proof.is_empty(); + } + // Verify the position list is sorted strictly ascending + in range. + for (i, &p) in sorted_unique_positions.iter().enumerate() { + if p >= num_leaves { + return false; + } + if i > 0 && sorted_unique_positions[i - 1] >= p { + return false; + } + } + // Edge case: 1-leaf tree, no proof needed. + if num_leaves == 1 { + return proof.is_empty() && leaf_hashes[0] == *root; + } + + let mut active: Vec<(usize, Hash)> = sorted_unique_positions + .iter() + .copied() + .zip(leaf_hashes.iter().copied()) + .collect(); + let mut proof_iter = proof.iter().copied(); + let mut level_len = num_leaves; + + while level_len > 1 { + let mut next = Vec::with_capacity(active.len()); + let mut i = 0; + while i < active.len() { + let (p, h) = active[i]; + let sib_active = i + 1 < active.len() && active[i + 1].0 == (p ^ 1); + let (left, right) = if sib_active { + let (_, h_sib) = active[i + 1]; + // Sorted strictly ascending → active[i+1].0 = p + 1 (= p ^ 1 + // since p is even when p ^ 1 = p + 1). So p is LEFT child. + debug_assert_eq!(p & 1, 0); + i += 2; + (h, h_sib) + } else { + let sib = match proof_iter.next() { + Some(s) => s, + None => return false, + }; + i += 1; + if p & 1 == 0 { (h, sib) } else { (sib, h) } + }; + next.push((p >> 1, hash_pair(&left, &right))); + } + active = next; + level_len >>= 1; + } + + // After the loop, `active` has exactly one element (the root). Reject + // any leftover proof bytes. + if proof_iter.next().is_some() { + return false; + } + active.len() == 1 && active[0].1 == *root +} + /// Reconstruct the full per-query Merkle paths from a *pruned* (octopus) proof — /// the inverse of [`merkle_multi_proof`]. Given the ORIGINAL `queries` (unsorted, /// possibly duplicate), the distinct leaves' hashes (`leaf_hashes`, aligned with @@ -429,7 +504,10 @@ pub fn restore_multi_proof( .map(|lvl| { let sib = (leaf >> lvl) ^ 1; let level = &known[lvl]; - level.binary_search_by_key(&sib, |&(j, _)| j).ok().map(|pos| level[pos].1) + level + .binary_search_by_key(&sib, |&(j, _)| j) + .ok() + .map(|pos| level[pos].1) }) .collect::>>() }) @@ -463,7 +541,10 @@ mod prune_tests { let mut sorted = queries.to_vec(); sorted.sort_unstable(); sorted.dedup(); // [1, 3, 5] - let leaf_hashes: Vec = sorted.iter().map(|&q| hash_leaf(&data[q * leaf_size..(q + 1) * leaf_size])).collect(); + let leaf_hashes: Vec = sorted + .iter() + .map(|&q| hash_leaf(&data[q * leaf_size..(q + 1) * leaf_size])) + .collect(); let pruned = merkle_multi_proof(&tree, num_leaves, &sorted); let flat = restore_multi_proof(num_leaves, &queries, &leaf_hashes, &pruned).expect("restore"); @@ -471,7 +552,10 @@ mod prune_tests { for (i, &q) in queries.iter().enumerate() { let leaf = hash_leaf(&data[q * leaf_size..(q + 1) * leaf_size]); let path = &flat[i * height..(i + 1) * height]; - assert!(verify_merkle_proof(&root, &leaf, q, path), "restored path for query {q} (pos {i}) must verify"); + assert!( + verify_merkle_proof(&root, &leaf, q, path), + "restored path for query {q} (pos {i}) must verify" + ); } // An extra (unconsumed) sibling is a malformed proof. @@ -483,6 +567,62 @@ mod prune_tests { #[cfg(test)] mod vmhash_batch_tests { + + /// The eight-wide NEON leaf kernel must reproduce `blake3::hash` for every + /// leaf, at every leaf size the dispatch uses, including batches that are + /// not a multiple of eight. + #[cfg(target_arch = "aarch64")] + #[test] + fn neon8_leaves_match_standard_blake3() { + fn check() { + for n_leaves in [1usize, 7, 8, 9, 16, 37, 64] { + let data: Vec = (0..n_leaves * N).map(|i| (i * 31 + 7) as u8).collect(); + let mut out: Vec> = + (0..n_leaves).map(|_| std::mem::MaybeUninit::uninit()).collect(); + let done = super::blake3_neon8::hash_complete_groups::(&data, &mut out); + assert_eq!(done, n_leaves / 8 * 8); + for i in 0..done { + // SAFETY: the kernel initialized the first `done` slots. + let got = unsafe { out[i].assume_init() }; + assert_eq!( + got, + *blake3::hash(&data[i * N..(i + 1) * N]).as_bytes(), + "leaf {i} of {n_leaves} at N={N}" + ); + } + } + } + check::<64>(); + check::<128>(); + check::<256>(); + check::<192>(); + check::<384>(); + check::<512>(); + check::<768>(); + check::<1024>(); + } + + /// End-to-end through the dispatcher at the L0 leaf size this branch + /// actually commits with (64 lanes x 8-byte F64 = 512 bytes). + #[test] + fn leaf_dispatch_matches_standard_blake3() { + for leaf in [64usize, 128, 192, 256, 384, 512, 768, 1024] { + let n_leaves = 37usize; + let data: Vec = (0..n_leaves * leaf).map(|i| (i * 17 + 3) as u8).collect(); + let mut out: Vec> = + (0..n_leaves).map(|_| std::mem::MaybeUninit::uninit()).collect(); + hash_leaves_batched_uninit(blake3::platform::Platform::detect(), &data, leaf, &mut out); + for i in 0..n_leaves { + // SAFETY: the dispatcher initializes every slot. + let got = unsafe { out[i].assume_init() }; + assert_eq!( + got, + *blake3::hash(&data[i * leaf..(i + 1) * leaf]).as_bytes(), + "leaf {i} at leaf_size={leaf}" + ); + } + } + } use super::*; /// The low-level multi-input invocation must exactly reproduce independent @@ -513,73 +653,32 @@ mod vmhash_batch_tests { /// Sequential (per-leaf `hash_leaf`) reference for the parallel /// [`merkle_tree`]. - /// The eight-wide NEON leaf kernel must reproduce `blake3::hash` for every - /// leaf, including when the batch is not a multiple of eight. - #[cfg(target_arch = "aarch64")] - #[test] - fn neon8_leaves_match_standard_blake3() { - for n_leaves in [1usize, 7, 8, 9, 16, 37, 64] { - let data: Vec = (0..n_leaves * 1024).map(|i| (i * 31 + 7) as u8).collect(); - let mut out = vec![[0u8; 32]; n_leaves]; - let done = blake3_neon8::hash_complete_groups_1024(&data, &mut out); - assert_eq!(done, n_leaves / 8 * 8); - for i in 0..done { - assert_eq!( - out[i], - *blake3::hash(&data[i * 1024..(i + 1) * 1024]).as_bytes(), - "leaf {i} of {n_leaves}" - ); - } - } - } - - /// End-to-end: the dispatcher's 1 KiB path must agree with per-leaf - /// `blake3::hash` for a batch size that is not a multiple of eight. - #[test] - fn leaf_dispatch_1024_matches_standard_blake3() { - let n_leaves = 37usize; - let data: Vec = (0..n_leaves * 1024).map(|i| (i * 17 + 3) as u8).collect(); - let mut out = vec![[0u8; 32]; n_leaves]; - hash_leaves_batched( - blake3::platform::Platform::detect(), - &data, - 1024, - &mut out, - ); - for i in 0..n_leaves { - assert_eq!( - out[i], - *blake3::hash(&data[i * 1024..(i + 1) * 1024]).as_bytes(), - "leaf {i}" - ); - } - } - fn merkle_tree_sequential(data: &[u8], num_leaves: usize) -> Vec { - assert!(num_leaves.is_power_of_two() && num_leaves > 0); - assert_eq!(data.len() % num_leaves, 0); + assert!(num_leaves.is_power_of_two() && num_leaves > 0); + assert_eq!(data.len() % num_leaves, 0); - let leaf_size = data.len() / num_leaves; - let total_nodes = 2 * num_leaves - 1; - let mut tree: Vec = primitives::alloc_uninit_vec(total_nodes); + let leaf_size = data.len() / num_leaves; + let total_nodes = 2 * num_leaves - 1; + let mut tree = Vec::with_capacity(total_nodes); - for (i, leaf) in data.chunks(leaf_size).enumerate() { - tree[i] = hash_leaf(leaf); - } - let mut read_start = 0usize; - let mut read_len = num_leaves; - while read_len > 1 { - let next_len = read_len >> 1; - for i in 0..next_len { - let left = tree[read_start + 2 * i]; - let right = tree[read_start + 2 * i + 1]; - tree[read_start + read_len + i] = hash_pair(&left, &right); + for (i, leaf) in data.chunks(leaf_size).enumerate() { + debug_assert_eq!(tree.len(), i); + tree.push(hash_leaf(leaf)); } - read_start += read_len; - read_len = next_len; + let mut read_start = 0usize; + let mut read_len = num_leaves; + while read_len > 1 { + let next_len = read_len >> 1; + for i in 0..next_len { + let left = tree[read_start + 2 * i]; + let right = tree[read_start + 2 * i + 1]; + tree.push(hash_pair(&left, &right)); + } + read_start += read_len; + read_len = next_len; + } + tree } - tree -} /// The parallel `merkle_tree` must be byte-identical to the per-leaf /// `merkle_tree_sequential` (which uses `hash_leaf`) — same root, same nodes — @@ -597,7 +696,9 @@ mod vmhash_batch_tests { (8192, 16), (1, 32), ] { - let data: Vec = (0..num_leaves * leaf_size).map(|i| (i.wrapping_mul(131) ^ 0x5a) as u8).collect(); + let data: Vec = (0..num_leaves * leaf_size) + .map(|i| (i.wrapping_mul(131) ^ 0x5a) as u8) + .collect(); assert_eq!( merkle_tree(&data, num_leaves), merkle_tree_sequential(&data, num_leaves), diff --git a/crates/pcs/src/merkle/blake3_neon8.rs b/crates/pcs/src/merkle/blake3_neon8.rs index 0dd449bc..7caa7d09 100644 --- a/crates/pcs/src/merkle/blake3_neon8.rs +++ b/crates/pcs/src/merkle/blake3_neon8.rs @@ -12,11 +12,12 @@ //! doubles the independent work available to the out-of-order engine at every //! step while still fitting both states plus both message blocks in registers. //! -//! The contract is fixed to the PCS's L0 Merkle leaf: eight contiguous -//! 1024-byte unkeyed chunks, counter zero, `CHUNK_START` on the first block -//! and `CHUNK_END | ROOT` on the last — i.e. byte-identical to +//! The contract is fixed to a PCS Merkle leaf: eight contiguous unkeyed +//! leaves of `LEAF` bytes, counter zero, `CHUNK_START` on the first block and +//! `CHUNK_END | ROOT` on the last — i.e. byte-identical to //! `blake3::hash(leaf)` for each of the eight leaves, exactly like -//! [`super::hash_many_oneshot`]. +//! [`super::hash_many_oneshot_uninit`]. `LEAF` must be a positive multiple of +//! 64 no greater than 1024, so that a leaf is exactly one whole chunk. //! //! Derived from the reference kernel in Layr-Labs/flock-challenge, which //! ships the same algorithm as pre-generated assembly. @@ -46,10 +47,6 @@ const MSG_SCHEDULE: [[u8; 16]; 7] = [ /// Leaves per group. Two 4-lane states. const LANES: usize = 8; -/// Bytes per leaf: one full BLAKE3 chunk. -const LEAF: usize = 1024; -/// 64-byte compression blocks per chunk. -const BLOCKS: usize = LEAF / 64; #[inline(always)] unsafe fn rot16(x: uint32x4_t) -> uint32x4_t { @@ -101,11 +98,7 @@ unsafe fn transpose4(vecs: &mut [uint32x4_t; 4]) { /// # Safety /// Each `inputs[i]` must be readable for `block_offset + 64` bytes. #[inline(always)] -unsafe fn transpose_block4( - inputs: [*const u8; 4], - block_offset: usize, - out: &mut [uint32x4_t; 16], -) { +unsafe fn transpose_block4(inputs: [*const u8; 4], block_offset: usize, out: &mut [uint32x4_t; 16]) { // SAFETY: the caller guarantees each input covers the requested block. unsafe { for quarter in 0..4 { @@ -218,11 +211,19 @@ unsafe fn store_cv4(h: &mut [uint32x4_t; 8], out: *mut u8) { } } -/// Hash as many complete groups of eight 1 KiB leaves as fit in `out`, +/// Hash as many complete groups of eight `LEAF`-byte leaves as fit in `out`, /// returning the number of leaves written. The caller handles the remainder /// through upstream `hash_many`, which also makes arbitrary chunk sizes safe /// without padding or over-read. -pub(super) fn hash_complete_groups_1024(data: &[u8], out: &mut [[u8; 32]]) -> usize { +pub(super) fn hash_complete_groups( + data: &[u8], + out: &mut [core::mem::MaybeUninit<[u8; 32]>], +) -> usize { + const { + assert!(LEAF > 0 && LEAF.is_multiple_of(64) && LEAF <= 1024); + } + // 64-byte compression blocks per leaf. + let blocks = LEAF / 64; debug_assert_eq!(data.len(), out.len() * LEAF); let groups = out.len() / LANES; if groups == 0 { @@ -241,12 +242,7 @@ pub(super) fn hash_complete_groups_1024(data: &[u8], out: &mut [[u8; 32]]) -> us h0[i] = vdupq_n_u32(IV[i]); h1[i] = h0[i]; } - let lanes_lo = [ - input, - input.add(LEAF), - input.add(2 * LEAF), - input.add(3 * LEAF), - ]; + let lanes_lo = [input, input.add(LEAF), input.add(2 * LEAF), input.add(3 * LEAF)]; let lanes_hi = [ input.add(4 * LEAF), input.add(5 * LEAF), @@ -256,7 +252,7 @@ pub(super) fn hash_complete_groups_1024(data: &[u8], out: &mut [[u8; 32]]) -> us let mut m0 = [vdupq_n_u32(0); 16]; let mut m1 = [vdupq_n_u32(0); 16]; - for block in 0..BLOCKS { + for block in 0..blocks { let block_offset = block * 64; transpose_block4(lanes_lo, block_offset, &mut m0); transpose_block4(lanes_hi, block_offset, &mut m1); @@ -264,7 +260,7 @@ pub(super) fn hash_complete_groups_1024(data: &[u8], out: &mut [[u8; 32]]) -> us // CHUNK_START on the first block; CHUNK_END | ROOT on the // last — a whole 1 KiB leaf is exactly one chunk and is its // own root, so this reproduces `blake3::hash(leaf)`. - let flags = u32::from(block == 0) | if block == BLOCKS - 1 { 2 | 8 } else { 0 }; + let flags = u32::from(block == 0) | if block == blocks - 1 { 2 | 8 } else { 0 }; let mut v0 = init_state(&h0, flags); let mut v1 = init_state(&h1, flags); diff --git a/crates/pcs/src/ntt.rs b/crates/pcs/src/ntt.rs index ed98c4b2..0dec0d7f 100644 --- a/crates/pcs/src/ntt.rs +++ b/crates/pcs/src/ntt.rs @@ -1,17 +1,15 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. //! Additive NTTs (Lin–Chung–Han novel polynomial basis). //! -//! Two transforms live here: -//! - [`additive_ntt_f128`] (re-exported as [`AdditiveNttF128`]): the -//! F_{2^128} LCH NTT used by the PCS commit and Ligerito. -//! - [`AdditiveNttGf8`] (this file) plus [`inv_table`]: a small GF(2^8) NTT +//! Two transforms live here: [`AdditiveNttF64`] for Ligerito commitments and +//! [`AdditiveNttGf8`] (this file) plus [`inv_table`] for a small GF(2^8) NTT //! used by flock's univariate-skip zerocheck round. use primitives::field::F8; -pub mod additive_ntt_f128; +pub mod additive_ntt_f64; pub mod inv_table; -pub use additive_ntt_f128::AdditiveNttF128; +pub use additive_ntt_f64::AdditiveNttF64; pub use inv_table::InvNttTableByteSingleGf8; /// Twiddle recurrence used to build the next subspace layer's evaluation points: @@ -143,11 +141,7 @@ impl AdditiveNttGf8 { } pub fn forward(&self, v: &mut [F8]) { - assert_eq!( - v.len(), - self.domain_size(), - "forward: input length must be 2^k" - ); + assert_eq!(v.len(), self.domain_size(), "forward: input length must be 2^k"); if v.len() <= 1 { return; } @@ -155,11 +149,7 @@ impl AdditiveNttGf8 { } pub fn inverse(&self, v: &mut [F8]) { - assert_eq!( - v.len(), - self.domain_size(), - "inverse: input length must be 2^k" - ); + assert_eq!(v.len(), self.domain_size(), "inverse: input length must be 2^k"); if v.len() <= 1 { return; } diff --git a/crates/pcs/src/ntt/additive_ntt_f128.rs b/crates/pcs/src/ntt/additive_ntt_f128.rs deleted file mode 100644 index 56ce066f..00000000 --- a/crates/pcs/src/ntt/additive_ntt_f128.rs +++ /dev/null @@ -1,1667 +0,0 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. -// Copyright 2024-2025 Irreducible, Inc. -// Modifications copyright 2026 Succinct Labs, Benedikt Bunz, William Wang -// SPDX-License-Identifier: Apache-2.0 OR MIT -// -// The algorithm skeleton (iterative LCH NTT, neighbors-last ordering) is -// derived from binius64's `NeighborsLastReference` -// (https://github.com/binius-zk/binius64, `crates/math/src/ntt/reference.rs`). -// The interleaved SoA layout, fused 2-layer butterfly, and parallelization -// strategy are original to Flock. - -//! Additive NTT over F_{2^128} using the LCH novel polynomial basis. -//! -//! Iterative LCH NTT skeleton derived from binius64's `NeighborsLastReference`, -//! with an interleaved SoA layout, a fused 2-layer butterfly, and rayon-based -//! parallelization added on top. The forward transform maps polynomial -//! coefficients (in the novel polynomial basis) to evaluations over an -//! F_2-affine subspace; the inverse reverses this. Used by the PCS commit and -//! by Ligerito's fold. -//! -//! ## Convention -//! -//! Given a basis `{β_0, …, β_{ℓ-1}}` of an F_2-subspace V ⊂ F_{2^128}, define -//! the subspace polynomials W_i recursively: -//! ```text -//! W_0(z) = z -//! W_i(z) = W_{i-1}(z) · (W_{i-1}(z) + W_{i-1}(β_{i-1})) (for i ≥ 1) -//! ``` -//! and the *normalized* forms `Ŵ_i(z) = W_i(z) / W_i(β_i)` so that -//! `Ŵ_i(β_i) = 1`. The "twiddle" at layer `l` and block `b` is then -//! `Ŵ_{ℓ-l-1}(z)` evaluated at the `b`-th element of the F_2-span of -//! `{β_{ℓ-l}, β_{ℓ-l+1}, …, β_{ℓ-1}}`. -//! -//! At forward-transform layer `l` (`l = 0, …, log_d − 1`): -//! - There are `2^l` blocks, each of size `2^(log_d − l)`. -//! - Within each block, pairs `(idx0, idx0 | block_size_half)` are -//! butterflied with the block's twiddle. -//! - **Pairing at layer `l`**: positions differ by `block_size_half = -//! 2^(log_d − l − 1)`. So at layer 0 pairs are far (N/2 apart), and at the -//! deepest layer pairs are adjacent (1 apart) — this is "neighbors-last." -//! -//! Ligerito's fold/transpose kernels consume `twiddle` in **reverse** layer -//! order (deepest first), at which level pairs are adjacent. - -use primitives::field::F128; - -/// Compute the normalized subspace-polynomial evaluation table. -/// -/// Returns `evals` where `evals[i] = [Ŵ_i(β_i), Ŵ_i(β_{i+1}), …, Ŵ_i(β_{ℓ-1})]`. -/// The 0-th element of each row is always `1` (by normalization). -fn generate_evals_from_subspace(basis: &[F128]) -> Vec> { - let l = basis.len(); - let mut evals: Vec> = Vec::with_capacity(l); - - // evals[0] = [W_0(β_0), W_0(β_1), …, W_0(β_{ℓ-1})] = basis. - evals.push(basis.to_vec()); - - // evals[i][k] = W_i(β_{i+k}) computed from evals[i-1]. - // evals[i-1] = [W_{i-1}(β_{i-1}), W_{i-1}(β_i), W_{i-1}(β_{i+1}), …] - // We want W_i(β_{i+k}) = W_{i-1}(β_{i+k}) · (W_{i-1}(β_{i+k}) + W_{i-1}(β_{i-1})) - // = evals[i-1][k+1] · (evals[i-1][k+1] + evals[i-1][0]) - for i in 1..l { - let mut row = Vec::with_capacity(l - i); - for k in 1..evals[i - 1].len() { - let val = evals[i - 1][k] * (evals[i - 1][k] + evals[i - 1][0]); - row.push(val); - } - evals.push(row); - } - - // Normalize each row by its 0-th element (= W_i(β_i)). - for row in evals.iter_mut() { - let inv = row[0].inv(); - for v in row.iter_mut() { - *v *= inv; - } - } - - evals -} - -/// Compute `Σ_j bit_j(idx) · basis[j]` — the `idx`-th element of the F_2-span -/// of `basis`. -#[inline] -fn span_get(basis: &[F128], idx: usize) -> F128 { - let mut acc = F128::ZERO; - for (j, &b) in basis.iter().enumerate() { - if (idx >> j) & 1 == 1 { - acc += b; - } - } - acc -} - -/// Additive NTT over F_{2^128} with the standard polynomial-basis subspace. -/// -/// The basis is `{1, x, x², …, x^(ℓ-1)}` in F_{2^128} = F_2[x]/(GHASH-poly). -/// This makes the F_2-subspace V = `{0, 1, …, 2^ℓ-1}` (under the natural -/// integer encoding of F_{2^128} elements). -#[derive(Clone, Debug)] -pub struct AdditiveNttF128 { - /// `evals[i]` of length `ℓ − i`, the normalized subspace polynomial values. - evals: Vec>, -} - -impl AdditiveNttF128 { - /// Construct an NTT from an explicit F_2-basis. - pub fn new(basis: &[F128]) -> Self { - Self { - evals: generate_evals_from_subspace(basis), - } - } - - /// Standard NTT with basis `{1, x, x², …, x^(dim-1)}`. Requires `dim ≤ 64` - /// (the low 64 bits of F_{2^128} hold these basis vectors). - pub fn standard(dim: usize) -> Self { - assert!(dim <= 64, "standard NTT requires dim ≤ 64"); - let basis: Vec = (0..dim).map(|i| F128::new(1u64 << i, 0)).collect(); - Self::new(&basis) - } - - pub fn log_domain_size(&self) -> usize { - self.evals.len() - } - - /// Twiddle at `(layer, block)` for the forward NTT and the Ligerito fold. - /// - /// At layer `l` ∈ `[0, ℓ)`, block index `b` ∈ `[0, 2^l)`: - /// `twiddle(l, b) = Σ_j bit_j(b) · Ŵ_{ℓ-l-1}(β_{ℓ-l+j})` - /// - /// (The 0-th element of the row corresponds to `Ŵ_{ℓ-l-1}(β_{ℓ-l-1}) = 1`, - /// which is "absorbed" into the butterfly and not in the twiddle.) - pub fn twiddle(&self, layer: usize, block: usize) -> F128 { - let v = &self.evals[self.log_domain_size() - layer - 1]; - span_get(&v[1..], block) - } - - /// Forward additive NTT in place. `data.len()` must be `2^log_d` for some - /// `log_d ≤ log_domain_size()`. Layer `l ∈ [0, log_d)` is processed in - /// order (neighbors-last: top layer first). - /// - /// Dispatches to the cache-blocked batched implementation when available - /// and the buffer is large enough to benefit; otherwise falls back to the - /// per-layer parallel path or scalar. - /// - /// The live commit path uses only the interleaved transform - /// ([`Self::forward_transform_interleaved_from_layer`]); this - /// non-interleaved family (and its batched/parallel/neon dispatch tiers) - /// is retained as test oracles only. - pub fn forward_transform(&self, data: &mut [F128]) { - #[cfg(any( - all(target_arch = "aarch64", target_feature = "aes"), - all(target_arch = "x86_64", target_feature = "pclmulqdq") - ))] - { - self.forward_transform_batched(data); - } - #[cfg(not(any( - all(target_arch = "aarch64", target_feature = "aes"), - all(target_arch = "x86_64", target_feature = "pclmulqdq") - )))] - { - self.forward_transform_scalar(data); - } - } - - /// Interleaved forward NTT: process `num_ntts` independent NTTs in - /// position-major SoA layout. - /// - /// `data` layout: `data[pos * num_ntts + lane]` for `pos ∈ 0..2^log_d`, - /// `lane ∈ 0..num_ntts`. Each "lane" is an independent NTT instance over - /// the same domain; all `num_ntts` instances share the twiddle structure - /// (same `self.twiddle(layer, block)` is applied to every lane at the - /// corresponding butterfly). - /// - /// `num_ntts` must be a positive power of 2. `data.len()` must equal - /// `(1 << log_d) * num_ntts` for some `log_d ≤ log_domain_size()`. - /// - /// This produces the SAME RS code per lane as `forward_transform`, with - /// fold-compatible twiddles. The SoA layout is what makes each Merkle leaf - /// = one position across all `num_ntts` lanes (= contiguous slice of - /// `num_ntts` F_{2^128} elements). - pub fn forward_transform_interleaved(&self, data: &mut [F128], num_ntts: usize) { - self.forward_transform_interleaved_from_layer(data, num_ntts, 0); - } - - /// Forward interleaved NTT starting at `start_layer`, assuming the first - /// `start_layer` layers have already been applied to `data`. - /// - /// The RS-encoding use case: with `log_inv_rate = r` the upper - /// `(2^r − 1)/2^r` of the coefficient buffer is zero, so each of the first - /// `r` layers degenerates to a copy (butterfly with `v = 0` gives - /// `(u, u)`). The caller replicates the message into all `2^r` sub-blocks - /// — which IS the exact post-layer-`r` state — and skips those layers' - /// reads and multiplies here. - pub fn forward_transform_interleaved_from_layer( - &self, - data: &mut [F128], - num_ntts: usize, - start_layer: usize, - ) { - assert!(num_ntts.is_power_of_two() && num_ntts > 0); - let n_total = data.len(); - assert_eq!(n_total % num_ntts, 0); - let log_d = log2_pow2(n_total / num_ntts); - assert!(log_d <= self.log_domain_size()); - assert!(start_layer <= log_d); - - // Scalar; SIMD/parallel variants below dispatch from `forward_transform_interleaved` - // on supported targets. - #[cfg(any( - all(target_arch = "aarch64", target_feature = "aes"), - all(target_arch = "x86_64", target_feature = "pclmulqdq") - ))] - { - self.forward_transform_interleaved_parallel_from_layer(data, num_ntts, start_layer); - } - #[cfg(not(any( - all(target_arch = "aarch64", target_feature = "aes"), - all(target_arch = "x86_64", target_feature = "pclmulqdq") - )))] - { - self.forward_transform_interleaved_scalar_from_layer(data, num_ntts, start_layer); - } - } - - /// Scalar reference for the interleaved forward NTT. - pub fn forward_transform_interleaved_scalar(&self, data: &mut [F128], num_ntts: usize) { - self.forward_transform_interleaved_scalar_from_layer(data, num_ntts, 0); - } - - /// Scalar interleaved forward NTT from `start_layer` (see - /// [`Self::forward_transform_interleaved_from_layer`]). - pub fn forward_transform_interleaved_scalar_from_layer( - &self, - data: &mut [F128], - num_ntts: usize, - start_layer: usize, - ) { - let n_total = data.len(); - let log_d = log2_pow2(n_total / num_ntts); - - for layer in start_layer..log_d { - let num_blocks = 1usize << layer; - let block_size = 1usize << (log_d - layer); - let block_size_half = block_size >> 1; - let block_size_bytes = block_size * num_ntts; - for block in 0..num_blocks { - let twiddle = self.twiddle(layer, block); - let block_start = block * block_size_bytes; - // Butterfly pairs (top, bot) at positions (row, row + block_size_half) - // within the block. Each "position" holds num_ntts lanes side-by-side. - for row in 0..block_size_half { - let off_top = block_start + row * num_ntts; - let off_bot = off_top + block_size_half * num_ntts; - for lane in 0..num_ntts { - let v = data[off_bot + lane]; - let new_u = data[off_top + lane] + v * twiddle; - data[off_top + lane] = new_u; - data[off_bot + lane] = v + new_u; - } - } - } - } - } - - /// Parallel + NEON interleaved forward NTT. Cache-blocks the same way as - /// `forward_transform_batched`: top layers process the full SoA buffer with - /// per-block parallelism; deep layers process each sub-NTT-group in cache. - /// - /// Internally calls [`forward_transform_interleaved_scalar`] for very small - /// inputs to avoid rayon overhead; for large inputs it uses an in-place - /// scalar butterfly per lane (per-lane vectorization is future work — the - /// big win at large `m` is cache locality + thread parallelism). - /// - /// The body is arch-generic (per-lane scalar `Mul`); gated to targets with - /// a hardware carry-less multiply so slow-mul targets keep the plain - /// scalar path whose rayon thresholds still make sense. - #[cfg(any( - all(target_arch = "aarch64", target_feature = "aes"), - all(target_arch = "x86_64", target_feature = "pclmulqdq") - ))] - pub fn forward_transform_interleaved_parallel(&self, data: &mut [F128], num_ntts: usize) { - self.forward_transform_interleaved_parallel_from_layer(data, num_ntts, 0); - } - - /// Parallel interleaved forward NTT from `start_layer` (see - /// [`Self::forward_transform_interleaved_from_layer`]). - #[cfg(any( - all(target_arch = "aarch64", target_feature = "aes"), - all(target_arch = "x86_64", target_feature = "pclmulqdq") - ))] - pub fn forward_transform_interleaved_parallel_from_layer( - &self, - data: &mut [F128], - num_ntts: usize, - start_layer: usize, - ) { - use rayon::prelude::*; - let n_total = data.len(); - let log_d = log2_pow2(n_total / num_ntts); - - // Target sub-group size = 2 MB total bytes. Each position is - // `num_ntts × 16` bytes, so positions per sub-group = - // 2^21 / (num_ntts · 16). With num_ntts=1: 2^17 positions. With - // num_ntts=32: 2^12 positions. (Without this scaling, sub-groups at - // num_ntts=32 would be 64 MB and overflow L2 cache.) - const TARGET_SUBGROUP_LOG_BYTES: usize = 21; - let log_bytes_per_position = 4 + log2_pow2(num_ntts); - let target_log_positions = TARGET_SUBGROUP_LOG_BYTES.saturating_sub(log_bytes_per_position); - let cache_n_top = log_d.saturating_sub(target_log_positions); - - // Parallelism floor. The cache heuristic keeps each sub-NTT ~2 MB, but - // for a mid-size transform whose whole codeword already fits that - // budget it yields `cache_n_top == 0` and the transform runs fully - // serial — e.g. the multilevel Ligerito commits (~1 ms of NTT each, - // previously 1.0× across threads). When the transform is big enough to - // amortize rayon overhead, raise `n_top` so the deep-layer split - // produces ~one sub-NTT per worker thread (capped to keep each sub-NTT - // ≥ 2^MIN_SUB_LOG positions). The large initial PCS commit is unaffected: - // its `cache_n_top` already exceeds this floor. - // - // The floor (log_d ≥ 12) is the measured dispatch-vs-compute crossover - // for num_ntts≈8 level commits: at log_d=12 parallelizing cuts the - // NTT ~0.22 → ~0.08 ms, but at log_d=10 the rayon dispatch costs more - // than the ~0.04 ms of work, so those stay scalar. - const PARALLEL_FLOOR_LOG_D: usize = 12; - const MIN_SUB_LOG: usize = 8; - let n_top = if log_d >= PARALLEL_FLOOR_LOG_D { - let want_subs_log = log2_pow2(rayon::current_num_threads().next_power_of_two()); - let max_n_top = log_d.saturating_sub(MIN_SUB_LOG); - cache_n_top.max(want_subs_log.min(max_n_top)) - } else { - cache_n_top - }; - if n_top == 0 || log_d < 8 { - self.forward_transform_interleaved_scalar_from_layer(data, num_ntts, start_layer); - return; - } - - // Top layers: full-buffer sweep. Parallelize **rows within each - // block** so even layer 0 (1 huge block) gets rayon parallelism. - // - // Layer fusion: at top layers each layer is a separate full-buffer - // sweep (read 512 MB + write 512 MB at m=31). Fusing two consecutive - // layers in one pass loads each row once, applies both butterflies - // in registers, stores once — halving memory traffic on the fused - // layers. Each "outer block" at layer L has 4 contributing rows per - // quarter-row; layer L butterflies (a,c) and (b,d) (distance = - // block_size/2), layer L+1 butterflies (a,b) and (c,d) (distance = - // block_size/4). Radix-8 fuses three layers the same way and cuts the - // sweeps by another third; see - // [`butterfly_interleaved_fused_3layer_par_rows`] for why 8 is the - // widest fusion that fits. - // - // Radix-8 is used on every target: its kernel batches lanes at the - // target's widest multiply width (see `MUL_LANES`), so it keeps the - // CLMUL batching the fused-2 kernel had on x86 while still cutting a - // third of the sweeps. - let fused3_ok = true; - let mut layer = start_layer.min(n_top); - while layer < n_top { - let num_blocks = 1usize << layer; - let block_size = 1usize << (log_d - layer); - let block_bytes = block_size * num_ntts; - - if fused3_ok && layer + 2 < n_top && block_size >= 8 { - // Fuse layers (layer, layer+1, layer+2): one read + one write - // per row group instead of three. - let eighth = block_size >> 3; - for block in 0..num_blocks { - let mut tw = [F128::ZERO; 7]; - tw[0] = self.twiddle(layer, block); - for s in 0..2 { - tw[1 + s] = self.twiddle(layer + 1, 2 * block + s); - } - for s in 0..4 { - tw[3 + s] = self.twiddle(layer + 2, 4 * block + s); - } - let start = block * block_bytes; - let region = &mut data[start..start + block_bytes]; - // Block 0's left spine is `twiddle(l, 0) = 0` at every - // layer, so seven of its twelve butterflies degenerate to - // XORs. That is a property of the block index, not of any - // particular transform shape. - if block == 0 { - butterfly_interleaved_fused_3layer_par_rows::( - region, &tw, eighth, num_ntts, - ); - } else { - butterfly_interleaved_fused_3layer_par_rows::( - region, &tw, eighth, num_ntts, - ); - } - } - layer += 3; - } else if layer + 1 < n_top && block_size >= 4 { - // Fuse layers (layer, layer+1). - let quarter = block_size >> 2; - for block in 0..num_blocks { - let t_outer = self.twiddle(layer, block); - let t_inner_a = self.twiddle(layer + 1, 2 * block); - let t_inner_b = self.twiddle(layer + 1, 2 * block + 1); - let start = block * block_bytes; - butterfly_interleaved_fused_2layer_par_rows( - &mut data[start..start + block_bytes], - t_outer, - t_inner_a, - t_inner_b, - quarter, - num_ntts, - ); - } - layer += 2; - } else { - let block_size_half = block_size >> 1; - for block in 0..num_blocks { - let t = self.twiddle(layer, block); - let start = block * block_bytes; - butterfly_interleaved_block_par_rows( - &mut data[start..start + block_bytes], - t, - block_size_half, - num_ntts, - ); - } - layer += 1; - } - } - - // Deep layers: process each sub-NTT-group cache-resident. - let sub_size_positions = 1usize << (log_d - n_top); - let sub_bytes = sub_size_positions * num_ntts; - - data.par_chunks_mut(sub_bytes) - .enumerate() - .for_each(|(sub_idx, sub_data)| { - for layer in n_top.max(start_layer)..log_d { - let layer_in_sub = layer - n_top; - let num_blocks_in_sub = 1usize << layer_in_sub; - let block_size = 1usize << (log_d - layer); - let block_size_half = block_size >> 1; - let block_bytes = block_size * num_ntts; - - for block_in_sub in 0..num_blocks_in_sub { - let global_block = sub_idx * num_blocks_in_sub + block_in_sub; - let twiddle = self.twiddle(layer, global_block); - let block_start = block_in_sub * block_bytes; - let block = &mut sub_data[block_start..block_start + block_bytes]; - butterfly_interleaved_block(block, twiddle, block_size_half, num_ntts); - } - } - }); - } - - /// Scalar reference implementation. Used as the test oracle and on - /// platforms without NEON+PMULL. - pub fn forward_transform_scalar(&self, data: &mut [F128]) { - let log_d = log2_pow2(data.len()); - assert!(log_d <= self.log_domain_size()); - - for layer in 0..log_d { - let num_blocks = 1usize << layer; - let block_size_half = 1usize << (log_d - layer - 1); - for block in 0..num_blocks { - let twiddle = self.twiddle(layer, block); - let block_start = block << (log_d - layer); - for idx0 in block_start..(block_start + block_size_half) { - let idx1 = idx0 | block_size_half; - // Forward butterfly: u += v·twiddle; v += u. - let v = data[idx1]; - let new_u = data[idx0] + v * twiddle; - data[idx0] = new_u; - data[idx1] = v + new_u; - } - } - } - } - - /// Single-threaded NEON forward transform (uses `ghash_mul_vec2_neon` to - /// batch 2 butterflies per PMULL pair). - #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] - pub fn forward_transform_neon(&self, data: &mut [F128]) { - let log_d = log2_pow2(data.len()); - assert!(log_d <= self.log_domain_size()); - - for layer in 0..log_d { - let num_blocks = 1usize << layer; - let block_size = 1usize << (log_d - layer); - let block_size_half = block_size >> 1; - // SAFETY: target_feature = "aes" enabled at compile time. - unsafe { - if block_size_half >= 2 { - // Within-block: batch 2 pairs with shared twiddle. - for block in 0..num_blocks { - let twiddle = self.twiddle(layer, block); - let block_start = block * block_size; - let chunk = &mut data[block_start..block_start + block_size]; - butterfly_block_neon(chunk, twiddle, block_size_half); - } - } else { - // Deepest layer (half = 1): batch across 2 adjacent blocks - // (different twiddles). Handle odd tail with scalar when - // num_blocks = 1 (only happens at log_d = 1). - debug_assert_eq!(block_size_half, 1); - let mut block = 0; - while block + 1 < num_blocks { - let t_a = self.twiddle(layer, block); - let t_b = self.twiddle(layer, block + 1); - butterfly_across_blocks_neon(data, block * 2, t_a, t_b); - block += 2; - } - // Scalar tail (num_blocks odd — only when num_blocks = 1). - while block < num_blocks { - let twiddle = self.twiddle(layer, block); - let idx0 = block * 2; - let idx1 = idx0 + 1; - let v = data[idx1]; - let new_u = data[idx0] + v * twiddle; - data[idx0] = new_u; - data[idx1] = v + new_u; - block += 1; - } - } - } - } - } - - /// Rayon-parallel + SIMD forward transform. - #[cfg(any( - all(target_arch = "aarch64", target_feature = "aes"), - all(target_arch = "x86_64", target_feature = "pclmulqdq") - ))] - pub fn forward_transform_parallel(&self, data: &mut [F128]) { - use rayon::prelude::*; - let log_d = log2_pow2(data.len()); - assert!(log_d <= self.log_domain_size()); - - // For small data (or shallow layers with few large blocks), the rayon - // overhead exceeds the gain — fall back to the single-thread path. - const PARALLEL_THRESHOLD_LOG: usize = 14; // 2^14 = 16K elements (256 KB) - if log_d <= PARALLEL_THRESHOLD_LOG { - self.forward_transform_single_thread(data); - return; - } - - for layer in 0..log_d { - let num_blocks = 1usize << layer; - let block_size = 1usize << (log_d - layer); - let block_size_half = block_size >> 1; - - // Parallelize across blocks when there are enough; otherwise process - // sequentially with SIMD (still fast for small block counts). - if num_blocks >= 4 && block_size_half >= 2 { - let twiddles: Vec = (0..num_blocks).map(|b| self.twiddle(layer, b)).collect(); - data.par_chunks_mut(block_size) - .zip(twiddles.par_iter()) - .for_each(|(chunk, &twiddle)| { - butterfly_block_dispatch(chunk, twiddle, block_size_half); - }); - } else if block_size_half >= 2 { - // Few large blocks — process sequentially with SIMD. - for block in 0..num_blocks { - let twiddle = self.twiddle(layer, block); - let block_start = block * block_size; - butterfly_block_dispatch( - &mut data[block_start..block_start + block_size], - twiddle, - block_size_half, - ); - } - } else { - // Deepest layer (half = 1): need num_blocks ≥ 2 to batch - // pairs; if there are at least 2 blocks, batch across them. - // (When num_blocks < 2, fall back to single-thread which - // handles the trivial cases.) - debug_assert_eq!(block_size_half, 1); - if num_blocks >= 2 { - let twiddles: Vec = - (0..num_blocks).map(|b| self.twiddle(layer, b)).collect(); - data.par_chunks_mut(4).zip(twiddles.par_chunks(2)).for_each( - |(chunk, twiddle_pair)| { - butterfly_pair_dispatch_in_chunk( - chunk, - twiddle_pair[0], - twiddle_pair[1], - ); - }, - ); - } else { - let twiddle = self.twiddle(layer, 0); - let v = data[1]; - let new_u = data[0] + v * twiddle; - data[0] = new_u; - data[1] = v + new_u; - } - } - } - } - - /// Single-threaded transform used below rayon thresholds: NEON-batched on - /// aarch64, plain scalar elsewhere (scalar `Mul` is CLMUL-backed on x86). - #[cfg(any( - all(target_arch = "aarch64", target_feature = "aes"), - all(target_arch = "x86_64", target_feature = "pclmulqdq") - ))] - #[inline] - fn forward_transform_single_thread(&self, data: &mut [F128]) { - #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] - { - self.forward_transform_neon(data); - } - #[cfg(not(all(target_arch = "aarch64", target_feature = "aes")))] - { - self.forward_transform_scalar(data); - } - } - - /// Cache-blocked + parallel + NEON forward transform. - /// - /// **Strategy**: decompose the NTT into two stages so the deep layers - /// (which dominate work) operate on sub-buffers small enough to fit in L2 - /// cache, avoiding the DRAM round-trip per layer. - /// - /// 1. **Top layers** (layers `0..n_top`): each layer touches the full buffer - /// in one sweep. Bandwidth-bound; parallelize across blocks. - /// 2. **Deep layers** (layers `n_top..log_d`): treat the data as `2^n_top` - /// independent sub-NTTs, each of size `2^(log_d − n_top)`. For each - /// sub-NTT, process ALL remaining layers in one cache-resident pass. - /// Parallelize across sub-NTTs via rayon. - /// - /// `n_top` is chosen so each sub-NTT is `≈ 2 MB` (= `2^17` F_{2^128} ≈ 2 MB). - /// For `log_d ≤ 17` the whole NTT fits in cache and we fall back to the - /// per-layer parallel path. - #[cfg(any( - all(target_arch = "aarch64", target_feature = "aes"), - all(target_arch = "x86_64", target_feature = "pclmulqdq") - ))] - pub fn forward_transform_batched(&self, data: &mut [F128]) { - use rayon::prelude::*; - let log_d = log2_pow2(data.len()); - assert!(log_d <= self.log_domain_size()); - - // Target sub-NTT size: 2^17 F_{2^128} = 2 MB. Tunable. - const TARGET_SUB_NTT_LOG: usize = 17; - if log_d <= TARGET_SUB_NTT_LOG { - self.forward_transform_parallel(data); - return; - } - let n_top = log_d - TARGET_SUB_NTT_LOG; - let sub_ntt_size = 1usize << (log_d - n_top); - - // ---- Stage 1: top layers (full-buffer, bandwidth-bound). - for layer in 0..n_top { - let num_blocks = 1usize << layer; - let block_size = 1usize << (log_d - layer); - let block_size_half = block_size >> 1; - - if num_blocks >= 4 { - let twiddles: Vec = (0..num_blocks).map(|b| self.twiddle(layer, b)).collect(); - data.par_chunks_mut(block_size) - .zip(twiddles.par_iter()) - .for_each(|(chunk, &t)| { - butterfly_block_dispatch(chunk, t, block_size_half); - }); - } else { - // Few large blocks at very top layers: sequential SIMD. - for block in 0..num_blocks { - let t = self.twiddle(layer, block); - let block_start = block * block_size; - butterfly_block_dispatch( - &mut data[block_start..block_start + block_size], - t, - block_size_half, - ); - } - } - } - - // ---- Stage 2: deep layers as parallel cache-resident sub-NTTs. - data.par_chunks_mut(sub_ntt_size) - .enumerate() - .for_each(|(sub_idx, sub_data)| { - for layer in n_top..log_d { - let layer_in_sub = layer - n_top; - let num_blocks_in_sub = 1usize << layer_in_sub; - let block_size = 1usize << (log_d - layer); - let block_size_half = block_size >> 1; - - for block_in_sub in 0..num_blocks_in_sub { - let global_block = sub_idx * num_blocks_in_sub + block_in_sub; - let twiddle = self.twiddle(layer, global_block); - let block_start = block_in_sub * block_size; - let block = &mut sub_data[block_start..block_start + block_size]; - if block_size_half >= 2 { - butterfly_block_dispatch(block, twiddle, block_size_half); - } else { - // Deepest layer: 1 pair per block, scalar. - let v = block[1]; - let new_u = block[0] + v * twiddle; - block[0] = new_u; - block[1] = v + new_u; - } - } - } - }); - } - - /// Inverse additive NTT in place. Exact inverse of `forward_transform`. - /// Test-oracle only, like the rest of the non-interleaved family. - pub fn inverse_transform(&self, data: &mut [F128]) { - let log_d = log2_pow2(data.len()); - assert!(log_d <= self.log_domain_size()); - - for layer in (0..log_d).rev() { - let num_blocks = 1usize << layer; - let block_size_half = 1usize << (log_d - layer - 1); - for block in 0..num_blocks { - let twiddle = self.twiddle(layer, block); - let block_start = block << (log_d - layer); - for idx0 in block_start..(block_start + block_size_half) { - let idx1 = idx0 | block_size_half; - // Inverse butterfly: v += u; u += v·twiddle. - let u = data[idx0]; - let new_v = data[idx1] + u; - data[idx1] = new_v; - data[idx0] = u + new_v * twiddle; - } - } - } - } -} - -/// Like [`butterfly_interleaved_block`] but parallelizes across rows via -/// rayon. Used at top layers where the block is large (≥ 1024 rows) and only -/// 1-2 blocks exist (so block-level parallelism would be too coarse). -/// -/// Falls back to sequential when the row count is small. -#[inline] -fn butterfly_interleaved_block_par_rows( - block: &mut [F128], - twiddle: F128, - block_size_half: usize, - num_ntts: usize, -) { - use rayon::prelude::*; - const PARALLEL_ROW_THRESHOLD: usize = 512; - if block_size_half < PARALLEL_ROW_THRESHOLD { - butterfly_interleaved_block(block, twiddle, block_size_half, num_ntts); - return; - } - let half_offset = block_size_half * num_ntts; - let (top, bot) = block.split_at_mut(half_offset); - top.par_chunks_mut(num_ntts) - .zip(bot.par_chunks_mut(num_ntts)) - .for_each(|(top_row, bot_row)| { - butterfly_rows(top_row, bot_row, twiddle); - }); -} - -/// Fused 2-layer butterfly: combines layer L (twiddle `t_outer`, shared by -/// the whole outer block) with layer L+1 (twiddles `t_inner_a` for the top -/// half, `t_inner_b` for the bottom half). Reads each row of the outer -/// block once and writes once — halving memory traffic vs running the two -/// layers as separate sweeps. -/// -/// `block` has length `4 * quarter * num_ntts` (= one layer-L block of -/// `4*quarter` rows). For each `r ∈ 0..quarter`, four rows participate: -/// `a=r`, `b=r+quarter`, `c=r+2*quarter`, `d=r+3*quarter`. Layer L -/// butterflies `(a,c)` and `(b,d)`; layer L+1 then butterflies `(a,b)` (in -/// the new top sub-block) and `(c,d)` (in the new bottom sub-block). -#[inline] -fn butterfly_interleaved_fused_2layer_par_rows( - block: &mut [F128], - t_outer: F128, - t_inner_a: F128, - t_inner_b: F128, - quarter: usize, - num_ntts: usize, -) { - use rayon::prelude::*; - const PARALLEL_ROW_THRESHOLD: usize = 256; - let stride = quarter * num_ntts; - debug_assert_eq!(block.len(), 4 * stride); - - let do_one = - |row_a: &mut [F128], row_b: &mut [F128], row_c: &mut [F128], row_d: &mut [F128]| { - #[cfg(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx2" - ))] - { - use primitives::field::gf2_128::x86_64::ghash_mul_vec2_clmul; - for lane in 0..num_ntts { - let mut a = row_a[lane]; - let mut b = row_b[lane]; - let c = row_c[lane]; - let d = row_d[lane]; - // Layer L: both muls share t_outer — one 2-wide CLMUL. - // SAFETY: vpclmulqdq+avx2 statically enabled by the cfg gate. - let prod = unsafe { ghash_mul_vec2_clmul([t_outer, t_outer], [c, d]) }; - let new_a = a + prod[0]; - let c = c + new_a; - a = new_a; - let new_b = b + prod[1]; - let d = d + new_b; - b = new_b; - // Layer L+1: (a, b) with t_inner_a; (c, d) with t_inner_b — - // one 2-wide CLMUL with distinct twiddles per lane. - // SAFETY: as above. - let prod2 = unsafe { ghash_mul_vec2_clmul([t_inner_a, t_inner_b], [b, d]) }; - let new_a2 = a + prod2[0]; - let new_c2 = c + prod2[1]; - row_a[lane] = new_a2; - row_b[lane] = b + new_a2; - row_c[lane] = new_c2; - row_d[lane] = d + new_c2; - } - } - #[cfg(not(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx2" - )))] - for lane in 0..num_ntts { - let mut a = row_a[lane]; - let mut b = row_b[lane]; - let mut c = row_c[lane]; - let mut d = row_d[lane]; - // Layer L (matches `butterfly_interleaved_block`'s formula: - // new_u = u + v*twiddle; new_v = v + new_u). - let new_a = a + c * t_outer; - c += new_a; - a = new_a; - let new_b = b + d * t_outer; - d += new_b; - b = new_b; - // Layer L+1: (a, b) with t_inner_a (top sub-block); - // (c, d) with t_inner_b (bottom sub-block). - let new_a2 = a + b * t_inner_a; - b += new_a2; - a = new_a2; - let new_c2 = c + d * t_inner_b; - d += new_c2; - c = new_c2; - row_a[lane] = a; - row_b[lane] = b; - row_c[lane] = c; - row_d[lane] = d; - } - }; - - // Split the block into four quarters, then zip row-wise. Each rayon task - // processes one quarter-row index = 4 logical rows of work. - let (top_half, bot_half) = block.split_at_mut(2 * stride); - let (q1, q2) = top_half.split_at_mut(stride); - let (q3, q4) = bot_half.split_at_mut(stride); - - if quarter < PARALLEL_ROW_THRESHOLD { - for r in 0..quarter { - let off = r * num_ntts; - let (q1r, q1_rest) = q1[off..].split_at_mut(num_ntts); - let _ = q1_rest; - let (q2r, _) = q2[off..].split_at_mut(num_ntts); - let (q3r, _) = q3[off..].split_at_mut(num_ntts); - let (q4r, _) = q4[off..].split_at_mut(num_ntts); - do_one(q1r, q2r, q3r, q4r); - } - } else { - q1.par_chunks_mut(num_ntts) - .zip(q2.par_chunks_mut(num_ntts)) - .zip(q3.par_chunks_mut(num_ntts)) - .zip(q4.par_chunks_mut(num_ntts)) - .for_each(|(((row_a, row_b), row_c), row_d)| { - do_one(row_a, row_b, row_c, row_d); - }); - } -} - -/// Lanes multiplied per batched call inside the radix-8 network. -/// -/// The eight values of one radix-8 group are chained across the three layers, -/// but the interleaved lanes are independent — so a butterfly at a fixed -/// position can multiply `MUL_LANES` lanes at once against its shared twiddle. -/// The width is whatever the target's widest reduced-product kernel is: -/// AVX-512 does four 128-bit lanes per `VPCLMULQDQ`, AVX2 does two. -/// -/// AArch64 deliberately stays at one: `ghash_mul_vec2_neon` was measured -/// slower than scalar-with-ILP here, because PMULL batching serializes what -/// the out-of-order engine was already overlapping (same finding as the note -/// on [`butterfly_interleaved_block`]). -#[cfg(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx512f", - target_feature = "avx512bw" -))] -const MUL_LANES: usize = 4; -#[cfg(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx2", - not(all(target_feature = "avx512f", target_feature = "avx512bw")) -))] -const MUL_LANES: usize = 2; -#[cfg(not(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - any( - target_feature = "avx2", - all(target_feature = "avx512f", target_feature = "avx512bw") - ) -)))] -const MUL_LANES: usize = 1; - -/// Multiply `L` values by one shared twiddle, using the widest batched kernel -/// the target has. `L = 1` is a plain scalar product. -#[inline(always)] -fn mul_shared(t: F128, v: [F128; L]) -> [F128; L] { - let mut out = [F128::ZERO; L]; - let mut i = 0; - #[cfg(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx512f", - target_feature = "avx512bw" - ))] - while i + 4 <= L { - use primitives::field::gf2_128::x86_64::ghash_mul_vec4_clmul; - // SAFETY: the required features are statically enabled by the cfg gate. - let p = unsafe { ghash_mul_vec4_clmul([t; 4], [v[i], v[i + 1], v[i + 2], v[i + 3]]) }; - out[i..i + 4].copy_from_slice(&p); - i += 4; - } - #[cfg(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx2" - ))] - while i + 2 <= L { - use primitives::field::gf2_128::x86_64::ghash_mul_vec2_clmul; - // SAFETY: the required features are statically enabled by the cfg gate. - let p = unsafe { ghash_mul_vec2_clmul([t; 2], [v[i], v[i + 1]]) }; - out[i..i + 2].copy_from_slice(&p); - i += 2; - } - while i < L { - out[i] = v[i] * t; - i += 1; - } - out -} - -/// One lane group's radix-8 (fused three-layer) butterfly network. -/// -/// `t` holds the seven twiddles in breadth-first order: `t[0]` is layer L -/// (shared by all four pairs), `t[1..3]` layer L+1 (one per half), `t[3..7]` -/// layer L+2 (one per quarter). Each of the 8 slots holds `L` independent -/// interleaved lanes. -#[inline(always)] -fn butterfly_fused_3layer(v: &mut [[F128; L]; 8], t: &[F128; 7]) { - for i in 0..4 { - bfly(v, i, i + 4, t[0]); - } - for s in 0..2 { - for i in 0..2 { - bfly(v, 4 * s + i, 4 * s + i + 2, t[1 + s]); - } - } - for s in 0..4 { - bfly(v, 2 * s, 2 * s + 1, t[3 + s]); - } -} - -/// `(v[u], v[w]) <- (v[u] + v[w]·t, v[w] + v[u] + v[w]·t)`, across `L` lanes. -#[inline(always)] -fn bfly(v: &mut [[F128; L]; 8], u: usize, w: usize, twiddle: F128) { - let prod = mul_shared(twiddle, v[w]); - for l in 0..L { - let new_u = v[u][l] + prod[l]; - v[w][l] += new_u; - v[u][l] = new_u; - } -} - -/// `v[w] += v[u]`, across `L` lanes — a butterfly whose twiddle is zero. -#[inline(always)] -fn bfly_zero(v: &mut [[F128; L]; 8], u: usize, w: usize) { - for l in 0..L { - let x = v[u][l]; - v[w][l] += x; - } -} - -/// [`butterfly_fused_3layer`] for block 0, whose left-spine twiddles -/// `t[0] = t[1] = t[3]` are identically zero (block index 0 selects no basis -/// vector at any layer). A zero-twiddle forward butterfly maps `(u, v)` to -/// `(u, v + u)`, so those seven butterflies need only XORs — five field -/// multiplies instead of twelve — and `v[0]` comes out unchanged. -#[inline(always)] -fn butterfly_fused_3layer_zero_root(v: &mut [[F128; L]; 8], t: &[F128; 7]) { - debug_assert_eq!(t[0], F128::ZERO); - debug_assert_eq!(t[1], F128::ZERO); - debug_assert_eq!(t[3], F128::ZERO); - - // Layer L (t[0] = 0): four XOR-only butterflies. - for i in 0..4 { - bfly_zero(v, i, i + 4); - } - // Layer L+1: the top half's twiddle t[1] is zero, t[2] is general. - for i in 0..2 { - bfly_zero(v, i, i + 2); - } - bfly(v, 4, 6, t[2]); - bfly(v, 5, 7, t[2]); - // Layer L+2: the first quarter's twiddle t[3] is zero. - bfly_zero(v, 0, 1); - bfly(v, 2, 3, t[4]); - bfly(v, 4, 5, t[5]); - bfly(v, 6, 7, t[6]); -} - -/// Apply [`butterfly_fused_3layer`] to row group `r` across every lane. -/// -/// # Safety -/// `ptr` must be valid for the `8 * eighth * num_ntts` elements of one -/// layer-L block, and concurrent calls must use distinct `r` (which select -/// disjoint rows). -#[inline] -unsafe fn butterfly_fused_3layer_row( - ptr: *mut F128, - eighth: usize, - num_ntts: usize, - r: usize, - t: &[F128; 7], -) { - // SAFETY: the caller supplies the pointer geometry and disjointness. - unsafe { - // Lanes are independent, so sweep them `MUL_LANES` at a time and let - // each butterfly multiply the whole group against its shared twiddle - // in one batched call. `num_ntts` is a power of two ≥ 1; a group - // narrower than `MUL_LANES` falls through to the scalar tail. - // A lane group is contiguous in the SoA layout, so gather and scatter - // it as one block move rather than element by element — the whole - // group is a single vector load/store at every width, including - // `MUL_LANES == 1`, where this compiles back to the plain scalar - // access the unbatched kernel used. - #[inline(always)] - unsafe fn group( - ptr: *mut F128, - eighth: usize, - num_ntts: usize, - r: usize, - lane: usize, - t: &[F128; 7], - ) { - // SAFETY: the caller's geometry and disjointness contract holds, - // and `lane + L <= num_ntts` bounds every access to this block. - unsafe { - let mut v = [[F128::ZERO; L]; 8]; - for (i, slot) in v.iter_mut().enumerate() { - let base = ptr.add((i * eighth + r) * num_ntts + lane); - core::ptr::copy_nonoverlapping(base, slot.as_mut_ptr(), L); - } - if ZR { - butterfly_fused_3layer_zero_root(&mut v, t); - } else { - butterfly_fused_3layer(&mut v, t); - } - for (i, slot) in v.iter().enumerate() { - let base = ptr.add((i * eighth + r) * num_ntts + lane); - core::ptr::copy_nonoverlapping(slot.as_ptr(), base, L); - } - } - } - - let mut lane = 0; - while lane + MUL_LANES <= num_ntts { - group::(ptr, eighth, num_ntts, r, lane, t); - lane += MUL_LANES; - } - // `num_ntts` is a power of two, so a partial group only occurs when - // the transform is narrower than the multiply width. - while lane < num_ntts { - group::<1, ZERO_ROOT>(ptr, eighth, num_ntts, r, lane, t); - lane += 1; - } - } -} - -/// Fused three-layer (radix-8) butterfly over one layer-L block: applies -/// layers L, L+1 and L+2 in a single read/write of each row, instead of the -/// three full-buffer sweeps they would otherwise cost. At the top layers, -/// where a sweep is a DRAM round-trip over the whole codeword, that is a -/// third fewer passes than the fused-2 kernel and two thirds fewer than -/// unfused. -/// -/// Eight is the widest fusion that pays: the eight live row streams sit at -/// stride `eighth * num_ntts`, a multiple of the L1 set-repeat period at -/// these shapes, so they all map to the same set — exactly 8 ways on an -/// 8-way L1D. Radix-16 would demand 16 ways and thrash. -#[inline] -fn butterfly_interleaved_fused_3layer_par_rows( - block: &mut [F128], - t: &[F128; 7], - eighth: usize, - num_ntts: usize, -) { - use rayon::prelude::*; - const PARALLEL_ROW_THRESHOLD: usize = 256; - debug_assert_eq!(block.len(), 8 * eighth * num_ntts); - - // Carry the base as `usize` so rayon's per-`r` closure can hold it without - // a raw-pointer `Sync` shim. Row group `r` writes rows - // `{i * eighth + r : i ∈ 0..8}`, disjoint across `r`. - let base = block.as_mut_ptr() as usize; - if eighth < PARALLEL_ROW_THRESHOLD { - for r in 0..eighth { - // SAFETY: `r` selects rows inside this block, run sequentially. - unsafe { - butterfly_fused_3layer_row::(base as *mut F128, eighth, num_ntts, r, t) - }; - } - } else { - (0..eighth).into_par_iter().for_each(|r| { - // SAFETY: distinct `r` → disjoint row groups → no aliasing. - unsafe { - butterfly_fused_3layer_row::(base as *mut F128, eighth, num_ntts, r, t) - }; - }); - } -} - -/// Butterfly `num_ntts` lanes of one (top_row, bot_row) pair with a shared -/// twiddle: `u += v·t; v += u`. On x86_64 with VPCLMULQDQ the lane muls run -/// 2-wide (ymm); elsewhere scalar-per-lane (see the ILP note on -/// [`butterfly_interleaved_block`]). -#[inline] -fn butterfly_rows(top_row: &mut [F128], bot_row: &mut [F128], twiddle: F128) { - debug_assert_eq!(top_row.len(), bot_row.len()); - #[cfg(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx2" - ))] - { - use primitives::field::gf2_128::x86_64::ghash_mul_vec2_clmul; - let n = top_row.len(); - let mut lane = 0; - while lane + 2 <= n { - // SAFETY: vpclmulqdq+avx2 statically enabled by the cfg gate. - let prod = unsafe { - ghash_mul_vec2_clmul([twiddle, twiddle], [bot_row[lane], bot_row[lane + 1]]) - }; - let new_u0 = top_row[lane] + prod[0]; - let new_u1 = top_row[lane + 1] + prod[1]; - top_row[lane] = new_u0; - top_row[lane + 1] = new_u1; - bot_row[lane] += new_u0; - bot_row[lane + 1] += new_u1; - lane += 2; - } - if lane < n { - let v = bot_row[lane]; - let new_u = top_row[lane] + v * twiddle; - top_row[lane] = new_u; - bot_row[lane] = v + new_u; - } - } - #[cfg(not(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx2" - )))] - { - for lane in 0..top_row.len() { - let v = bot_row[lane]; - let new_u = top_row[lane] + v * twiddle; - top_row[lane] = new_u; - bot_row[lane] = v + new_u; - } - } -} - -/// Butterfly one block of an interleaved (SoA) buffer with shared twiddle. -/// -/// `block` has length `(2 * block_size_half) * num_ntts` and is laid out as -/// `num_ntts` lanes interleaved per row, `2 * block_size_half` rows total. -/// Pairs row `r` with row `r + block_size_half` for `r ∈ 0..block_size_half`. -/// -/// **Note**: On aarch64 this is scalar-per-lane on purpose. With `num_ntts = -/// 32` and shared twiddle, the inner loop has 32 independent F_{2^128} muls -/// per row that the compiler ILPs effectively (each mul uses NEON via the -/// field's `binius_mul` already). An explicit 2-lane `ghash_mul_vec2_neon` -/// variant was tried but **regressed** by ~10-30% because the explicit -/// batching prevented ILP across more than 2 muls and added load/store -/// overhead. On x86_64, `butterfly_rows` batches 2 lanes per 256-bit -/// VPCLMULQDQ instead — there the batching halves the instruction count on -/// the same ports, which measured faster than scalar-with-ILP. -#[inline] -fn butterfly_interleaved_block( - block: &mut [F128], - twiddle: F128, - block_size_half: usize, - num_ntts: usize, -) { - let off_bot = block_size_half * num_ntts; - let (top, bot) = block.split_at_mut(off_bot); - for r in 0..block_size_half { - let off_top = r * num_ntts; - butterfly_rows( - &mut top[off_top..off_top + num_ntts], - &mut bot[off_top..off_top + num_ntts], - twiddle, - ); - } -} - -#[inline] -fn log2_pow2(n: usize) -> usize { - assert!( - n.is_power_of_two() && n > 0, - "length must be a positive power of 2" - ); - n.trailing_zeros() as usize -} - -// --------------------------------------------------------------------------- -// Portable butterfly dispatch — NEON kernels on aarch64; generic loops that -// lean on the fast scalar `Mul` (CLMUL-backed) on x86_64. -// --------------------------------------------------------------------------- - -/// Butterfly one block (pairs `(i, i + half)` for `i ∈ 0..half`) with a -/// shared twiddle. Precondition: `half >= 2`. -#[cfg(any( - all(target_arch = "aarch64", target_feature = "aes"), - all(target_arch = "x86_64", target_feature = "pclmulqdq") -))] -#[inline] -fn butterfly_block_dispatch(chunk: &mut [F128], twiddle: F128, half: usize) { - #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] - // SAFETY: aes target feature enabled at compile time. - unsafe { - butterfly_block_neon(chunk, twiddle, half) - }; - #[cfg(not(all(target_arch = "aarch64", target_feature = "aes")))] - butterfly_block_generic(chunk, twiddle, half); -} - -/// Generic block butterfly: the two halves are one long (top, bot) row pair, -/// so [`butterfly_rows`] applies directly (2-wide CLMUL on x86_64 with -/// VPCLMULQDQ, scalar-with-ILP otherwise). -#[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] -#[inline] -fn butterfly_block_generic(chunk: &mut [F128], twiddle: F128, half: usize) { - debug_assert!(half >= 2); - debug_assert_eq!(chunk.len(), 2 * half); - let (top, bot) = chunk.split_at_mut(half); - butterfly_rows(top, bot, twiddle); -} - -/// Two butterflies on a 4-element chunk `(u_a, v_a, u_b, v_b)` with DIFFERENT -/// twiddles (deepest layer, across 2 adjacent blocks). -#[cfg(any( - all(target_arch = "aarch64", target_feature = "aes"), - all(target_arch = "x86_64", target_feature = "pclmulqdq") -))] -#[inline] -fn butterfly_pair_dispatch_in_chunk(chunk: &mut [F128], t_a: F128, t_b: F128) { - #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] - // SAFETY: aes target feature enabled at compile time. - unsafe { - butterfly_across_blocks_neon_in_chunk(chunk, t_a, t_b) - }; - #[cfg(not(all(target_arch = "aarch64", target_feature = "aes")))] - { - debug_assert_eq!(chunk.len(), 4); - let v = chunk[1]; - let new_u = chunk[0] + v * t_a; - chunk[0] = new_u; - chunk[1] = v + new_u; - let v = chunk[3]; - let new_u = chunk[2] + v * t_b; - chunk[2] = new_u; - chunk[3] = v + new_u; - } -} - -// --------------------------------------------------------------------------- -// NEON butterfly helpers — batch 2 F128 butterflies per `ghash_mul_vec2_neon`. -// --------------------------------------------------------------------------- - -/// Two butterflies within a single block (shared twiddle). -/// -/// `chunk` is one block of length `2 * half`. Pairs (idx0, idx0 + half) for -/// idx0 = 0..half are butterflied. We process two consecutive idx0's at once -/// to share the twiddle across the `ghash_mul_vec2_neon` call. -/// -/// Precondition: `half >= 2`. (At deepest layer where half=1, use -/// [`butterfly_across_blocks_neon`] instead.) -#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] -#[inline] -#[target_feature(enable = "aes")] -unsafe fn butterfly_block_neon(chunk: &mut [F128], twiddle: F128, half: usize) { - use primitives::field::gf2_128::aarch64::ghash_mul_vec2_neon; - debug_assert!(half >= 2); - debug_assert_eq!(chunk.len(), 2 * half); - let mut idx0 = 0; - while idx0 < half { - let idx1 = idx0 + half; - let u_a = chunk[idx0]; - let v_a = chunk[idx1]; - let u_b = chunk[idx0 + 1]; - let v_b = chunk[idx1 + 1]; - - // SAFETY: aes target feature enabled. - let prod = unsafe { ghash_mul_vec2_neon([twiddle, twiddle], [v_a, v_b]) }; - - let new_u_a = F128 { - lo: u_a.lo ^ prod[0].lo, - hi: u_a.hi ^ prod[0].hi, - }; - let new_u_b = F128 { - lo: u_b.lo ^ prod[1].lo, - hi: u_b.hi ^ prod[1].hi, - }; - let new_v_a = F128 { - lo: v_a.lo ^ new_u_a.lo, - hi: v_a.hi ^ new_u_a.hi, - }; - let new_v_b = F128 { - lo: v_b.lo ^ new_u_b.lo, - hi: v_b.hi ^ new_u_b.hi, - }; - - chunk[idx0] = new_u_a; - chunk[idx1] = new_v_a; - chunk[idx0 + 1] = new_u_b; - chunk[idx1 + 1] = new_v_b; - idx0 += 2; - } -} - -/// Two butterflies across 2 adjacent blocks at the deepest layer (each block -/// has just 1 pair, i.e., block_size_half = 1). The two pairs have DIFFERENT -/// twiddles. -/// -/// Operates on `data[base..base+4]` = (block0_lo, block0_hi, block1_lo, block1_hi). -#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] -#[inline] -#[target_feature(enable = "aes")] -unsafe fn butterfly_across_blocks_neon(data: &mut [F128], base: usize, t_a: F128, t_b: F128) { - // SAFETY: caller's `aes` target-feature attribute covers this call. - unsafe { butterfly_across_blocks_neon_in_chunk(&mut data[base..base + 4], t_a, t_b) }; -} - -#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] -#[inline] -#[target_feature(enable = "aes")] -unsafe fn butterfly_across_blocks_neon_in_chunk(chunk: &mut [F128], t_a: F128, t_b: F128) { - use primitives::field::gf2_128::aarch64::ghash_mul_vec2_neon; - debug_assert_eq!(chunk.len(), 4); - let u_a = chunk[0]; - let v_a = chunk[1]; - let u_b = chunk[2]; - let v_b = chunk[3]; - - // SAFETY: aes target feature enabled. - let prod = unsafe { ghash_mul_vec2_neon([t_a, t_b], [v_a, v_b]) }; - - let new_u_a = F128 { - lo: u_a.lo ^ prod[0].lo, - hi: u_a.hi ^ prod[0].hi, - }; - let new_u_b = F128 { - lo: u_b.lo ^ prod[1].lo, - hi: u_b.hi ^ prod[1].hi, - }; - let new_v_a = F128 { - lo: v_a.lo ^ new_u_a.lo, - hi: v_a.hi ^ new_u_a.hi, - }; - let new_v_b = F128 { - lo: v_b.lo ^ new_u_b.lo, - hi: v_b.hi ^ new_u_b.hi, - }; - - chunk[0] = new_u_a; - chunk[1] = new_v_a; - chunk[2] = new_u_b; - chunk[3] = new_v_b; -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_rng::Rng; - - fn rand_vec(rng: &mut Rng, n: usize) -> Vec { - (0..n).map(|_| rng.f128()).collect() - } - - #[test] - fn forward_inverse_roundtrip() { - let mut rng = Rng::new(0xAB1); - for log_d in [1usize, 2, 3, 4, 6, 8] { - let ntt = AdditiveNttF128::standard(log_d); - let original = rand_vec(&mut rng, 1 << log_d); - let mut v = original.clone(); - ntt.forward_transform(&mut v); - ntt.inverse_transform(&mut v); - assert_eq!(v, original, "roundtrip failed at log_d={log_d}"); - } - } - - #[test] - fn inverse_forward_roundtrip() { - let mut rng = Rng::new(0xAB2); - for log_d in [1usize, 2, 3, 4, 6, 8] { - let ntt = AdditiveNttF128::standard(log_d); - let original = rand_vec(&mut rng, 1 << log_d); - let mut v = original.clone(); - ntt.inverse_transform(&mut v); - ntt.forward_transform(&mut v); - assert_eq!( - v, original, - "inverse∘forward roundtrip failed at log_d={log_d}" - ); - } - } - - #[test] - fn forward_is_linear() { - let mut rng = Rng::new(0xAB3); - for log_d in [1usize, 2, 3, 5] { - let ntt = AdditiveNttF128::standard(log_d); - let n = 1 << log_d; - let a = rand_vec(&mut rng, n); - let b = rand_vec(&mut rng, n); - let ab: Vec = a.iter().zip(&b).map(|(x, y)| *x + *y).collect(); - - let mut fa = a.clone(); - ntt.forward_transform(&mut fa); - let mut fb = b.clone(); - ntt.forward_transform(&mut fb); - let mut fab = ab.clone(); - ntt.forward_transform(&mut fab); - - for i in 0..n { - assert_eq!( - fa[i] + fb[i], - fab[i], - "linearity fails at i={i}, log_d={log_d}" - ); - } - } - } - - #[test] - fn ntt_of_zero_is_zero() { - for log_d in [1usize, 2, 3, 6] { - let ntt = AdditiveNttF128::standard(log_d); - let mut v = vec![F128::ZERO; 1 << log_d]; - ntt.forward_transform(&mut v); - assert!(v.iter().all(|&x| x == F128::ZERO)); - } - } - - #[test] - fn twiddle_at_layer_0_uses_full_basis_minus_one() { - // At layer 0 (topmost forward butterfly), there's 1 block. - // twiddle(0, 0) = 0 (no bits set in block index 0). - let ntt = AdditiveNttF128::standard(4); - assert_eq!(ntt.twiddle(0, 0), F128::ZERO); - } - - #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] - #[test] - fn neon_matches_scalar() { - let mut rng = Rng::new(0xBB1); - for log_d in 1..=10 { - let ntt = AdditiveNttF128::standard(log_d); - let original = rand_vec(&mut rng, 1 << log_d); - let mut v_scalar = original.clone(); - ntt.forward_transform_scalar(&mut v_scalar); - let mut v_neon = original.clone(); - ntt.forward_transform_neon(&mut v_neon); - assert_eq!( - v_neon, v_scalar, - "NEON disagrees with scalar at log_d={log_d}" - ); - } - } - - #[test] - fn interleaved_matches_per_lane() { - let mut rng = Rng::new(0xCC1); - // For several log_d and num_ntts, verify the interleaved transform - // matches running the per-lane scalar transform on each sub-NTT. - for log_d in [3usize, 4, 8] { - for num_ntts in [1usize, 2, 4, 8] { - let ntt = AdditiveNttF128::standard(log_d); - let n_total = (1 << log_d) * num_ntts; - let original = rand_vec(&mut rng, n_total); - - // Interleaved. - let mut v_inter = original.clone(); - ntt.forward_transform_interleaved_scalar(&mut v_inter, num_ntts); - - // Reference: per-lane, gather + scalar transform + scatter. - let mut v_ref = original.clone(); - for lane in 0..num_ntts { - let mut sub: Vec = (0..(1 << log_d)) - .map(|pos| v_ref[pos * num_ntts + lane]) - .collect(); - ntt.forward_transform_scalar(&mut sub); - for pos in 0..(1 << log_d) { - v_ref[pos * num_ntts + lane] = sub[pos]; - } - } - - assert_eq!( - v_inter, v_ref, - "interleaved mismatch at log_d={log_d}, num_ntts={num_ntts}" - ); - } - } - } - - #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] - #[test] - fn interleaved_parallel_matches_scalar() { - let mut rng = Rng::new(0xCC2); - for log_d in [4usize, 10, 14, 17, 19] { - for &num_ntts in &[2usize, 8, 32] { - let ntt = AdditiveNttF128::standard(log_d); - let n_total = (1 << log_d) * num_ntts; - let original = rand_vec(&mut rng, n_total); - let mut v_scalar = original.clone(); - ntt.forward_transform_interleaved_scalar(&mut v_scalar, num_ntts); - let mut v_par = original.clone(); - ntt.forward_transform_interleaved_parallel(&mut v_par, num_ntts); - assert_eq!( - v_par, v_scalar, - "interleaved parallel mismatch at log_d={log_d}, num_ntts={num_ntts}" - ); - } - } - } - - #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] - #[test] - fn batched_matches_scalar() { - let mut rng = Rng::new(0xBB4); - // Include sizes above the TARGET_SUB_NTT_LOG threshold (17) so we - // exercise the cache-blocked path. - for log_d in [4usize, 8, 12, 17, 18, 19, 20] { - let ntt = AdditiveNttF128::standard(log_d); - let original = rand_vec(&mut rng, 1 << log_d); - let mut v_scalar = original.clone(); - ntt.forward_transform_scalar(&mut v_scalar); - let mut v_batched = original.clone(); - ntt.forward_transform_batched(&mut v_batched); - assert_eq!( - v_batched, v_scalar, - "batched disagrees with scalar at log_d={log_d}" - ); - } - } - - #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] - #[test] - fn parallel_matches_scalar() { - let mut rng = Rng::new(0xBB2); - for log_d in [4usize, 8, 12, 15, 16] { - let ntt = AdditiveNttF128::standard(log_d); - let original = rand_vec(&mut rng, 1 << log_d); - let mut v_scalar = original.clone(); - ntt.forward_transform_scalar(&mut v_scalar); - let mut v_par = original.clone(); - ntt.forward_transform_parallel(&mut v_par); - assert_eq!( - v_par, v_scalar, - "parallel disagrees with scalar at log_d={log_d}" - ); - } - } - - /// The radix-8 network, and its zero-root specialization, must reproduce - /// three separate single-layer passes over the same 8 values. - #[test] - fn fused_3layer_matches_three_single_layers() { - fn three_single_layers(v: &mut [F128; 8], t: &[F128; 7]) { - let mut bfly = |v: &mut [F128; 8], u: usize, w: usize, tw: F128| { - let new_u = v[u] + v[w] * tw; - v[w] += new_u; - v[u] = new_u; - }; - for i in 0..4 { - bfly(v, i, i + 4, t[0]); - } - for s in 0..2 { - for i in 0..2 { - bfly(v, 4 * s + i, 4 * s + i + 2, t[1 + s]); - } - } - for s in 0..4 { - bfly(v, 2 * s, 2 * s + 1, t[3 + s]); - } - } - - let mut rng = Rng::new(0xF3); - for zero_root in [false, true] { - for _ in 0..64 { - let mut t = [F128::ZERO; 7]; - for slot in t.iter_mut() { - *slot = rng.f128(); - } - if zero_root { - t[0] = F128::ZERO; - t[1] = F128::ZERO; - t[3] = F128::ZERO; - } - let mut v = [F128::ZERO; 8]; - for slot in v.iter_mut() { - *slot = rng.f128(); - } - let mut want = v; - three_single_layers(&mut want, &t); - let mut got = [[F128::ZERO; 1]; 8]; - for (g, &x) in got.iter_mut().zip(v.iter()) { - g[0] = x; - } - if zero_root { - butterfly_fused_3layer_zero_root(&mut got, &t); - } else { - butterfly_fused_3layer(&mut got, &t); - } - let got: [F128; 8] = std::array::from_fn(|i| got[i][0]); - assert_eq!(got, want, "radix-8 mismatch (zero_root={zero_root})"); - } - } - } - - /// Block 0's left spine is zero at every layer, which is what licenses the - /// unconditional zero-root dispatch in the top-layer loop. - #[test] - fn block_zero_left_spine_twiddles_vanish() { - for log_d in [4usize, 8, 12] { - let ntt = AdditiveNttF128::standard(log_d); - for layer in 0..log_d { - assert_eq!(ntt.twiddle(layer, 0), F128::ZERO); - } - } - } - - #[test] - fn deepest_layer_twiddle_count() { - let log_d = 4; - let ntt = AdditiveNttF128::standard(log_d); - // At layer log_d - 1 = 3, there are 2^3 = 8 blocks. twiddle(3, b) for b ∈ 0..8. - for b in 0..8 { - let _t = ntt.twiddle(log_d - 1, b); - } - } -} diff --git a/crates/pcs/src/ntt/additive_ntt_f64.rs b/crates/pcs/src/ntt/additive_ntt_f64.rs new file mode 100644 index 00000000..4f28cdc6 --- /dev/null +++ b/crates/pcs/src/ntt/additive_ntt_f64.rs @@ -0,0 +1,792 @@ +//! Additive NTT over GF(2^64) using the LCH novel polynomial basis: the +//! encoding layer of the 64-bit transition (commitments over K = F_{2^64}). +//! +//! The transform uses the same subspace-polynomial construction, +//! neighbors-last layer ordering, and SoA interleaved layout as the +//! extension-field transform. Its inner butterflies use architecture-specific +//! SIMD kernels where available; large transforms are memory-bandwidth bound. + +use primitives::field::F64; + +/// Normalized subspace-polynomial evaluation table (see the extension-field twin). +fn generate_evals_from_subspace(basis: &[F64]) -> Vec> { + let l = basis.len(); + let mut evals: Vec> = Vec::with_capacity(l); + evals.push(basis.to_vec()); + for i in 1..l { + let mut row = Vec::with_capacity(l - i); + for k in 1..evals[i - 1].len() { + let val = evals[i - 1][k] * (evals[i - 1][k] + evals[i - 1][0]); + row.push(val); + } + evals.push(row); + } + for row in evals.iter_mut() { + let inv = row[0].inv(); + for v in row.iter_mut() { + *v *= inv; + } + } + evals +} + +/// `Σ_j bit_j(idx) · basis[j]`. +#[inline] +fn span_get(basis: &[F64], idx: usize) -> F64 { + let mut acc = F64::ZERO; + for (j, &b) in basis.iter().enumerate() { + if (idx >> j) & 1 == 1 { + acc += b; + } + } + acc +} + +/// Additive NTT over F_{2^64} with the standard polynomial-basis subspace +/// `{1, x, x², …}`: the F_2-subspace is `{0, 1, …, 2^ℓ−1}` under the natural +/// integer encoding, exactly as in the extension-field version (whose domain already +/// lived inside this very subfield). +#[derive(Clone, Debug)] +pub struct AdditiveNttF64 { + evals: Vec>, +} + +impl AdditiveNttF64 { + pub fn new(basis: &[F64]) -> Self { + Self { + evals: generate_evals_from_subspace(basis), + } + } + + /// Standard NTT with basis `{1, x, …, x^(dim-1)}`. Requires `dim ≤ 63` so + /// the evaluation domain (and the twiddles) stay inside F_{2^64} without + /// wrap; far beyond any codeword size in use. + pub fn standard(dim: usize) -> Self { + assert!(dim <= 63, "standard NTT requires dim ≤ 63"); + let basis: Vec = (0..dim).map(|i| F64(1u64 << i)).collect(); + Self::new(&basis) + } + + pub fn log_domain_size(&self) -> usize { + self.evals.len() + } + + /// Twiddle at `(layer, block)`; see the extension-field twin for the convention. + pub fn twiddle(&self, layer: usize, block: usize) -> F64 { + let v = &self.evals[self.log_domain_size() - layer - 1]; + span_get(&v[1..], block) + } + + /// Forward additive NTT in place (scalar; used directly for tests and as + /// the small-input path). + pub fn forward_transform_scalar(&self, data: &mut [F64]) { + let log_d = log2_pow2(data.len()); + assert!(log_d <= self.log_domain_size()); + for layer in 0..log_d { + let num_blocks = 1usize << layer; + let block_size_half = 1usize << (log_d - layer - 1); + for block in 0..num_blocks { + let twiddle = self.twiddle(layer, block); + let block_start = block << (log_d - layer); + for idx0 in block_start..(block_start + block_size_half) { + let idx1 = idx0 | block_size_half; + let v = data[idx1]; + let new_u = data[idx0] + v * twiddle; + data[idx0] = new_u; + data[idx1] = v + new_u; + } + } + } + } + + /// Forward NTT in place, dispatching to the parallel path for large + /// inputs (single-lane case of the interleaved transform). + pub fn forward_transform(&self, data: &mut [F64]) { + self.forward_transform_interleaved_from_layer(data, 1, 0); + } + + /// Interleaved (SoA) forward NTT: `num_ntts` independent lanes sharing + /// the twiddle structure; `data[pos * num_ntts + lane]`. Same layout + /// contract as the extension-field twin (one Merkle leaf = one position = a + /// contiguous slice of `num_ntts` F_{2^64} elements). + pub fn forward_transform_interleaved(&self, data: &mut [F64], num_ntts: usize) { + self.forward_transform_interleaved_from_layer(data, num_ntts, 0); + } + + /// Interleaved forward NTT starting at `start_layer` (the RS-encoding + /// caller replicates the message into all `2^rate` sub-blocks, which IS + /// the exact post-layer-`rate` state, and skips those layers here). + pub fn forward_transform_interleaved_from_layer(&self, data: &mut [F64], num_ntts: usize, start_layer: usize) { + assert!(num_ntts.is_power_of_two() && num_ntts > 0); + let n_total = data.len(); + assert_eq!(n_total % num_ntts, 0); + let log_d = log2_pow2(n_total / num_ntts); + assert!(log_d <= self.log_domain_size()); + assert!(start_layer <= log_d); + + self.forward_transform_interleaved_parallel_from_layer(data, num_ntts, start_layer); + } + + /// Scalar reference for the interleaved forward NTT (test oracle). + pub fn forward_transform_interleaved_scalar_from_layer( + &self, + data: &mut [F64], + num_ntts: usize, + start_layer: usize, + ) { + let n_total = data.len(); + let log_d = log2_pow2(n_total / num_ntts); + + for layer in start_layer..log_d { + let num_blocks = 1usize << layer; + let block_size = 1usize << (log_d - layer); + let block_size_half = block_size >> 1; + let block_elems = block_size * num_ntts; + for block in 0..num_blocks { + let twiddle = self.twiddle(layer, block); + let block_start = block * block_elems; + for row in 0..block_size_half { + let off_top = block_start + row * num_ntts; + let off_bot = off_top + block_size_half * num_ntts; + for lane in 0..num_ntts { + let v = data[off_bot + lane]; + let new_u = data[off_top + lane] + v * twiddle; + data[off_top + lane] = new_u; + data[off_bot + lane] = v + new_u; + } + } + } + } + } + + /// Parallel interleaved forward NTT, cache-blocked like the extension-field twin: + /// top layers sweep the full buffer (fused two-layer passes, row-parallel), + /// deep layers run as cache-resident sub-NTTs in parallel. Constants are + /// re-derived for 8-byte elements. + pub fn forward_transform_interleaved_parallel_from_layer( + &self, + data: &mut [F64], + num_ntts: usize, + start_layer: usize, + ) { + use rayon::prelude::*; + let n_total = data.len(); + let log_d = log2_pow2(n_total / num_ntts); + + // Target sub-group ≈ 2 MB; each position is num_ntts × 8 bytes. + const TARGET_SUBGROUP_LOG_BYTES: usize = 21; + let log_bytes_per_position = 3 + log2_pow2(num_ntts); + let target_log_positions = TARGET_SUBGROUP_LOG_BYTES.saturating_sub(log_bytes_per_position); + let cache_n_top = log_d.saturating_sub(target_log_positions); + + const PARALLEL_FLOOR_LOG_D: usize = 12; + const MIN_SUB_LOG: usize = 8; + let n_top = if log_d >= PARALLEL_FLOOR_LOG_D { + let want_subs_log = log2_pow2(rayon::current_num_threads().next_power_of_two()); + let max_n_top = log_d.saturating_sub(MIN_SUB_LOG); + cache_n_top.max(want_subs_log.min(max_n_top)) + } else { + cache_n_top + }; + if n_top == 0 || log_d < 8 { + self.forward_transform_interleaved_scalar_from_layer(data, num_ntts, start_layer); + return; + } + + // Top layers: full-buffer sweeps, fusing two layers where possible. + let mut layer = start_layer.min(n_top); + while layer < n_top { + let num_blocks = 1usize << layer; + let block_size = 1usize << (log_d - layer); + let block_elems = block_size * num_ntts; + + if layer + 2 < n_top && block_size >= 8 { + // Fuse three layers: one pass over the block instead of + // three. At the top layers a pass is a DRAM round-trip of the + // whole codeword, so the pass count sets the cost. + let eighth = block_size >> 3; + for block in 0..num_blocks { + let mut t = [F64::ZERO; 7]; + t[0] = self.twiddle(layer, block); + for s in 0..2 { + t[1 + s] = self.twiddle(layer + 1, 2 * block + s); + } + for s in 0..4 { + t[3 + s] = self.twiddle(layer + 2, 4 * block + s); + } + let start = block * block_elems; + butterfly_interleaved_fused_3layer_par_rows( + &mut data[start..start + block_elems], + &t, + eighth, + num_ntts, + ); + } + layer += 3; + } else if layer + 1 < n_top && block_size >= 4 { + let quarter = block_size >> 2; + for block in 0..num_blocks { + let t_outer = self.twiddle(layer, block); + let t_inner_a = self.twiddle(layer + 1, 2 * block); + let t_inner_b = self.twiddle(layer + 1, 2 * block + 1); + let start = block * block_elems; + butterfly_interleaved_fused_2layer_par_rows( + &mut data[start..start + block_elems], + t_outer, + t_inner_a, + t_inner_b, + quarter, + num_ntts, + ); + } + layer += 2; + } else { + let block_size_half = block_size >> 1; + for block in 0..num_blocks { + let t = self.twiddle(layer, block); + let start = block * block_elems; + butterfly_interleaved_block_par_rows( + &mut data[start..start + block_elems], + t, + block_size_half, + num_ntts, + ); + } + layer += 1; + } + } + + // Deep layers: parallel cache-resident sub-NTTs. + let sub_size_positions = 1usize << (log_d - n_top); + let sub_elems = sub_size_positions * num_ntts; + data.par_chunks_mut(sub_elems) + .enumerate() + .for_each(|(sub_idx, sub_data)| { + for layer in n_top.max(start_layer)..log_d { + let layer_in_sub = layer - n_top; + let num_blocks_in_sub = 1usize << layer_in_sub; + let block_size = 1usize << (log_d - layer); + let block_size_half = block_size >> 1; + let block_elems = block_size * num_ntts; + for block_in_sub in 0..num_blocks_in_sub { + let global_block = sub_idx * num_blocks_in_sub + block_in_sub; + let twiddle = self.twiddle(layer, global_block); + let block_start = block_in_sub * block_elems; + let block = &mut sub_data[block_start..block_start + block_elems]; + butterfly_interleaved_block(block, twiddle, block_size_half, num_ntts); + } + } + }); + } + + /// Inverse additive NTT in place (scalar). Exact inverse of the forward + /// transform; used by tests. + pub fn inverse_transform(&self, data: &mut [F64]) { + let log_d = log2_pow2(data.len()); + assert!(log_d <= self.log_domain_size()); + for layer in (0..log_d).rev() { + let num_blocks = 1usize << layer; + let block_size_half = 1usize << (log_d - layer - 1); + for block in 0..num_blocks { + let twiddle = self.twiddle(layer, block); + let block_start = block << (log_d - layer); + for idx0 in block_start..(block_start + block_size_half) { + let idx1 = idx0 | block_size_half; + let u = data[idx0]; + let new_v = data[idx1] + u; + data[idx1] = new_v; + data[idx0] = u + new_v * twiddle; + } + } + } + } +} + +fn butterfly_interleaved_block_par_rows(block: &mut [F64], twiddle: F64, block_size_half: usize, num_ntts: usize) { + use rayon::prelude::*; + const PARALLEL_ROW_THRESHOLD: usize = 1024; + if block_size_half < PARALLEL_ROW_THRESHOLD { + butterfly_interleaved_block(block, twiddle, block_size_half, num_ntts); + return; + } + let half_offset = block_size_half * num_ntts; + let (top, bot) = block.split_at_mut(half_offset); + top.par_chunks_mut(num_ntts) + .zip(bot.par_chunks_mut(num_ntts)) + .for_each(|(top_row, bot_row)| { + butterfly_lanes(top_row, bot_row, twiddle); + }); +} + +/// Fused 2-layer butterfly, row-parallel; see the extension-field twin for the shape. +/// Fused three-layer (radix-8) butterfly over one layer-L block: applies +/// layers L, L+1 and L+2 in a single pass over the block's rows, instead of +/// the three full-buffer sweeps they would otherwise cost. +/// +/// The eight participating rows stay L1-resident across all twelve +/// butterflies, exactly as the four rows do in the fused-2 kernel; each +/// butterfly is the same lane kernel, already 8-wide on NEON and AVX-512. +/// +/// `block` is `8 * eighth * num_ntts` elements. For each `r ∈ 0..eighth` the +/// rows `r + i·eighth` for `i ∈ 0..8` participate. Layer L pairs them at +/// distance `4·eighth`, layer L+1 at `2·eighth`, layer L+2 at `eighth`. +/// `t` holds the seven twiddles breadth-first: `t[0]` is layer L, `t[1..3]` +/// layer L+1 (one per half), `t[3..7]` layer L+2 (one per quarter). +fn butterfly_interleaved_fused_3layer_par_rows(block: &mut [F64], t: &[F64; 7], eighth: usize, num_ntts: usize) { + use rayon::prelude::*; + const PARALLEL_ROW_THRESHOLD: usize = 512; + let stride = eighth * num_ntts; + debug_assert_eq!(block.len(), 8 * stride); + + let do_one = |rows: &mut [&mut [F64]; 8]| { + let [r0, r1, r2, r3, r4, r5, r6, r7] = rows; + // Layer L, distance 4·eighth. + butterfly_lanes(r0, r4, t[0]); + butterfly_lanes(r1, r5, t[0]); + butterfly_lanes(r2, r6, t[0]); + butterfly_lanes(r3, r7, t[0]); + // Layer L+1, distance 2·eighth: t[1] on the new top half, t[2] on the + // new bottom half. + butterfly_lanes(r0, r2, t[1]); + butterfly_lanes(r1, r3, t[1]); + butterfly_lanes(r4, r6, t[2]); + butterfly_lanes(r5, r7, t[2]); + // Layer L+2, distance eighth: one twiddle per quarter. + butterfly_lanes(r0, r1, t[3]); + butterfly_lanes(r2, r3, t[4]); + butterfly_lanes(r4, r5, t[5]); + butterfly_lanes(r6, r7, t[6]); + }; + + let (h0, h1) = block.split_at_mut(4 * stride); + let (q0, q1) = h0.split_at_mut(2 * stride); + let (q2, q3) = h1.split_at_mut(2 * stride); + let (e0, e1) = q0.split_at_mut(stride); + let (e2, e3) = q1.split_at_mut(stride); + let (e4, e5) = q2.split_at_mut(stride); + let (e6, e7) = q3.split_at_mut(stride); + + if eighth < PARALLEL_ROW_THRESHOLD { + for r in 0..eighth { + let off = r * num_ntts; + let mut rows = [ + &mut e0[off..off + num_ntts], + &mut e1[off..off + num_ntts], + &mut e2[off..off + num_ntts], + &mut e3[off..off + num_ntts], + &mut e4[off..off + num_ntts], + &mut e5[off..off + num_ntts], + &mut e6[off..off + num_ntts], + &mut e7[off..off + num_ntts], + ]; + do_one(&mut rows); + } + } else { + e0.par_chunks_mut(num_ntts) + .zip(e1.par_chunks_mut(num_ntts)) + .zip(e2.par_chunks_mut(num_ntts)) + .zip(e3.par_chunks_mut(num_ntts)) + .zip(e4.par_chunks_mut(num_ntts)) + .zip(e5.par_chunks_mut(num_ntts)) + .zip(e6.par_chunks_mut(num_ntts)) + .zip(e7.par_chunks_mut(num_ntts)) + .for_each(|(((((((a, b), c), d), e), f), g), h)| { + let mut rows = [a, b, c, d, e, f, g, h]; + do_one(&mut rows); + }); + } +} + +fn butterfly_interleaved_fused_2layer_par_rows( + block: &mut [F64], + t_outer: F64, + t_inner_a: F64, + t_inner_b: F64, + quarter: usize, + num_ntts: usize, +) { + use rayon::prelude::*; + const PARALLEL_ROW_THRESHOLD: usize = 512; + let stride = quarter * num_ntts; + debug_assert_eq!(block.len(), 4 * stride); + + let do_one = |row_a: &mut [F64], row_b: &mut [F64], row_c: &mut [F64], row_d: &mut [F64]| { + // Layer L butterflies (a,c) and (b,d), then layer L+1 (a,b) and + // (c,d); each stage runs the NEON lane-pair kernel over the rows. + butterfly_lanes(row_a, row_c, t_outer); + butterfly_lanes(row_b, row_d, t_outer); + butterfly_lanes(row_a, row_b, t_inner_a); + butterfly_lanes(row_c, row_d, t_inner_b); + }; + + let (top_half, bot_half) = block.split_at_mut(2 * stride); + let (q1, q2) = top_half.split_at_mut(stride); + let (q3, q4) = bot_half.split_at_mut(stride); + + if quarter < PARALLEL_ROW_THRESHOLD { + for r in 0..quarter { + let off = r * num_ntts; + let (q1r, _) = q1[off..].split_at_mut(num_ntts); + let (q2r, _) = q2[off..].split_at_mut(num_ntts); + let (q3r, _) = q3[off..].split_at_mut(num_ntts); + let (q4r, _) = q4[off..].split_at_mut(num_ntts); + do_one(q1r, q2r, q3r, q4r); + } + } else { + q1.par_chunks_mut(num_ntts) + .zip(q2.par_chunks_mut(num_ntts)) + .zip(q3.par_chunks_mut(num_ntts)) + .zip(q4.par_chunks_mut(num_ntts)) + .for_each(|(((row_a, row_b), row_c), row_d)| { + do_one(row_a, row_b, row_c, row_d); + }); + } +} + +#[inline] +fn butterfly_interleaved_block(block: &mut [F64], twiddle: F64, block_size_half: usize, num_ntts: usize) { + let half_offset = block_size_half * num_ntts; + let (top, bot) = block.split_at_mut(half_offset); + for r in 0..block_size_half { + let off = r * num_ntts; + butterfly_lanes(&mut top[off..off + num_ntts], &mut bot[off..off + num_ntts], twiddle); + } +} + +/// Butterfly all `num_ntts` lanes of one (top row, bottom row) pair with a +/// shared twiddle: new_u = u + v*t; new_v = v + new_u. +/// +/// On NEON this processes eight lanes per iteration. Four independent pair +/// reductions stay in the vector register file, exposing their PMULL chains +/// in parallel and amortizing the loop branch and constant setup. The pair +/// kernel handles a short even tail, and the scalar path handles an odd tail. +#[inline] +fn butterfly_lanes(top: &mut [F64], bot: &mut [F64], twiddle: F64) { + debug_assert_eq!(top.len(), bot.len()); + #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] + { + let vectors = top.len() / 8; + // SAFETY: the target features are enabled at compile time and each + // iteration reads and writes exactly eight elements from both rows. + unsafe { + for i in 0..vectors { + butterfly_lanes_avx512(top.as_mut_ptr().add(8 * i), bot.as_mut_ptr().add(8 * i), twiddle.0); + } + } + for lane in 8 * vectors..top.len() { + let v = bot[lane]; + let new_u = top[lane] + v * twiddle; + top[lane] = new_u; + bot[lane] = v + new_u; + } + } + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + { + let vectors = top.len() / 8; + // SAFETY: aes target feature is enabled at compile time; the kernel + // reads/writes exactly lanes [8i, 8i+8) of each row. + unsafe { + for i in 0..vectors { + butterfly_lanes_neon_8(top.as_mut_ptr().add(8 * i), bot.as_mut_ptr().add(8 * i), twiddle.0); + } + let mut lane = 8 * vectors; + while lane + 2 <= top.len() { + butterfly_lane_pair_neon(top.as_mut_ptr().add(lane), bot.as_mut_ptr().add(lane), twiddle.0); + lane += 2; + } + } + if top.len() % 2 == 1 { + let last = top.len() - 1; + let v = bot[last]; + let new_u = top[last] + v * twiddle; + top[last] = new_u; + bot[last] = v + new_u; + } + } + #[cfg(not(any( + all(target_arch = "aarch64", target_feature = "aes"), + all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f") + )))] + { + for lane in 0..top.len() { + let v = bot[lane]; + let new_u = top[lane] + v * twiddle; + top[lane] = new_u; + bot[lane] = v + new_u; + } + } +} + +/// Eight F64 butterflies as four independent NEON lane-pair reductions. +/// Loading all four bottom vectors before reducing them gives the out-of-order +/// core four independent PMULL chains to schedule, while one call amortizes +/// loop control and the duplicated twiddle/reduction constants over 8 lanes. +/// +/// # Safety +/// Requires the `aes` target feature; `top`/`bot` must each point at eight +/// readable+writable F64 values. +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +#[inline] +#[target_feature(enable = "aes")] +unsafe fn butterfly_lanes_neon_8(top: *mut F64, bot: *mut F64, twiddle: u64) { + use core::arch::aarch64::*; + use primitives::field::gf2_64::aarch64::reduce_pair_pmull4; + + // SAFETY: caller guarantees the two eight-element regions; F64 is + // repr(transparent) over u64 and this function carries the aes feature. + unsafe { + let v0 = vld1q_u64(bot.cast()); + let v1 = vld1q_u64(bot.cast::().add(2)); + let v2 = vld1q_u64(bot.cast::().add(4)); + let v3 = vld1q_u64(bot.cast::().add(6)); + let tw = vdupq_n_u64(twiddle); + + let p00: uint64x2_t = core::mem::transmute(vmull_p64(vgetq_lane_u64::<0>(v0), twiddle)); + let p01: uint64x2_t = core::mem::transmute(vmull_high_p64( + core::mem::transmute::(v0), + core::mem::transmute::(tw), + )); + let p10: uint64x2_t = core::mem::transmute(vmull_p64(vgetq_lane_u64::<0>(v1), twiddle)); + let p11: uint64x2_t = core::mem::transmute(vmull_high_p64( + core::mem::transmute::(v1), + core::mem::transmute::(tw), + )); + let p20: uint64x2_t = core::mem::transmute(vmull_p64(vgetq_lane_u64::<0>(v2), twiddle)); + let p21: uint64x2_t = core::mem::transmute(vmull_high_p64( + core::mem::transmute::(v2), + core::mem::transmute::(tw), + )); + let p30: uint64x2_t = core::mem::transmute(vmull_p64(vgetq_lane_u64::<0>(v3), twiddle)); + let p31: uint64x2_t = core::mem::transmute(vmull_high_p64( + core::mem::transmute::(v3), + core::mem::transmute::(tw), + )); + + let prod0 = reduce_pair_pmull4(p00, p01); + let prod1 = reduce_pair_pmull4(p10, p11); + let prod2 = reduce_pair_pmull4(p20, p21); + let prod3 = reduce_pair_pmull4(p30, p31); + + let u0 = vld1q_u64(top.cast()); + let u1 = vld1q_u64(top.cast::().add(2)); + let u2 = vld1q_u64(top.cast::().add(4)); + let u3 = vld1q_u64(top.cast::().add(6)); + let new_u0 = veorq_u64(u0, prod0); + let new_u1 = veorq_u64(u1, prod1); + let new_u2 = veorq_u64(u2, prod2); + let new_u3 = veorq_u64(u3, prod3); + let new_v0 = veorq_u64(v0, new_u0); + let new_v1 = veorq_u64(v1, new_u1); + let new_v2 = veorq_u64(v2, new_u2); + let new_v3 = veorq_u64(v3, new_u3); + + vst1q_u64(top.cast(), new_u0); + vst1q_u64(top.cast::().add(2), new_u1); + vst1q_u64(top.cast::().add(4), new_u2); + vst1q_u64(top.cast::().add(6), new_u3); + vst1q_u64(bot.cast(), new_v0); + vst1q_u64(bot.cast::().add(2), new_v1); + vst1q_u64(bot.cast::().add(4), new_v2); + vst1q_u64(bot.cast::().add(6), new_v3); + } +} + +/// Eight F64 butterflies using the four independent 128-bit lanes of +/// VPCLMULQDQ. Even and odd u64 lanes are multiplied separately, reduced in +/// parallel, then packed back into their original order. +/// +/// # Safety +/// Requires VPCLMULQDQ + AVX-512F; `top` and `bot` must each address eight +/// readable and writable F64 values. +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx512f", enable = "avx2")] +unsafe fn butterfly_lanes_avx512(top: *mut F64, bot: *mut F64, twiddle: u64) { + use core::arch::x86_64::*; + + #[inline] + #[target_feature(enable = "vpclmulqdq", enable = "avx512f")] + unsafe fn reduce(p: __m512i, r: __m512i) -> __m512i { + let t = _mm512_clmulepi64_epi128::<0x01>(p, r); + let u = _mm512_clmulepi64_epi128::<0x01>(t, r); + _mm512_xor_si512(_mm512_xor_si512(p, t), u) + } + + // SAFETY: the caller supplies valid eight-element rows and the function's + // target features cover every intrinsic below. + unsafe { + let u = _mm512_loadu_si512(top.cast()); + let v = _mm512_loadu_si512(bot.cast()); + let tw = _mm512_set1_epi64(twiddle as i64); + let r = _mm512_set1_epi64(0x1b); + + let even = reduce(_mm512_clmulepi64_epi128::<0x00>(v, tw), r); + let odd = reduce(_mm512_clmulepi64_epi128::<0x11>(v, tw), r); + let odd = _mm512_shuffle_epi32::<0x4e>(odd); + let product = _mm512_mask_blend_epi64(0xaa, even, odd); + + let new_u = _mm512_xor_si512(u, product); + let new_v = _mm512_xor_si512(v, new_u); + _mm512_storeu_si512(top.cast(), new_u); + _mm512_storeu_si512(bot.cast(), new_v); + } +} + +/// Two F64 butterflies with a shared twiddle, NEON-resident end to end. +/// The two products issue as PMULL/PMULL2 on the loaded row (no lane +/// extraction) and reduce through the all-PMULL lane-pair fold +/// ([`primitives::field::gf2_64::aarch64::reduce_pair_pmull4`]), replacing the +/// old 10-op shift-XOR fold chain. +/// +/// # Safety +/// Requires the `aes` target feature; `top`/`bot` must each point at two +/// readable+writable F64 values. +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +#[inline] +#[target_feature(enable = "aes")] +unsafe fn butterfly_lane_pair_neon(top: *mut F64, bot: *mut F64, twiddle: u64) { + use core::arch::aarch64::*; + use primitives::field::gf2_64::aarch64::reduce_pair_pmull4; + // SAFETY: caller guarantees the pointees; F64 is repr(transparent) u64. + unsafe { + let u = vld1q_u64(top as *const u64); + let v = vld1q_u64(bot as *const u64); + // Products v_lane * twiddle: PMULL on the low lanes, PMULL2 on the + // highs (the dup is loop-invariant and hoisted after inlining). + let tw = vdupq_n_u64(twiddle); + let p0: uint64x2_t = core::mem::transmute(vmull_p64(vgetq_lane_u64::<0>(v), twiddle)); + let p1: uint64x2_t = core::mem::transmute(vmull_high_p64( + core::mem::transmute::(v), + core::mem::transmute::(tw), + )); + let prod = reduce_pair_pmull4(p0, p1); + let new_u = veorq_u64(u, prod); + let new_v = veorq_u64(v, new_u); + vst1q_u64(top as *mut u64, new_u); + vst1q_u64(bot as *mut u64, new_v); + } +} + +#[inline] +fn log2_pow2(n: usize) -> usize { + assert!(n.is_power_of_two() && n > 0, "length must be a positive power of 2"); + n.trailing_zeros() as usize +} + +#[cfg(test)] +mod tests { + use super::*; + + fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// The NTT of the coefficient vector of the constant-1 polynomial in the + /// novel basis must be all-ones (Ŵ_0 normalization); more usefully, the + /// forward transform must equal naive per-point evaluation of the novel + /// basis expansion. We check forward∘inverse = id and scalar==interleaved + /// ==parallel instead, plus linearity. + #[test] + fn inverse_roundtrip_and_variants_agree() { + let ntt = AdditiveNttF64::standard(12); + let mut s = 1u64; + for log_d in [1usize, 3, 6, 10] { + let n = 1usize << log_d; + let orig: Vec = (0..n).map(|_| F64(splitmix64(&mut s))).collect(); + + let mut a = orig.clone(); + ntt.forward_transform_scalar(&mut a); + let mut b = orig.clone(); + ntt.forward_transform_interleaved_scalar_from_layer(&mut b, 1, 0); + assert_eq!(a, b, "interleaved(1 lane) == scalar at log_d={log_d}"); + let mut c = orig.clone(); + ntt.forward_transform_interleaved_parallel_from_layer(&mut c, 1, 0); + assert_eq!(a, c, "parallel == scalar at log_d={log_d}"); + + ntt.inverse_transform(&mut a); + assert_eq!(a, orig, "inverse roundtrip at log_d={log_d}"); + } + } + + /// The parallel interleaved path must match the scalar reference at sizes + /// that actually reach the fused multi-layer passes. + /// + /// `interleaved_lanes_are_independent_ntts` runs at `log_d = 7`, below the + /// driver's `log_d < 8` bail-out, so it only ever exercises the scalar + /// path. These shapes give `n_top >= 3`, which is what selects the + /// radix-8 fused pass, and cover a non-zero `start_layer` because the + /// commit path enters at `log_inv_rate`. + #[test] + fn interleaved_parallel_matches_scalar_at_fused_sizes() { + let mut s = 0xC0FFEEu64; + for log_d in [12usize, 14] { + for lanes in [8usize, 64] { + for start_layer in [0usize, 1] { + let ntt = AdditiveNttF64::standard(log_d); + let n = (1usize << log_d) * lanes; + let original: Vec = (0..n).map(|_| F64(splitmix64(&mut s))).collect(); + + let mut want = original.clone(); + ntt.forward_transform_interleaved_scalar_from_layer(&mut want, lanes, start_layer); + let mut got = original.clone(); + ntt.forward_transform_interleaved_from_layer(&mut got, lanes, start_layer); + + assert_eq!( + got, want, + "parallel != scalar at log_d={log_d}, lanes={lanes}, start_layer={start_layer}" + ); + } + } + } + } + + #[test] + fn interleaved_lanes_are_independent_ntts() { + let ntt = AdditiveNttF64::standard(10); + let mut s = 2u64; + let log_d = 7; + let n = 1usize << log_d; + for lanes in [1usize, 2, 4, 8, 64] { + // SoA buffer + per-lane copies. + let mut soa = vec![F64::ZERO; n * lanes]; + let mut per_lane: Vec> = vec![vec![F64::ZERO; n]; lanes]; + for pos in 0..n { + for lane in 0..lanes { + let v = F64(splitmix64(&mut s)); + soa[pos * lanes + lane] = v; + per_lane[lane][pos] = v; + } + } + ntt.forward_transform_interleaved(&mut soa, lanes); + for (lane, lane_data) in per_lane.iter_mut().enumerate() { + ntt.forward_transform_scalar(lane_data); + for pos in 0..n { + assert_eq!(soa[pos * lanes + lane], lane_data[pos]); + } + } + } + } + + #[test] + fn linearity() { + let ntt = AdditiveNttF64::standard(8); + let mut s = 3u64; + let n = 256; + let a: Vec = (0..n).map(|_| F64(splitmix64(&mut s))).collect(); + let b: Vec = (0..n).map(|_| F64(splitmix64(&mut s))).collect(); + let sum: Vec = a.iter().zip(&b).map(|(x, y)| *x + *y).collect(); + let mut ta = a.clone(); + let mut tb = b.clone(); + let mut tsum = sum.clone(); + ntt.forward_transform_scalar(&mut ta); + ntt.forward_transform_scalar(&mut tb); + ntt.forward_transform_scalar(&mut tsum); + for i in 0..n { + assert_eq!(tsum[i], ta[i] + tb[i]); + } + } +} diff --git a/crates/pcs/src/ntt/inv_table.rs b/crates/pcs/src/ntt/inv_table.rs index cc3a449d..dfb92deb 100644 --- a/crates/pcs/src/ntt/inv_table.rs +++ b/crates/pcs/src/ntt/inv_table.rs @@ -1,4 +1,4 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. //! §2.1 single-table collapse of the LDE matrix `M = fwd_NTT_Λ ∘ inv_NTT_S`. //! //! Background: the URM round-1 needs to map each `ell`-bit row of the boolean @@ -18,8 +18,8 @@ //! Storage: 256 × ell bytes (16 KB at k=6, 32 KB at k=7) — fits in L1. //! Lookups per row: n_chunks (= ell/8), each load is `ell` contiguous bytes. -use primitives::field::F8; use crate::ntt::AdditiveNttGf8; +use primitives::field::F8; #[derive(Clone, Debug)] pub struct InvNttTableByteSingleGf8 { @@ -41,10 +41,7 @@ impl InvNttTableByteSingleGf8 { let ell = 1usize << k; assert!(ell >= 8, "ell must be ≥ 8 so n_chunks ≥ 1"); let n_chunks = ell / 8; - assert!( - n_chunks <= 16, - "n_chunks must fit the i'/chunk XOR encoding" - ); + assert!(n_chunks <= 16, "n_chunks must fit the i'/chunk XOR encoding"); let mut data = vec![F8::ZERO; 256 * ell]; @@ -81,12 +78,7 @@ impl InvNttTableByteSingleGf8 { } } - Self { - k, - ell, - n_chunks, - data, - } + Self { k, ell, n_chunks, data } } /// Raw pointer to the table data (`256 × ell` bytes, row-major). Used by @@ -192,7 +184,7 @@ impl InvNttTableByteSingleGf8 { } } - /// SSE2 variant of `apply` — the x86 twin of [`Self::apply_neon_unchecked`]. + /// SSE2 variant of `apply` — the x86 twin of `apply_neon_unchecked`. /// Same 16-byte-chunk structure; the odd-`b` within-chunk half-swap is /// `_mm_shuffle_epi32::<0b01_00_11_10>` (swap the two 64-bit halves). /// @@ -336,9 +328,7 @@ mod tests { let mut rng = Rng::new(100 + k as u64); for _ in 0..32 { - let bytes: Vec = (0..n_chunks) - .map(|_| (rng.next_u64() & 0xff) as u8) - .collect(); + let bytes: Vec = (0..n_chunks).map(|_| (rng.next_u64() & 0xff) as u8).collect(); let mut out_scalar = vec![F8::ZERO; ell]; let mut out_neon = vec![F8::ZERO; ell]; table.apply_scalar(&bytes, &mut out_scalar); @@ -367,9 +357,7 @@ mod tests { let mut rng = Rng::new(100 + k as u64); for _ in 0..32 { - let bytes: Vec = (0..n_chunks) - .map(|_| (rng.next_u64() & 0xff) as u8) - .collect(); + let bytes: Vec = (0..n_chunks).map(|_| (rng.next_u64() & 0xff) as u8).collect(); let mut out_scalar = vec![F8::ZERO; ell]; let mut out_sse2 = vec![F8::ZERO; ell]; table.apply_scalar(&bytes, &mut out_scalar); diff --git a/crates/pcs/src/pack.rs b/crates/pcs/src/pack.rs index 09c75aed..efbd6825 100644 --- a/crates/pcs/src/pack.rs +++ b/crates/pcs/src/pack.rs @@ -1,33 +1,33 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. -//! Bit-witness packing into F_{2^128} for the PCS commitment phase. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +//! Bit-witness packing into K = F_{2^64} for the 64-bit transition PCS. //! -//! The witness `z : {0,1}^m → {0,1}` is laid out as a flat 2^m-length bool -//! array. Packing groups the **first** `LOG_PACKING = 7` boolean coordinates -//! into one F_{2^128} element, leaving an array of 2^(m−7) packed elements -//! indexed by the remaining m−7 outer coords. +//! The witness +//! `z : {0,1}^m -> {0,1}` is laid out as a flat 2^m-length bool array, and +//! packing groups the **first** `LOG_PACKING = 6` boolean coordinates into +//! one F_{2^64} element, leaving `2^(m-6)` packed words indexed by the +//! remaining m-6 outer coords. //! -//! Layout convention: for packed index `i_rest ∈ {0..2^(m−7)}` and bit position -//! `i_skip ∈ {0..128}`, +//! Layout convention: for packed index `i_rest` and bit position `i`, //! ```text -//! bit i_skip of out[i_rest] == z[i_rest * 128 + i_skip] +//! bit i of out[i_rest] == z[i_rest * 64 + i] //! ``` -//! where "bit i_skip of an F_{2^128} element" means the i_skip-th coordinate of -//! its natural polynomial basis decomposition (i.e. the i_skip-th bit of the -//! u128 representation, little-endian). +//! where "bit i of an F_{2^64} element" is the i-th coordinate of its +//! polynomial-basis decomposition (bit i of the u64, little-endian). //! -//! This matches the convention used in the [DP24] ring-switching reduction: -//! `s_hat_v[i_skip] = ẑ_{i_skip}(x_mlv)`, the multilinear extension of the -//! `i_skip`-th bit-slice of the witness. -//! -//! [DP24]: https://eprint.iacr.org/2024/504 +//! This matches the packing basis of the generalized ring-switching reduction +//! ([`super::ring_switch`]): `s_hat_v[i]` is the MLE of the i-th bit-slice +//! of the witness, and the i-th bit-slice is exactly bit i of every word. + +use primitives::field::F64; -use primitives::field::F128; +/// `log_2` of the packing width. F_{2^64} holds 64 bits = 2^6. +pub const LOG_PACKING: usize = 6; -/// `log_2` of the packing width. F_{2^128} holds 128 bits = 2^7. -pub const LOG_PACKING: usize = 7; +/// Packing width (number of bits per F_{2^64} element). +pub const PACKING_WIDTH: usize = 1 << LOG_PACKING; -/// Pack a Boolean witness `z` of length `2^m` into `2^(m − LOG_PACKING)` -/// F_{2^128} elements. +/// Pack a Boolean witness `z` of length `2^m` into `2^(m - LOG_PACKING)` +/// F_{2^64} elements. /// /// See module docs for the layout convention. /// @@ -35,7 +35,7 @@ pub const LOG_PACKING: usize = 7; /// /// - if `z.len() != 1 << m` /// - if `m < LOG_PACKING` -pub fn pack_witness(z: &[bool], m: usize) -> Vec { +pub fn pack_witness(z: &[bool], m: usize) -> Vec { use rayon::prelude::*; assert_eq!(z.len(), 1usize << m, "z length must be 2^m"); assert!( @@ -46,8 +46,9 @@ pub fn pack_witness(z: &[bool], m: usize) -> Vec { // `bool` is guaranteed 1 byte holding 0x00/0x01, so 8 bools read as one // little-endian u64 pack to an LSB-first byte with one multiply: - // byte 7 of `x * 0x0102040810204080` is Σ_r b_r·2^r (each lower product - // byte sums distinct powers of two ≤ 0xFE — no carry into byte 7). + // byte 7 of `x * 0x0102040810204080` is the sum of b_r * 2^r (each lower + // product byte sums distinct powers of two <= 0xFE, so nothing carries + // into byte 7). // SAFETY: same length, and any &[bool] is a valid &[u8]. let bytes: &[u8] = unsafe { core::slice::from_raw_parts(z.as_ptr() as *const u8, z.len()) }; #[inline] @@ -61,10 +62,7 @@ pub fn pack_witness(z: &[bool], m: usize) -> Vec { } let one = |i_rest: usize| { let base = i_rest << LOG_PACKING; - F128 { - lo: pack64(&bytes[base..base + 64]), - hi: pack64(&bytes[base + 64..base + 128]), - } + F64(pack64(&bytes[base..base + PACKING_WIDTH])) }; // Parallel for real witnesses; sequential below the dispatch-overhead // floor (tiny test instances). @@ -75,60 +73,70 @@ pub fn pack_witness(z: &[bool], m: usize) -> Vec { } } +/// Inverse of [`pack_witness`]: unpack F_{2^64} elements back to a Boolean +/// witness of length `2^m`. +/// +/// Round-trips with [`pack_witness`] by construction. +#[cfg(test)] +pub fn unpack_witness(packed: &[F64], m: usize) -> Vec { + let n_packed = 1usize << (m - LOG_PACKING); + assert_eq!(packed.len(), n_packed, "packed length must be 2^(m - LOG_PACKING)"); + let mut out = vec![false; 1usize << m]; + for (i_rest, elem) in packed.iter().enumerate() { + let base = i_rest << LOG_PACKING; + for r in 0..PACKING_WIDTH { + out[base | r] = (elem.0 >> r) & 1 == 1; + } + } + out +} + #[cfg(test)] mod tests { use super::*; - #[test] - fn pack_layout_matches_natural_bit_order() { - // For m = LOG_PACKING (= 7): exactly one packed element, holding the - // entire 128-bit witness in natural u128 bit order. - let mut z = vec![false; 128]; - // Set a known bit pattern: bits at positions 0, 1, 5, 63, 64, 127. - for &i in &[0usize, 1, 5, 63, 64, 127] { - z[i] = true; - } - let packed = pack_witness(&z, LOG_PACKING); - assert_eq!(packed.len(), 1); - let expected = F128 { - lo: (1u64 << 0) | (1u64 << 1) | (1u64 << 5) | (1u64 << 63), - hi: (1u64 << 0) | (1u64 << 63), - }; - assert_eq!(packed[0], expected); + fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn rand_bits(m: usize, seed: u64) -> Vec { + let mut s = seed; + (0..1usize << m).map(|_| splitmix64(&mut s) & 1 == 1).collect() } #[test] - fn pack_independent_chunks() { - // Two adjacent 128-bit chunks should pack independently — flipping a - // bit in one chunk affects only that chunk. - let z = vec![true; 256]; - let packed = pack_witness(&z, 8); - assert_eq!(packed.len(), 2); - assert_eq!( - packed[0], - F128 { - lo: u64::MAX, - hi: u64::MAX - } - ); - assert_eq!( - packed[1], - F128 { - lo: u64::MAX, - hi: u64::MAX - } - ); + fn roundtrip() { + for (m, seed) in [(6usize, 1u64), (7, 2), (10, 3), (13, 4)] { + let z = rand_bits(m, seed); + let packed = pack_witness(&z, m); + assert_eq!(packed.len(), 1 << (m - LOG_PACKING)); + assert_eq!(unpack_witness(&packed, m), z, "roundtrip failed at m={m}"); + } } + /// Bit-level layout: bit i of word i_rest is z[i_rest * 64 + i]. #[test] - #[should_panic(expected = "witness too small")] - fn rejects_undersized_witness() { - let z = vec![false; 64]; // m = 6 < LOG_PACKING = 7 - let _ = pack_witness(&z, 6); + fn bit_layout() { + let m = 9; + let z = rand_bits(m, 5); + let packed = pack_witness(&z, m); + for i_rest in 0..packed.len() { + for i in 0..PACKING_WIDTH { + assert_eq!( + (packed[i_rest].0 >> i) & 1 == 1, + z[(i_rest << LOG_PACKING) | i], + "bit ({i_rest}, {i}) disagrees with the flat layout" + ); + } + } } } -/// Use [`PaddingSpec::dense`] when the witness has no padding holes. +/// Describes zero padding within each logical witness block. #[derive(Clone, Copy, Debug)] pub struct PaddingSpec { pub k_log: usize, @@ -136,8 +144,7 @@ pub struct PaddingSpec { } impl PaddingSpec { - /// "No padding": every bit of the witness is treated as useful. Equivalent - /// to the legacy URM path with no skipping. + /// Treat every bit as useful. pub fn dense(m: usize) -> Self { Self { k_log: m, diff --git a/crates/pcs/src/ring_switch.rs b/crates/pcs/src/ring_switch.rs index b007dd52..22f3fe5a 100644 --- a/crates/pcs/src/ring_switch.rs +++ b/crates/pcs/src/ring_switch.rs @@ -1,974 +1,358 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/binius-zk/binius64 (`eval_rs_eq`), Apache-2.0. // Copyright 2025 The Binius Developers // Copyright 2025 Irreducible, Inc. // Modifications copyright 2026 Succinct Labs, Benedikt Bunz, William Wang // SPDX-License-Identifier: Apache-2.0 OR MIT // -// The verifier's polylog `eval_rs_eq` helper is ported from binius64's -// `crates/verifier/src/ring_switch.rs` -// (https://github.com/binius-zk/binius64). The rest of this module (the -// prover-side reduction adapted for the φ_8 LCH basis) is original to Flock. +// The DP24 iterative `eval_rs_eq` is ported from binius64. The module is +// the rectangular (f = 64, e = 192) generalization described in +// the ring-switching-generalized note. -//! Ring-switching reduction (DP24-style, adapted for the φ_8 LCH basis). +//! Ring-switching reduction for the 64-bit transition: F_2 to K = GF(2^64) +//! packing, opened over E = GF(2^192) (the tower [`F192`]). //! -//! Converts the zerocheck's claim `ẑ_skip(z_skip, x_outer) = v` into a sumcheck -//! claim over the packed multilinear `f_packed` with a transparent multilinear -//! `rs_eq_ind`, discharged by the stacked Ligerito opening. +//! With f = 64 (packing degree over F_2) and e = 192 (opening degree), this +//! converts one evaluation claim on +//! the bit-witness MLE at an E-point into a Ligerito sumcheck claim on the +//! packed multilinear (a `Vec`, one word per 64 bits, see +//! [`super::pack`]) against a transparent E-valued weight vector +//! `rs_eq_ind`. //! -//! ## Non-novelty basis: only affects the claim-check step +//! ## Rectangular shape //! -//! Binius's DP24 ring-switching uses tensor-product (`eq_ind`) weights for the -//! verifier's claim check. That requires the prefix's LCH-Lagrange to factor -//! as `eq(x_skip, i_skip)`, which holds only for the *novelty basis* of the -//! subspace. +//! - **Rectangular shape**: `s_hat_v` has 64 entries (one per packing bit), +//! each an E element; its tensor-algebra transpose `s_hat_u = (t_i)` has +//! 192 K-entries. A random `F_2`-linear map batches all coordinates directly, +//! without padding them to a 256-entry Boolean cube. +//! - **No "7 = 6 + 1" prefix split**: with 64-bit packing the packed prefix +//! is exactly the 6-bit skip domain, and the old 7th bit is an ordinary +//! suffix coordinate of the packed witness (which has `2^(m-6)` words). +//! - **Generalized prefix weights**: the consumed claim is +//! `claim == sum_{i in 0..64} prefix_weights[i] * s_hat_v[i]`. For a plain +//! multilinear point claim the weights are the eq tensor of the 6 prefix +//! coords ([`eq_prefix_weights`]); for flock's univariate-skip claim (whose +//! first coordinate ranges over the phi_8 Lagrange domain, not the boolean +//! cube) the caller passes the 64 phi_8 Lagrange weights +//! `lagrange_weights_naive(6, z_skip)`. +//! This module never looks inside the weights, so flock's `z_skip` flows +//! through unchanged. //! -//! Our zerocheck uses the φ_8 image of {1,2,4,…,32} as the 6-dim LCH basis. -//! That basis is **not** a novelty basis (verified at k=2: the ratio of -//! Lagrange values doesn't satisfy the tensor identity), so the 64 weights -//! `ν_φ8(i_skip)(z_skip)` are not tensor-factorizable. +//! ## Protocol (prover) //! -//! Resolution: replace the verifier's claim check with **direct** Lagrange -//! weights (computed via [`lagrange_weights_naive`]); every other component of -//! the reduction (`s_hat_v`, `s_hat_u`, the sumcheck target `T`, `rs_eq_ind`) is -//! independent of the prefix and stays identical to Binius. +//! 1. Send `s_hat_v[i] = sum_y eq(r_suffix, y) * bit_i(packed[y])`, the MLE +//! of the i-th bit-slice at the suffix point (i in 0..64, values in E). +//! 2. Verifier checks `claim == sum_i prefix_weights[i] * s_hat_v[i]`. +//! 3. Sample six challenges in E and compose the maps +//! `v <- v + f_t v^(2^d_t)` for `d_t = 32, 16, 8, 4, 2, 1`. For the +//! coordinate basis `(b_i)`, define `coord_weights[i] = Phi(b_i)`. Transpose +//! `s_hat_v` to `t_i = s_hat_u[i] in K` (see +//! [`super::tensor_algebra::transpose_s_hat`]); the batched target is +//! `sumcheck_claim = sum_i Phi(b_i) * t_i` (K x E via `mul_base`). +//! 4. Both sides define the transparent weights +//! `rs_eq_ind[y] = Phi(eq(r_suffix, y))` where `Phi : E -> E` is the +//! composed map above. Completeness: +//! `sum_y rs_eq_ind[y] * packed[y] == sumcheck_claim`, which is exactly +//! the claim shape [`super::ligerito::recursive_prover_with_basis`] +//! proves (with `b_initial = rs_eq_ind`, `target = sumcheck_claim`). +//! A nonzero discrepancy gives a nonzero polynomial in the six challenges; +//! its total degree is below `2^32`, hence its failure probability is below +//! `2^-160` before the Ligerito list-size factor. //! //! ## Prover vs. verifier paths for `rs_eq_ind` //! -//! - **Prover side** ([`prove_batched_padded_with_precomputed`]): materializes -//! `rs_eq_ind` per claim (dense, deferred-dense, or sparse — [`RsEqInd`]); -//! the γ-combined vector becomes the opening's basis, so the prover does -//! need the full `2^(m-7)` entries. -//! - **Verifier side** ([`verify_bind`] + [`eval_rs_eq`]): never materializes -//! `rs_eq_ind`. Instead, evaluates `MLE(rs_eq_ind)(c)` at the opening's -//! final challenge point in `O((m-7) · 128)` field ops via the linearized -//! trace form ([`linearized_eq_coeffs`]). Polylog in the witness size. +//! - The prover keeps the equality tensor factored, folds each claim into a +//! small byte table, and combines the claims directly into one dense PCS +//! weight. It never materializes a dense vector per claim. +//! - [`eval_rs_eq`] lets the verifier avoid materializing the vector entirely: +//! its MLE at the Ligerito final point is evaluated in +//! `O((m-6) * 192^2)` bit-ops plus `O((m-6) * 192)` E-multiplications via +//! the DP24 tensor-algebra iterative algorithm (DP24 section 1.3 Figure 3). //! //! [DP24]: -//! -//! ## Layout (for m-bit witness, F_{2^128} packing with LOG_PACKING = 7) -//! -//! Zerocheck output: `(z_skip ∈ F, x_outer ∈ F^{m−6})` with claim `v`. -//! -//! After translation: -//! - **prefix bits 0..6**: weighted by `ν_φ8(·)(z_skip)` (the 64 Lagrange weights). -//! - **prefix bit 6**: weighted by `eq(x_outer[0], ·)`. -//! - **suffix coords**: `x_outer[1..]`, length `m − 7`. -//! -//! The packed witness has `2^(m−7)` F_{2^128} elements indexed by the suffix. -//! `s_hat_v` has 128 entries indexed by the 7-bit prefix. +use fiat_shamir::Sponge; use primitives::bits::transpose_8x8_bits; -use crate::{ProverState, VerifierState}; -use primitives::field::F128; -use crate::pack::PaddingSpec; -use primitives::multilinear::lagrange_weights_naive; -use primitives::multilinear::build_eq; - -use super::pack::LOG_PACKING; - -/// Per-block padding descriptor in F_{2^128} units. Computed once from a bit- -/// level [`PaddingSpec`] and reused across the fold kernels: any chunk whose -/// index modulo `chunks_per_block` is ≥ `useful_chunks_per_block` is fully -/// inside the zero-padded suffix of every block and can be skipped. -#[derive(Clone, Copy, Debug)] -struct ChunkPadding { - /// `chunks_per_block - 1` for fast `idx % chunks_per_block` via AND; - /// `usize::MAX` (= "no skip") when there is only one block (e.g. dense - /// paddings). - chunk_in_block_mask: usize, - /// Index of the first fully-padding chunk within each block. - useful_chunks_per_block: usize, -} - -impl ChunkPadding { - /// Build the per-chunk skip table for a given F128-chunk width - /// (e.g. `chunk_width = 8` for the 8-wide MFR path). Returns a "no skip" - /// descriptor if either (a) the spec covers the entire packed witness as - /// one block, or (b) every chunk in a block is at least partially useful. - fn new(padding: &PaddingSpec, chunk_width: usize) -> Self { - // Block size in F128 elements = 2^(k_log - LOG_PACKING). - if padding.k_log <= LOG_PACKING { - // Block smaller than one F128 — no per-block structure to exploit. - return Self::no_skip(); - } - let block_size_f128 = 1usize << (padding.k_log - LOG_PACKING); - if block_size_f128 < chunk_width { - return Self::no_skip(); - } - let chunks_per_block = block_size_f128 / chunk_width; - let useful_f128 = padding.useful_bits_per_block.div_ceil(1 << LOG_PACKING); - let useful_chunks_per_block = useful_f128.div_ceil(chunk_width).min(chunks_per_block); - if useful_chunks_per_block == chunks_per_block { - return Self::no_skip(); - } - debug_assert!(chunks_per_block.is_power_of_two()); - Self { - chunk_in_block_mask: chunks_per_block - 1, - useful_chunks_per_block, - } - } - - fn no_skip() -> Self { - Self { - chunk_in_block_mask: usize::MAX, - useful_chunks_per_block: usize::MAX, - } - } - - /// True iff the chunk at this global index is fully inside padding. - #[inline(always)] - fn skip(&self, chunk_idx: usize) -> bool { - (chunk_idx & self.chunk_in_block_mask) >= self.useful_chunks_per_block - } -} - -/// Build the 128-entry weights vector for the verifier's ring-switching claim -/// check, given the zerocheck's `z_skip` (univariate-skip coord, absorbs 6 -/// boolean coords via the φ_8 basis) and `x_outer_0` (the 7th prefix bit, a -/// fresh F_{2^128} multilinear coord). +use primitives::field::{F64, F192}; +use serde::{Deserialize, Serialize}; + +use super::ligerito::{build_eq_table_ext, inner_product_base_ext}; +use super::pack::{LOG_PACKING, PACKING_WIDTH}; +use super::tensor_algebra::{DEGREE_E, TensorAlgebraE, transpose_s_hat}; + +/// Total degree of the six-challenge composed batching map. This is the +/// conservative degree used by the Ligerito list-size soundness accounting. +pub const RING_SWITCH_SOUNDNESS_DEGREE: usize = + (1usize << 31) + (1usize << 15) + (1usize << 7) + (1usize << 3) + (1usize << 1) + 1; + +/// Frobenius shifts in the order in which the two-term maps are composed. +/// Descending order bounds every challenge's exponent by `2^31`. +pub const COMPOSITION_SHIFTS: [usize; 6] = [32, 16, 8, 4, 2, 1]; + +/// Number of Frobenius terms in the batching map [`build_coordinate_weights`] +/// builds. It is the F_2-dimension of `K`, and that is the floor: the weights +/// must separate any nonzero error on the 192 transposed `K`-columns, which is +/// `|S|` `E`-equations, i.e. `3·|S|` `K`-equations, in `192` `K`-unknowns — with +/// `3·|S| < 192` a nonzero error lies in the kernel for EVERY coefficient +/// choice and passes with probability one. See [`build_coordinate_weights`]. +pub const LINEARIZED_TERMS: usize = PACKING_WIDTH; + +/// The coordinate batching weights: `weights[w] = Phi(b_w)`, where `b_w` is the +/// `w`-th `F_2`-coordinate basis element of `E` (the order [`transpose_s_hat`] +/// produces). Starting from `v`, the map composes /// /// ```text -/// weights[i] = ν_φ8(i & 63)(z_skip) · eq(x_outer_0, (i >> 6) & 1) -/// for i ∈ {0..128} +/// v <- v + f_t · v^(2^shift_t), shift_t = 32, 16, 8, 4, 2, 1. /// ``` /// -/// `i & 63` selects the low 6 bits (LCH dimensions); `(i >> 6) & 1` is the 7th -/// bit (a standard multilinear coord). -pub fn build_claim_weights(z_skip: F128, x_outer_0: F128) -> Vec { - const K_SKIP: usize = 6; - let lambda = lagrange_weights_naive(K_SKIP, z_skip); // length 64 - debug_assert_eq!(lambda.len(), 1 << K_SKIP); - - let eq_lo = F128::ONE + x_outer_0; // eq(x_outer_0, 0) - let eq_hi = x_outer_0; // eq(x_outer_0, 1) - - let n = 1 << LOG_PACKING; // 128 - let mut weights = Vec::with_capacity(n); - // Layout: i ∈ {0..64} → bit-6 = 0 branch (eq_lo); i ∈ {64..128} → bit-6 = 1. - for i in 0..n { - let i_lo = i & 63; - let bit_6 = (i >> 6) & 1; - let eq_b6 = if bit_6 == 1 { eq_hi } else { eq_lo }; - weights.push(lambda[i_lo] * eq_b6); - } - weights -} - -/// Padding-aware multi-claim fold. Routes the k=2 MFR fast -/// paths through their `_padded` kernels; the scalar bit-scan fallback (k ≠ 2 -/// or non-divisible len) is untouched — those `m` are tiny anyway. -pub fn fold_1b_rows_multi_padded( - packed_witness: &[F128], - suffix_tensors: &[&[F128]], - padding: &PaddingSpec, -) -> Vec> { - use rayon::prelude::*; - let k = suffix_tensors.len(); - let n = 1 << LOG_PACKING; - assert!( - suffix_tensors - .iter() - .all(|t| t.len() == packed_witness.len()) - ); - - let zero_acc = || vec![vec![F128::ZERO; n]; k]; - - // The k=2 case (one pair of outers) is the hot path used by `open_batch` - // for zerocheck + lincheck claims. Method-of-four-Russians fold (ported - // from Binius): process several elements at a time with subset-sum table - // lookups per output bit, eliminating the scalar bit-scan's data-dependent - // control flow. The 16-wide variant groups 16 elements (four 4-element - // tables, 16-bit masks) so each acc entry is touched once per 16 elements, - // halving acc RMW traffic (the fold is LSU-bound) for ~1.25× over 8-wide. - // We run two *independent* 1-way 16-wide folds rather than one fused 2-way - // fold: the fused kernel's two accumulators + eight tables cause register - // pressure that eats most of the 16-wide win, and the shared bit-transpose - // it would save is nearly free. Falls back to the fused 8-wide → 4-wide → - // scalar as divisibility drops (only at toy m). - if k == 2 { - if packed_witness.len().is_multiple_of(16) { - let a0 = - fold_1b_rows_1way_mfr_16wide_padded(packed_witness, suffix_tensors[0], padding); - let a1 = - fold_1b_rows_1way_mfr_16wide_padded(packed_witness, suffix_tensors[1], padding); - return vec![a0, a1]; - } - if packed_witness.len().is_multiple_of(8) { - let (a0, a1) = fold_1b_rows_2way_mfr_8wide_padded( - packed_witness, - suffix_tensors[0], - suffix_tensors[1], - padding, - ); - return vec![a0, a1]; - } - if packed_witness.len().is_multiple_of(4) { - let (a0, a1) = fold_1b_rows_2way_mfr_padded( - packed_witness, - suffix_tensors[0], - suffix_tensors[1], - padding, - ); - return vec![a0, a1]; - } - } - - packed_witness - .par_iter() - .enumerate() - .fold(zero_acc, |mut acc, (i_rest, elem)| { - // Single bit-scan, write into all k accumulators. - let mut lo = elem.lo; - while lo != 0 { - let r = lo.trailing_zeros() as usize; - for (j, t) in suffix_tensors.iter().enumerate() { - acc[j][r] += t[i_rest]; - } - lo &= lo - 1; - } - let mut hi = elem.hi; - while hi != 0 { - let r = hi.trailing_zeros() as usize; - for (j, t) in suffix_tensors.iter().enumerate() { - acc[j][64 | r] += t[i_rest]; - } - hi &= hi - 1; - } - acc - }) - .reduce(zero_acc, |mut a, b| { - for (av, bv) in a.iter_mut().zip(b.iter()) { - for (avi, bvi) in av.iter_mut().zip(bv.iter()) { - *avi += *bvi; - } - } - a - }) -} - -/// Parallel `build_eq` for ring-switching's suffix tensors. +/// The result is `F_2`-linear, so +/// `sum_w weights[w]·t_w = sum_j x^j·Phi(y_j)` for the row +/// view `y` and the column view `t` of the same tensor-algebra element. That +/// identity is what lets a verifier evaluate the batched claim from `Phi`'s +/// six challenges instead of the 192 weights; the recursion guest does exactly +/// that. Expanding the composition puts a distinct monomial at every Frobenius +/// exponent `0..64`: writing `k = sum_p k_p·2^(5-p)` for the binary digits of +/// `k`, the coefficient is `C_k = prod_{p : k_p = 1} f_p^(2^(k mod 2^(5-p)))`, +/// which is what the guest's coefficient table builds. Applying the composed +/// form directly costs only 63 squarings and six multiplications. /// -/// Now just [`primitives::multilinear::build_eq`], which parallelizes its own -/// doubling levels. Retained as the local name used throughout this module. -#[inline] -pub(crate) fn build_eq_parallel(r: &[F128]) -> Vec { - primitives::multilinear::build_eq(r) -} - -/// Tensor-factored `build_eq`: split the point `r` (length `n`) into a low -/// part `r[..n_lo]` and a high part `r[n_lo..]`, returning the two smaller -/// eq-tables `(eq_lo, eq_hi)` of lengths `2^n_lo` and `2^(n - n_lo)`. +/// ## Soundness /// -/// The full tensor factors **exactly** (GF(2^128) is a field — multiply is -/// associative and has no rounding): +/// Let `delta != 0` be the prover's error on the transposed columns, fixed +/// before the six challenges (`s_hat_v` is bound first). The check misses it iff +/// `sum_w Phi(b_w)·delta_w = sum_{k<64} C_k(f)·V_k = 0` with +/// `V_k = sum_w b_w^(2^k)·delta_w`. Writing `w = 64j + i` and `b_w = x^i·Y^j`, /// /// ```text -/// build_eq_parallel(r)[i] == eq_lo[i & (2^n_lo - 1)] * eq_hi[i >> n_lo] +/// V_k = sum_{j<3} (Y^(2^k))^j · R_{j,k}, R_{j,k} = sum_{i<64} x^(i·2^k)·delta_{64j+i} in K. /// ``` /// -/// because round `j` of `build_eq` splits on bit `j` of the index and bit `j` -/// selects `r[j]`. So the low `n_lo` index bits depend only on `r[..n_lo]` and -/// the high bits only on `r[n_lo..]`. -/// -/// Materializing the two factors costs `2^n_lo + 2^(n - n_lo)` entries instead -/// of `2^n`. Consumers either reconstruct each full entry on demand as one GF -/// multiply ([`fold_b128_elems_split`]) or never form it at all when the -/// consumer is linear in the tensor ([`fold_1b_rows_split`]). -pub fn build_eq_split(r: &[F128], n_lo: usize) -> (Vec, Vec) { - assert!(n_lo <= r.len()); - let eq_lo = build_eq_parallel(&r[..n_lo]); - let eq_hi = build_eq_parallel(&r[n_lo..]); - (eq_lo, eq_hi) -} - -/// Pick the low-split width `n_lo` for a suffix tensor of length `2^n`. -/// Balanced near `n/2` so both factors are ~`2^(n/2)` (L1/L2-resident), and -/// clamped to `[4, n]` so the low block `2^n_lo` is a whole number of 16-wide -/// MFR chunks (`n_lo ≥ 4` ⇒ block ≥ 16). The high part drives block-level -/// parallelism (`2^(n - n_lo)` blocks). Only meaningful for `n ≥ 4` (the -/// split path requires `len` divisible by 16). -pub fn split_n_lo(n: usize) -> usize { - (n / 2).clamp(4, n) -} - -/// Build the 16-entry subset-sum lookup table over 4 F128 elements. -/// -/// `sums[mask]` = `Σ_{k=0..4 : bit_k(mask) = 1} elems[k]` for `mask ∈ 0..16`. -/// Cost: 15 F128 additions (8 + 4 + 2 + 1) via the standard doubling pattern. -#[inline(always)] -fn subset_sums_4(elems: [F128; 4]) -> [F128; 16] { - let mut sums = [F128::ZERO; 16]; - // After processing elem[i], sums[0..2^(i+1)] are populated with the - // subset sums over elems[0..=i]. - for (i, &e) in elems.iter().enumerate() { - let half = 1 << i; - // sums[half..2*half] = sums[0..half] + e - for k in 0..half { - sums[half + k] = sums[k] + e; +/// `Y^(2^k)` is not in `K` (squaring is a bijection of `K`, so `Y^(2^k) in K` +/// would force `Y in K` and `E = K[Y] = K`), and `[E:K] = 3` is prime, so +/// `{1, Y^(2^k), Y^(2·2^k)}` is a `K`-basis: `V_k = 0` forces every +/// `R_{j,k} = 0`. At fixed `j` those 64 equations say that the polynomial +/// `D_j(U) = sum_{i<64} delta_{64j+i}·U^i`, of degree at most 63, vanishes at +/// `x, x^2, x^4, ..., x^(2^63)`. Those are 64 distinct points, since `x` has +/// degree 64 over `F_2`, so `D_j = 0` and `delta = 0`. Hence some `V_k != 0`. +/// The 64 `C_k` are distinct monomials, so the discrepancy is a nonzero +/// polynomial in the challenges. In descending shift order its total degree is +/// [`RING_SWITCH_SOUNDNESS_DEGREE`], below `2^32`. +fn apply_composed_map(mut value: F192, challenges: &[F192; COMPOSITION_SHIFTS.len()]) -> F192 { + for (&challenge, &shift) in challenges.iter().zip(COMPOSITION_SHIFTS.iter()) { + let mut frobenius = value; + for _ in 0..shift { + frobenius = frobenius.square(); } + value += challenge * frobenius; } - sums + value } -/// Padding-aware fused 2-claim MFR fold. Skips chunks of 4 -/// F128s that fall entirely in the zero padding of every block. -pub fn fold_1b_rows_2way_mfr_padded( - packed_witness: &[F128], - t0: &[F128], - t1: &[F128], - padding: &PaddingSpec, -) -> (Vec, Vec) { - use rayon::prelude::*; - let n = 1 << LOG_PACKING; // 128 - assert_eq!(t0.len(), packed_witness.len()); - assert_eq!(t1.len(), packed_witness.len()); - assert!( - packed_witness.len().is_multiple_of(4), - "fold_1b_rows_2way_mfr requires len divisible by 4 (got {})", - packed_witness.len() - ); - let skip = ChunkPadding::new(padding, 4); - - let pair = packed_witness - .par_chunks(4) - .zip(t0.par_chunks(4)) - .zip(t1.par_chunks(4)) - .enumerate() - .fold( - || (vec![F128::ZERO; n], vec![F128::ZERO; n]), - |(mut a0, mut a1), (chunk_idx, ((m_chunk, t0_chunk), t1_chunk))| { - if skip.skip(chunk_idx) { - return (a0, a1); - } - let v0: [F128; 4] = [t0_chunk[0], t0_chunk[1], t0_chunk[2], t0_chunk[3]]; - let v1: [F128; 4] = [t1_chunk[0], t1_chunk[1], t1_chunk[2], t1_chunk[3]]; - - // Build the two 16-entry subset-sum lookup tables. - let lookup0 = subset_sums_4(v0); - let lookup1 = subset_sums_4(v1); - - // Cache all 16 bytes of each m element for fast indexed access. - let m_bytes: [[u8; 16]; 4] = std::array::from_fn(|e| m_chunk[e].to_le_bytes()); - - // For each byte position (16 total = bits [r_byte*8, r_byte*8+8)): - // - Gather the same byte from each of the 4 m elements. - // - Pack into a u64 with the 4 bytes occupying byte slots 0..4 - // (slots 4..8 are zero). - // - Apply 8×8 bit transpose. After transpose, byte p of the - // u64 has its low-bit positions filled with - // (bit-p of m[0]'s r_byte, bit-p of m[1]'s, bit-p of m[2]'s, - // bit-p of m[3]'s) — that's exactly the 4-bit mask for - // output position r = r_byte*8 + p. - // - Look up the mask in the subset-sum tables and XOR into - // a0[r], a1[r]. - for r_byte in 0..16 { - let combined: u64 = (m_bytes[0][r_byte] as u64) - | ((m_bytes[1][r_byte] as u64) << 8) - | ((m_bytes[2][r_byte] as u64) << 16) - | ((m_bytes[3][r_byte] as u64) << 24); - let transposed = transpose_8x8_bits(combined); - let tb = transposed.to_le_bytes(); - let base = r_byte * 8; - // 8 unrolled lookups + RMWs. Each transposed byte's low - // 4 bits hold the mask; high 4 bits are always zero (the - // upper 4 byte-slots of `combined` were zero). - a0[base] += lookup0[(tb[0] & 0x0F) as usize]; - a1[base] += lookup1[(tb[0] & 0x0F) as usize]; - a0[base + 1] += lookup0[(tb[1] & 0x0F) as usize]; - a1[base + 1] += lookup1[(tb[1] & 0x0F) as usize]; - a0[base + 2] += lookup0[(tb[2] & 0x0F) as usize]; - a1[base + 2] += lookup1[(tb[2] & 0x0F) as usize]; - a0[base + 3] += lookup0[(tb[3] & 0x0F) as usize]; - a1[base + 3] += lookup1[(tb[3] & 0x0F) as usize]; - a0[base + 4] += lookup0[(tb[4] & 0x0F) as usize]; - a1[base + 4] += lookup1[(tb[4] & 0x0F) as usize]; - a0[base + 5] += lookup0[(tb[5] & 0x0F) as usize]; - a1[base + 5] += lookup1[(tb[5] & 0x0F) as usize]; - a0[base + 6] += lookup0[(tb[6] & 0x0F) as usize]; - a1[base + 6] += lookup1[(tb[6] & 0x0F) as usize]; - a0[base + 7] += lookup0[(tb[7] & 0x0F) as usize]; - a1[base + 7] += lookup1[(tb[7] & 0x0F) as usize]; - } - - (a0, a1) - }, - ) - .reduce( - || (vec![F128::ZERO; n], vec![F128::ZERO; n]), - |(mut a0, mut a1), (b0, b1)| { - for r in 0..n { - a0[r] += b0[r]; - a1[r] += b1[r]; - } - (a0, a1) - }, - ); - - (pair.0, pair.1) +pub fn build_coordinate_weights(challenges: &[F192; COMPOSITION_SHIFTS.len()]) -> Vec { + // b_w has only bit w set: bits 0..64 are K's power basis, and bits 64/128 + // shift it by Y / Y^2. + let basis = |w: usize| match w / PACKING_WIDTH { + 0 => F192::new(1u64 << (w % PACKING_WIDTH), 0, 0), + 1 => F192::new(0, 1u64 << (w % PACKING_WIDTH), 0), + _ => F192::new(0, 0, 1u64 << (w % PACKING_WIDTH)), + }; + (0..DEGREE_E) + .map(|w| apply_composed_map(basis(w), challenges)) + .collect() } -/// Padding-aware fused 2-claim 8-wide MFR fold. Skips chunks of -/// 8 F128s that fall entirely in the zero padding of every block — those -/// chunks contribute nothing (witness bytes = 0 → subset-sum mask = 0 → -/// `lookup[0] = 0`). -pub fn fold_1b_rows_2way_mfr_8wide_padded( - packed_witness: &[F128], - t0: &[F128], - t1: &[F128], - padding: &PaddingSpec, -) -> (Vec, Vec) { - use rayon::prelude::*; - let n = 1 << LOG_PACKING; - assert_eq!(t0.len(), packed_witness.len()); - assert_eq!(t1.len(), packed_witness.len()); - assert!(packed_witness.len().is_multiple_of(8)); - let skip = ChunkPadding::new(padding, 8); - - packed_witness - .par_chunks(8) - .zip(t0.par_chunks(8)) - .zip(t1.par_chunks(8)) - .enumerate() - .fold( - || (vec![F128::ZERO; n], vec![F128::ZERO; n]), - |(mut a0, mut a1), (chunk_idx, ((m_chunk, t0_chunk), t1_chunk))| { - if skip.skip(chunk_idx) { - return (a0, a1); - } - let t0_lo = subset_sums_4([t0_chunk[0], t0_chunk[1], t0_chunk[2], t0_chunk[3]]); - let t0_hi = subset_sums_4([t0_chunk[4], t0_chunk[5], t0_chunk[6], t0_chunk[7]]); - let t1_lo = subset_sums_4([t1_chunk[0], t1_chunk[1], t1_chunk[2], t1_chunk[3]]); - let t1_hi = subset_sums_4([t1_chunk[4], t1_chunk[5], t1_chunk[6], t1_chunk[7]]); - - let m_bytes: [[u8; 16]; 8] = std::array::from_fn(|e| m_chunk[e].to_le_bytes()); - - for r_byte in 0..16 { - let combined: u64 = (m_bytes[0][r_byte] as u64) - | ((m_bytes[1][r_byte] as u64) << 8) - | ((m_bytes[2][r_byte] as u64) << 16) - | ((m_bytes[3][r_byte] as u64) << 24) - | ((m_bytes[4][r_byte] as u64) << 32) - | ((m_bytes[5][r_byte] as u64) << 40) - | ((m_bytes[6][r_byte] as u64) << 48) - | ((m_bytes[7][r_byte] as u64) << 56); - let tb = transpose_8x8_bits(combined).to_le_bytes(); - let base = r_byte * 8; - for p in 0..8 { - let mask = tb[p]; - let lo = (mask & 0x0F) as usize; - let hi = (mask >> 4) as usize; - a0[base + p] += t0_lo[lo] + t0_hi[hi]; - a1[base + p] += t1_lo[lo] + t1_hi[hi]; - } - } - (a0, a1) - }, - ) - .reduce( - || (vec![F128::ZERO; n], vec![F128::ZERO; n]), - |(mut a0, mut a1), (b0, b1)| { - for r in 0..n { - a0[r] += b0[r]; - a1[r] += b1[r]; - } - (a0, a1) - }, - ) +/// Sample the composed map's challenges after every ring-switch message has +/// been absorbed. +pub fn sample_map_challenges(sponge: &mut Sponge) -> [F192; COMPOSITION_SHIFTS.len()] { + std::array::from_fn(|_| sponge.sample()) } -/// Single-tensor 16-wide method-of-four-Russians fold. Processes 16 witness -/// elements per group (four 4-element subset-sum tables, 16-bit per-position -/// masks) so each length-128 accumulator entry is touched once per 16 elements -/// instead of once per 8, halving acc load+store traffic. Gathers (32·N), eor3 -/// count, and table-build adds match the 8-wide kernel; the only delta is fewer -/// acc RMWs. Measured ~1.25× over the 8-wide kernel (the fold is LSU-bound). -/// -/// `open_batch`'s k=2 path runs this **twice** (once per suffix tensor) rather -/// than one fused 2-way fold: keeping a single length-128 accumulator + four -/// tables in flight avoids the register pressure of the 2-way's two -/// accumulators + eight tables, which ate most of the 16-wide win there. The -/// shared bit-transpose recomputed per call is nearly free (the fold is not -/// memory-bandwidth bound). -pub fn fold_1b_rows_1way_mfr_16wide_padded( - packed_witness: &[F128], - t: &[F128], - padding: &PaddingSpec, -) -> Vec { - use rayon::prelude::*; - let n = 1 << LOG_PACKING; - assert_eq!(t.len(), packed_witness.len()); - assert!(packed_witness.len().is_multiple_of(16)); - let skip = ChunkPadding::new(padding, 16); +// --------------------------------------------------------------------------- +// Sponge helpers: every 24-byte pattern is a valid F192. +// --------------------------------------------------------------------------- - packed_witness - .par_chunks(16) - .zip(t.par_chunks(16)) - .enumerate() - .fold( - || vec![F128::ZERO; n], - |mut acc, (chunk_idx, (m_chunk, t_chunk))| { - if skip.skip(chunk_idx) { - return acc; - } - let tbl0 = subset_sums_4([t_chunk[0], t_chunk[1], t_chunk[2], t_chunk[3]]); - let tbl1 = subset_sums_4([t_chunk[4], t_chunk[5], t_chunk[6], t_chunk[7]]); - let tbl2 = subset_sums_4([t_chunk[8], t_chunk[9], t_chunk[10], t_chunk[11]]); - let tbl3 = subset_sums_4([t_chunk[12], t_chunk[13], t_chunk[14], t_chunk[15]]); - - let m_bytes: [[u8; 16]; 16] = std::array::from_fn(|e| m_chunk[e].to_le_bytes()); - - for r_byte in 0..16 { - let lo8: u64 = (m_bytes[0][r_byte] as u64) - | ((m_bytes[1][r_byte] as u64) << 8) - | ((m_bytes[2][r_byte] as u64) << 16) - | ((m_bytes[3][r_byte] as u64) << 24) - | ((m_bytes[4][r_byte] as u64) << 32) - | ((m_bytes[5][r_byte] as u64) << 40) - | ((m_bytes[6][r_byte] as u64) << 48) - | ((m_bytes[7][r_byte] as u64) << 56); - let hi8: u64 = (m_bytes[8][r_byte] as u64) - | ((m_bytes[9][r_byte] as u64) << 8) - | ((m_bytes[10][r_byte] as u64) << 16) - | ((m_bytes[11][r_byte] as u64) << 24) - | ((m_bytes[12][r_byte] as u64) << 32) - | ((m_bytes[13][r_byte] as u64) << 40) - | ((m_bytes[14][r_byte] as u64) << 48) - | ((m_bytes[15][r_byte] as u64) << 56); - let tlo = transpose_8x8_bits(lo8).to_le_bytes(); - let thi = transpose_8x8_bits(hi8).to_le_bytes(); - let base = r_byte * 8; - for p in 0..8 { - let m_lo = tlo[p]; - let m_hi = thi[p]; - acc[base + p] += tbl0[(m_lo & 0x0F) as usize] - + tbl1[(m_lo >> 4) as usize] - + tbl2[(m_hi & 0x0F) as usize] - + tbl3[(m_hi >> 4) as usize]; - } - } - acc - }, - ) - .reduce( - || vec![F128::ZERO; n], - |mut a, b| { - for r in 0..n { - a[r] += b[r]; - } - a - }, - ) +fn observe_ext_slice(sponge: &mut Sponge, values: &[F192]) { + for &e in values { + sponge.observe(e); + } } -/// Tensor-split sibling of [`fold_1b_rows_1way_mfr_16wide_padded`]. Instead of -/// streaming a fully-materialized length-`2^n` suffix tensor `t`, it takes the -/// two factors `(eq_lo, eq_hi)` from [`build_eq_split`] and reassociates the -/// fold as inner-then-outer: -/// -/// ```text -/// s_hat_v[r] = Σ_i bit_r(W[i]) · t[i] -/// = Σ_{i_hi} eq_hi[i_hi] · ( Σ_{i_lo} bit_r(W[i_hi·B + i_lo]) · eq_lo[i_lo] ) -/// ``` -/// -/// with `B = eq_lo.len()` (a multiple of 16) and `i = i_hi·B + i_lo`. The inner -/// sum is the same 16-wide method-of-four-Russians fold over one length-`B` -/// block against `eq_lo`; the outer step scales that length-128 block result by -/// `eq_hi[i_hi]` and XORs it into the global accumulator. +// --------------------------------------------------------------------------- +// Building blocks +// --------------------------------------------------------------------------- + +/// Prefix weights for a plain multilinear point claim: the eq tensor of the +/// 6 intra-word coordinates, +/// `weights[i] = prod_j (bit_j(i) ? r_prefix[j] : 1 + r_prefix[j])`. /// -/// Result is **byte-identical** to -/// `fold_1b_rows_1way_mfr_16wide_padded(W, build_eq_parallel(r), padding)`: -/// GF(2^128) add is XOR (associative/commutative) and multiply is exact and -/// distributes, so the reassociation reproduces the same multiset of XOR terms. -/// Two wins over the materialized kernel: -/// 1. The four MFR subset-sum tables per 16-element chunk are built from -/// `eq_lo` and are **identical for every block**, so they are precomputed -/// once and reused across all `2^(n - n_lo)` blocks (no per-chunk table -/// rebuilds). -/// 2. The `2^n`-entry tensor is never streamed from RAM — only `eq_lo` -/// (+ its tables) and `eq_hi` are read, and they stay cache-resident. -/// Since the fold is LSU-bound, dropping that traffic is the main win. -pub fn fold_1b_rows_split( - packed_witness: &[F128], - eq_lo: &[F128], - eq_hi: &[F128], - padding: &PaddingSpec, -) -> Vec { - use rayon::prelude::*; - let n = 1 << LOG_PACKING; // 128 - let b = eq_lo.len(); - assert!( - b.is_multiple_of(16), - "fold_1b_rows_split: eq_lo block size must be a multiple of 16 (got {b})" +/// The prefix here is a plain boolean 6-cube (the bit index inside a K +/// word), so plain eq weights are correct; the old module needed phi_8 +/// Lagrange weights only because its prefix was the univariate-skip domain. +pub fn eq_prefix_weights(r_prefix: &[F192]) -> Vec { + assert_eq!( + r_prefix.len(), + LOG_PACKING, + "eq_prefix_weights: prefix must have LOG_PACKING = 6 coords" ); - assert_eq!(packed_witness.len(), b * eq_hi.len()); - let chunks_per_block = b / 16; - let skip = ChunkPadding::new(padding, 16); - - // Precompute the eq_lo subset-sum tables once and reuse for every block. - // `tables[c]` holds the four 16-entry tables for local chunk `c`'s 16 eq_lo - // values — exactly what the materialized kernel rebuilds per chunk. - let tables: Vec<[[F128; 16]; 4]> = (0..chunks_per_block) - .map(|c| { - let o = c * 16; - [ - subset_sums_4([eq_lo[o], eq_lo[o + 1], eq_lo[o + 2], eq_lo[o + 3]]), - subset_sums_4([eq_lo[o + 4], eq_lo[o + 5], eq_lo[o + 6], eq_lo[o + 7]]), - subset_sums_4([eq_lo[o + 8], eq_lo[o + 9], eq_lo[o + 10], eq_lo[o + 11]]), - subset_sums_4([eq_lo[o + 12], eq_lo[o + 13], eq_lo[o + 14], eq_lo[o + 15]]), - ] - }) - .collect(); - - packed_witness - .par_chunks(b) - .enumerate() - .fold( - || vec![F128::ZERO; n], - |mut acc, (i_hi, w_block)| { - let mut inner = [F128::ZERO; 128]; - let base_chunk = i_hi * chunks_per_block; - for c in 0..chunks_per_block { - // Same per-chunk skip predicate as the materialized kernel, - // evaluated at the identical global chunk index — so the two - // touch the exact same set of chunks. - if skip.skip(base_chunk + c) { - continue; - } - let m_chunk = &w_block[c * 16..c * 16 + 16]; - let [tbl0, tbl1, tbl2, tbl3] = &tables[c]; - - let m_bytes: [[u8; 16]; 16] = std::array::from_fn(|e| m_chunk[e].to_le_bytes()); - - for r_byte in 0..16 { - let lo8: u64 = (m_bytes[0][r_byte] as u64) - | ((m_bytes[1][r_byte] as u64) << 8) - | ((m_bytes[2][r_byte] as u64) << 16) - | ((m_bytes[3][r_byte] as u64) << 24) - | ((m_bytes[4][r_byte] as u64) << 32) - | ((m_bytes[5][r_byte] as u64) << 40) - | ((m_bytes[6][r_byte] as u64) << 48) - | ((m_bytes[7][r_byte] as u64) << 56); - let hi8: u64 = (m_bytes[8][r_byte] as u64) - | ((m_bytes[9][r_byte] as u64) << 8) - | ((m_bytes[10][r_byte] as u64) << 16) - | ((m_bytes[11][r_byte] as u64) << 24) - | ((m_bytes[12][r_byte] as u64) << 32) - | ((m_bytes[13][r_byte] as u64) << 40) - | ((m_bytes[14][r_byte] as u64) << 48) - | ((m_bytes[15][r_byte] as u64) << 56); - let tlo = transpose_8x8_bits(lo8).to_le_bytes(); - let thi = transpose_8x8_bits(hi8).to_le_bytes(); - let base = r_byte * 8; - for p in 0..8 { - let m_lo = tlo[p]; - let m_hi = thi[p]; - inner[base + p] += tbl0[(m_lo & 0x0F) as usize] - + tbl1[(m_lo >> 4) as usize] - + tbl2[(m_hi & 0x0F) as usize] - + tbl3[(m_hi >> 4) as usize]; - } - } - } - // Outer: scale this block's length-128 partial by eq_hi[i_hi]. - // `e · (Σ eq_lo·bit) = Σ (e·eq_lo)·bit` distributes exactly, so - // each term equals the materialized `t[i] = eq_lo·eq_hi` term. - let e = eq_hi[i_hi]; - for r in 0..n { - acc[r] += e * inner[r]; - } - acc - }, - ) - .reduce( - || vec![F128::ZERO; n], - |mut a, b| { - for r in 0..n { - a[r] += b[r]; - } - a - }, - ) + build_eq_table_ext(r_prefix) } -/// Two-claim variant of [`fold_1b_rows_split`] with stack-allocated per-claim -/// inner accumulators. The common batched case (exactly 2 dense claims, e.g. -/// `[ab, c]` or `[ab, c]` alongside a sparse chain claim) hits this fast path. -/// -/// Cross-claim sharing per chunk: -/// * one streaming read of the 16 packed_witness entries -/// * one bit transpose ([`transpose_8x8_bits`]) -/// * per-claim subset-sum table lookups + per-claim inner accumulator update -/// -/// Per-claim outputs are **byte-identical** to calling [`fold_1b_rows_split`] -/// twice — same chunk-skip predicate, same XOR multiset. -pub fn fold_1b_rows_split_2way( - packed_witness: &[F128], - eq_lo_0: &[F128], - eq_hi_0: &[F128], - eq_lo_1: &[F128], - eq_hi_1: &[F128], - padding: &PaddingSpec, -) -> (Vec, Vec) { - use rayon::prelude::*; - let n = 1 << LOG_PACKING; // 128 - let b = eq_lo_0.len(); - assert_eq!(eq_lo_1.len(), b); - let n_hi = eq_hi_0.len(); - assert_eq!(eq_hi_1.len(), n_hi); - assert!( - b.is_multiple_of(16), - "fold_1b_rows_split_2way: eq_lo block size must be a multiple of 16 (got {b})" - ); - assert_eq!(packed_witness.len(), b * n_hi); - let chunks_per_block = b / 16; - let skip = ChunkPadding::new(padding, 16); - - // Precompute both claims' subset-sum tables once. - let tables_0: Vec<[[F128; 16]; 4]> = (0..chunks_per_block) - .map(|c| { - let o = c * 16; - [ - subset_sums_4([eq_lo_0[o], eq_lo_0[o + 1], eq_lo_0[o + 2], eq_lo_0[o + 3]]), - subset_sums_4([ - eq_lo_0[o + 4], - eq_lo_0[o + 5], - eq_lo_0[o + 6], - eq_lo_0[o + 7], - ]), - subset_sums_4([ - eq_lo_0[o + 8], - eq_lo_0[o + 9], - eq_lo_0[o + 10], - eq_lo_0[o + 11], - ]), - subset_sums_4([ - eq_lo_0[o + 12], - eq_lo_0[o + 13], - eq_lo_0[o + 14], - eq_lo_0[o + 15], - ]), - ] - }) - .collect(); - let tables_1: Vec<[[F128; 16]; 4]> = (0..chunks_per_block) - .map(|c| { - let o = c * 16; - [ - subset_sums_4([eq_lo_1[o], eq_lo_1[o + 1], eq_lo_1[o + 2], eq_lo_1[o + 3]]), - subset_sums_4([ - eq_lo_1[o + 4], - eq_lo_1[o + 5], - eq_lo_1[o + 6], - eq_lo_1[o + 7], - ]), - subset_sums_4([ - eq_lo_1[o + 8], - eq_lo_1[o + 9], - eq_lo_1[o + 10], - eq_lo_1[o + 11], - ]), - subset_sums_4([ - eq_lo_1[o + 12], - eq_lo_1[o + 13], - eq_lo_1[o + 14], - eq_lo_1[o + 15], - ]), - ] - }) - .collect(); +/// Standard inner product `sum_i a[i] * b[i]` over E. +pub fn inner_product_ext(a: &[F192], b: &[F192]) -> F192 { + assert_eq!(a.len(), b.len()); + let mut acc = F192::ZERO; + for (&x, &y) in a.iter().zip(b.iter()) { + acc += x * y; + } + acc +} - let zero_acc = || (vec![F128::ZERO; n], vec![F128::ZERO; n]); +/// The verifier's claim check: `sum_i prefix_weights[i] * s_hat_v[i]`. +pub fn claim_check(prefix_weights: &[F192], s_hat_v: &[F192]) -> F192 { + inner_product_ext(prefix_weights, s_hat_v) +} - packed_witness - .par_chunks(b) - .enumerate() - .fold(zero_acc, |(mut acc0, mut acc1), (i_hi, w_block)| { - // Two stack-allocated inner accumulators — identical layout to - // the single-claim split path, just two of them. - let mut inner0 = [F128::ZERO; 128]; - let mut inner1 = [F128::ZERO; 128]; - let base_chunk = i_hi * chunks_per_block; - for c in 0..chunks_per_block { - if skip.skip(base_chunk + c) { - continue; +/// Tower (`F192`) trace-dual basis: `TRACE_DUAL_BASIS[i]` is the unique element +/// with `bit_i(y) = Tr(TRACE_DUAL_BASIS[i] · y)` for the coordinate bit `i` of +/// `y ∈ F192` (c0 bits 0..64, c1 bits 64..128, c2 bits 128..192), where `Tr` is the absolute +/// trace `F192 → F2`, using the tower's coordinate basis and trace form. +/// The recursion guest replays bit extraction with these. +pub fn trace_dual_basis() -> &'static [F192; 192] { + use std::sync::OnceLock; + static DUAL: OnceLock<[F192; 192]> = OnceLock::new(); + DUAL.get_or_init(|| { + let basis = |j: usize| { + if j < 64 { + F192::new(1u64 << j, 0, 0) + } else if j < 128 { + F192::new(0, 1u64 << (j - 64), 0) + } else { + F192::new(0, 0, 1u64 << (j - 128)) + } + }; + // Absolute trace to F2: Tr(x) = Σ_{k=0}^{191} x^{2^k}. + let tr = |x: F192| { + let (mut acc, mut p) = (F192::ZERO, x); + for _ in 0..192 { + acc += p; + p = p.square(); + } + acc + }; + // Invert the 192x192 trace Gram matrix over F2. This runs once and is + // deliberately simple; protocol hot paths only read the cached basis. + let mut aug = vec![vec![0u8; 2 * DEGREE_E]; DEGREE_E]; + for i in 0..DEGREE_E { + for j in 0..DEGREE_E { + if tr(basis(i) * basis(j)) == F192::ONE { + aug[i][j] = 1; } - let m_chunk = &w_block[c * 16..c * 16 + 16]; - let [t0a, t0b, t0c, t0d] = &tables_0[c]; - let [t1a, t1b, t1c, t1d] = &tables_1[c]; - - let m_bytes: [[u8; 16]; 16] = std::array::from_fn(|e| m_chunk[e].to_le_bytes()); - - for r_byte in 0..16 { - let lo8: u64 = (m_bytes[0][r_byte] as u64) - | ((m_bytes[1][r_byte] as u64) << 8) - | ((m_bytes[2][r_byte] as u64) << 16) - | ((m_bytes[3][r_byte] as u64) << 24) - | ((m_bytes[4][r_byte] as u64) << 32) - | ((m_bytes[5][r_byte] as u64) << 40) - | ((m_bytes[6][r_byte] as u64) << 48) - | ((m_bytes[7][r_byte] as u64) << 56); - let hi8: u64 = (m_bytes[8][r_byte] as u64) - | ((m_bytes[9][r_byte] as u64) << 8) - | ((m_bytes[10][r_byte] as u64) << 16) - | ((m_bytes[11][r_byte] as u64) << 24) - | ((m_bytes[12][r_byte] as u64) << 32) - | ((m_bytes[13][r_byte] as u64) << 40) - | ((m_bytes[14][r_byte] as u64) << 48) - | ((m_bytes[15][r_byte] as u64) << 56); - let tlo = transpose_8x8_bits(lo8).to_le_bytes(); - let thi = transpose_8x8_bits(hi8).to_le_bytes(); - let base = r_byte * 8; - for p in 0..8 { - let m_lo = tlo[p]; - let m_hi = thi[p]; - let i_lo4 = (m_lo & 0x0F) as usize; - let i_hi4 = (m_lo >> 4) as usize; - let i_lo4h = (m_hi & 0x0F) as usize; - let i_hi4h = (m_hi >> 4) as usize; - inner0[base + p] += t0a[i_lo4] + t0b[i_hi4] + t0c[i_lo4h] + t0d[i_hi4h]; - inner1[base + p] += t1a[i_lo4] + t1b[i_hi4] + t1c[i_lo4h] + t1d[i_hi4h]; + } + aug[i][DEGREE_E + i] = 1; + } + for col in 0..DEGREE_E { + let piv = (col..DEGREE_E) + .find(|&r| aug[r][col] == 1) + .expect("trace Gram matrix is invertible"); + aug.swap(col, piv); + for r in 0..DEGREE_E { + if r != col && aug[r][col] == 1 { + for j in col..2 * DEGREE_E { + aug[r][j] ^= aug[col][j]; } } } - let e0 = eq_hi_0[i_hi]; - let e1 = eq_hi_1[i_hi]; - for r in 0..n { - acc0[r] += e0 * inner0[r]; - acc1[r] += e1 * inner1[r]; - } - (acc0, acc1) - }) - .reduce(zero_acc, |(mut a0, mut a1), (b0, b1)| { - for r in 0..n { - a0[r] += b0[r]; - a1[r] += b1[r]; + } + let mut out = [F192::ZERO; 192]; + for (i, o) in out.iter_mut().enumerate() { + for j in 0..DEGREE_E { + if aug[i][DEGREE_E + j] == 1 { + *o += basis(j); + } } - (a0, a1) - }) + } + out + }) } -/// AB-claim `s_hat_v` specialization that **skips `fold_1b_rows` entirely** -/// when the upstream layer has already produced -/// `z_vec[i_inner] = ẑ(i_inner, x_outer)` (length `2^k_log`) — the pre-sumcheck -/// partial fold lincheck builds via `partial_fold_packed_z`. -/// -/// # Identity -/// -/// For a PCS opening at point `(r_inner_skip, r_inner_rest, x_outer)` where -/// `x_outer` matches lincheck's, the AB-suffix tensor in `fold_1b_rows` -/// factors over the same axis decomposition that `z_vec` was built along: -/// -/// ```text -/// s_hat_v[b] = Σ_{j ∈ {0,1}^(m−7)} eq(suffix, j) · bit_b(packed_witness[j]) -/// = Σ_{k ∈ {0,1}^(k_log − LOG_PACKING)} -/// eq(r_inner_rest[1..], k) · z_vec[b + 2^LOG_PACKING · k] -/// ``` -/// -/// `r_inner_rest[0]` becomes ring-switch's `prefix0` (`x_outer_full[0]`); -/// `r_inner_rest[1..]` is the suffix's inner part. The witness's outer -/// coords were already folded into `z_vec` by the partial fold. -/// -/// Output is **byte-identical** to -/// `fold_1b_rows(packed_witness, build_eq(suffix))` for the AB claim — same -/// algebraic identity, just reassociated to use the lincheck intermediate. -/// -/// # Cost +/// Compute the slice-MLE vector `s_hat_v` (length 64) from a packed witness +/// and a tensor-expanded suffix point. /// -/// `128 · 2^(k_log − LOG_PACKING)` F128 mul-adds + a tiny eq tensor build. -/// At keccak m=29, k_log=17: 128 · 1024 = 131k mul-adds — tens of µs MT, vs -/// the ~7 ms share that AB contributes to `fold_1b_rows_split_2way`. +/// `packed_witness[y] in K` for `y in 0..2^L`; `suffix_tensor` is +/// `eq(r_suffix, .)` over the same range (from +/// [`build_eq_table_ext`]). /// -/// # Panics +/// Output: `s_hat_v[i] = sum_y bit_i(packed_witness[y]) * suffix_tensor[y]` +/// for `i in 0..64` (bit i = polynomial-basis coordinate of the u64). /// -/// - if `z_vec.len() != 2^(LOG_PACKING + tail.len())`. -pub fn s_hat_v_from_z_vec(z_vec: &[F128], x_inner_rest_tail: &[F128]) -> Vec { - use rayon::prelude::*; - let n_packed = 1usize << LOG_PACKING; // 128 - let n_tail = 1usize << x_inner_rest_tail.len(); - assert_eq!( - z_vec.len(), - n_packed * n_tail, - "z_vec length {} mismatches 2^(LOG_PACKING + tail.len()) = {}", - z_vec.len(), - n_packed * n_tail, - ); +/// Dispatch: the method-of-four-Russians kernel +/// (`fold_1b_rows_mfr_8wide`) for lengths divisible by 8 (any real +/// witness), the scalar bit-scan otherwise (tiny test instances). Both +/// compute the same per-bit XOR-sums, only regrouped, and GF(2^192) +/// addition is XOR (commutative, associative, exact), so the output and +/// hence the transcript are byte-identical either way. +pub fn fold_1b_rows(packed_witness: &[F64], suffix_tensor: &[F192]) -> Vec { + assert_eq!(packed_witness.len(), suffix_tensor.len()); + if !packed_witness.is_empty() && packed_witness.len().is_multiple_of(8) { + fold_1b_rows_mfr_8wide(packed_witness, suffix_tensor) + } else { + fold_1b_rows_scalar(packed_witness, suffix_tensor) + } +} - if x_inner_rest_tail.is_empty() { - // Degenerate case (k_log == LOG_PACKING): the LOG_PACKING boundary - // ate the only inner-rest coord — z_vec IS the per-prefix-bit answer. +/// Reuse lincheck's partial fold to derive the 64 slice evaluations needed by +/// the K ring switch, avoiding a second pass over the packed witness. +pub fn s_hat_v_from_z_vec(z_vec: &[F192], inner_rest_tail: &[F192]) -> Vec { + use rayon::prelude::*; + let n_packed = PACKING_WIDTH; + let n_tail = 1usize << inner_rest_tail.len(); + assert_eq!(z_vec.len(), n_packed * n_tail); + if inner_rest_tail.is_empty() { return z_vec.to_vec(); } - - let eq_tail = build_eq_parallel(x_inner_rest_tail); - - // Iterate over k outer (sequential per-thread → cache-friendly stride-1 - // reads of z_vec). Parallelize across k-ranges; each thread accumulates - // a private length-128 buffer and the reduce step XORs them together. - eq_tail + build_eq_table_ext(inner_rest_tail) .par_iter() .enumerate() .fold( - || vec![F128::ZERO; n_packed], - |mut acc, (k, &w)| { - let block = &z_vec[k * n_packed..(k + 1) * n_packed]; - for b in 0..n_packed { - acc[b] += w * block[b]; + || vec![F192::ZERO; n_packed], + |mut acc, (k, &weight)| { + for (slot, &value) in acc.iter_mut().zip(&z_vec[k * n_packed..(k + 1) * n_packed]) { + *slot += weight * value; } acc }, ) .reduce( - || vec![F128::ZERO; n_packed], - |mut a, b| { - for i in 0..n_packed { - a[i] += b[i]; + || vec![F192::ZERO; n_packed], + |mut acc, part| { + for (slot, value) in acc.iter_mut().zip(part) { + *slot += value; } - a + acc }, ) } -/// Compute the slice-MLE vector `s_hat_v` (length 128) from a packed witness -/// and a tensor-expanded suffix point. -/// -/// `packed_witness[i_rest] ∈ F_{2^128}` with `i_rest ∈ {0..2^L}` where -/// `L = log2(packed_witness.len())`. `suffix_tensor` is `eq_ind(suffix)` over a -/// suffix point of length `L`. -/// -/// Output: `s_hat_v[i_skip] = Σ_{i_rest} (i_skip-th bit of packed_witness[i_rest]) · suffix_tensor[i_rest]` -/// for `i_skip ∈ {0..128}`. The bit-index uses the natural polynomial-basis -/// decomposition of F_{2^128} (i.e., bit-i of the u128 .lo:.hi). -/// -/// O(2^L · 128) algorithm parallelized across packed-witness positions via -/// rayon: each thread folds a chunk into a per-thread length-128 partial -/// accumulator; the reduce step XORs partials elementwise into the final -/// output. -pub fn fold_1b_rows_naive(packed_witness: &[F128], suffix_tensor: &[F128]) -> Vec { +/// Scalar reference path of [`fold_1b_rows`]: mirror of +/// `ring_switch::fold_1b_rows_naive` at 64-bit width, a rayon bit-scan with +/// per-thread length-64 partial accumulators XOR-reduced at the end. +/// Data-dependent cost: `trailing_zeros` + RMW + branch per set bit +/// (~32/word on a random witness). +fn fold_1b_rows_scalar(packed_witness: &[F64], suffix_tensor: &[F192]) -> Vec { use rayon::prelude::*; assert_eq!(packed_witness.len(), suffix_tensor.len()); - let n = 1 << LOG_PACKING; - - // Partition into chunks; each chunk computes its own partial. - // Empty accumulator allocator returns Vec(n) for the fold's init. - let zero_acc = || vec![F128::ZERO; n]; + let n = PACKING_WIDTH; + let zero_acc = || vec![F192::ZERO; n]; packed_witness .par_iter() .zip(suffix_tensor.par_iter()) .fold(zero_acc, |mut acc, (elem, &w)| { - // Bit r ∈ 0..64: from elem.lo. - let mut lo = elem.lo; - while lo != 0 { - let r = lo.trailing_zeros() as usize; + let mut bits = elem.0; + while bits != 0 { + let r = bits.trailing_zeros() as usize; acc[r] += w; - lo &= lo - 1; - } - // Bit r ∈ 64..128: from elem.hi. - let mut hi = elem.hi; - while hi != 0 { - let r = hi.trailing_zeros() as usize; - acc[64 | r] += w; - hi &= hi - 1; + bits &= bits - 1; } acc }) @@ -980,430 +364,139 @@ pub fn fold_1b_rows_naive(packed_witness: &[F128], suffix_tensor: &[F128]) -> Ve }) } -/// Compute the verifier's claim check: `Σ_i weights[i] · s_hat_v[i]`. -pub fn claim_check(weights: &[F128], s_hat_v: &[F128]) -> F128 { - inner_product(weights, s_hat_v) -} - -/// Standard inner product `Σ_i a[i] · b[i]` over F_{2^128}. -/// The trace-dual basis {δ_i} of the polynomial basis {B_j = x^j}: -/// `Tr(δ_i·B_j) = [i = j]`, so `bit_i(y) = Tr(δ_i·y)` for every `y`. -/// Computed once (Gram matrix of the trace form inverted over F₂). -pub fn trace_dual_basis() -> &'static [F128; 128] { - use std::sync::OnceLock; - static DUAL: OnceLock<[F128; 128]> = OnceLock::new(); - DUAL.get_or_init(|| { - let basis = |j: usize| { - if j < 64 { F128::new(1u64 << j, 0) } else { F128::new(0, 1u64 << (j - 64)) } - }; - let tr = |x: F128| { - let (mut acc, mut p) = (F128::ZERO, x); - for _ in 0..128 { - acc += p; - p *= p; - } - acc - }; - let mut g: Vec = vec![0; 128]; - for i in 0..128 { - for j in 0..128 { - if tr(basis(i) * basis(j)) == F128::ONE { - g[i] |= 1u128 << j; - } - } - } - let mut inv: Vec = (0..128).map(|i| 1u128 << i).collect(); - for col in 0..128 { - let piv = (col..128).find(|&r| (g[r] >> col) & 1 == 1).expect("trace Gram matrix is invertible"); - g.swap(col, piv); - inv.swap(col, piv); - for r in 0..128 { - if r != col && (g[r] >> col) & 1 == 1 { - g[r] ^= g[col]; - inv[r] ^= inv[col]; - } - } - } - let mut out = [F128::ZERO; 128]; - for (i, o) in out.iter_mut().enumerate() { - for j in 0..128 { - if (inv[i] >> j) & 1 == 1 { - *o += basis(j); - } - } - } - out - }) -} - -/// The linearized-polynomial coefficients of the eq-weighted bit-sum: -/// `Σ_i w_i·bit_i(y) = L_w(y) = Σ_k c_k·y^{2^k}` with `c_k = Σ_i w_i·δ_i^{2^k}`. -/// Every tensor-algebra fold against `w` reduces to `L_w`, replacing bit-level -/// transposes with field arithmetic (squaring is one multiplication). -pub fn linearized_eq_coeffs(w: &[F128]) -> [F128; 128] { - assert_eq!(w.len(), 128); - let delta = trace_dual_basis(); - let mut c = [F128::ZERO; 128]; - for i in 0..128 { - let mut p = delta[i]; - for ck in c.iter_mut() { - *ck += w[i] * p; - p *= p; +/// Build the 16-entry subset-sum lookup table over 4 E elements: +/// `sums[mask] = sum_{k in 0..4 : bit_k(mask) = 1} elems[k]`. 15 additions +/// via the standard doubling pattern (mirror of +/// `ring_switch::subset_sums_4` retyped to the tower). +#[inline(always)] +fn subset_sums_4_ext(elems: [F192; 4]) -> [F192; 16] { + let mut sums = [F192::ZERO; 16]; + for (i, &e) in elems.iter().enumerate() { + let half = 1 << i; + for k in 0..half { + sums[half + k] = sums[k] + e; } } - c -} - -/// Seeds for the factored coefficient construction in -/// [`linearized_eq_coeffs_eq`]. -/// -/// The layout is -/// `main | factor[0..7] | correction[0..7]`. Squaring the whole array advances -/// it from Frobenius level `k` to level `k + 1`. -/// -/// For the GHASH power basis, with `a = x` and `d = p'(a) = a^6 + 1`, the -/// trace-dual basis is -/// -/// ```text -/// δ_i = d^-1 (a^(127-i) + ε_i), -/// ε = [a^6+a+1, a^5+1, a^4, a^3, a^2, a, 1, 0, ..., 0]. -/// ``` -/// -/// The seeds are respectively `d^-1 a^127`, `1 + a^(-2^t)`, and -/// `d^-1 ε_i`. -pub fn eq_linearized_seed_constants() -> [F128; 15] { - let a = F128::generator(); - let a_inv = a.inv(); - let mut a_pow = F128::ONE; - let mut powers = [F128::ONE; 128]; - for slot in powers.iter_mut() { - *slot = a_pow; - a_pow *= a; - } - let d_inv = (powers[6] + F128::ONE).inv(); - - let mut seeds = [F128::ZERO; 15]; - seeds[0] = d_inv * powers[127]; - - let mut neg_frob = a_inv; - for t in 0..LOG_PACKING { - seeds[1 + t] = F128::ONE + neg_frob; - neg_frob *= neg_frob; - } - - let corrections = [ - powers[6] + a + F128::ONE, - powers[5] + F128::ONE, - powers[4], - powers[3], - powers[2], - a, - F128::ONE, - ]; - for (slot, correction) in seeds[1 + LOG_PACKING..] - .iter_mut() - .zip(corrections) - { - *slot = d_inv * correction; - } - seeds -} - -/// The 15 factored constants at all 128 Frobenius levels. -/// -/// Native verification reuses this table directly. Recursive verifiers bake -/// it into their fixed program, so coefficient construction needs neither a -/// runtime orbit table nor runtime squarings of constants. -pub fn eq_linearized_orbit_constants() -> &'static [[F128; 15]; 128] { - use std::sync::OnceLock; - static ORBITS: OnceLock<[[F128; 15]; 128]> = OnceLock::new(); - ORBITS.get_or_init(|| { - let mut rows = [[F128::ZERO; 15]; 128]; - rows[0] = eq_linearized_seed_constants(); - for k in 1..128 { - rows[k] = rows[k - 1]; - for value in rows[k].iter_mut() { - *value *= *value; - } - } - rows - }) + sums } -/// Coefficients of the equality-weighted bit functional, specialized to -/// `w = eq(r, ·)` for a seven-coordinate point `r`. +/// Method-of-four-Russians [`fold_1b_rows`] kernel: the extension-field layer's +/// `fold_1b_rows_1way_mfr_8wide_k4` ported to 8-byte K words (where 8 words +/// per transpose group cover ALL 64 output bits with the 8 byte positions, +/// no wasted transpose rows). /// -/// This is exactly [`linearized_eq_coeffs`] applied to `build_eq(r)`, but uses -/// the sparse closed form of the trace-dual GHASH power basis: -/// -/// ```text -/// Σ_i eq(r,i) a^((127-i)2^k) -/// = a^(127·2^k) Π_t (1 + r_t + r_t a^(-2^(k+t))). -/// ``` -/// -/// Only the first seven dual-basis elements have correction terms. Thus each -/// coefficient costs `O(LOG_PACKING)` operations instead of a 128-term dense -/// Moore transform. -pub fn linearized_eq_coeffs_eq(r: &[F128]) -> [F128; 128] { - assert_eq!(r.len(), LOG_PACKING); - - // Only weights 0..6 occur in the sparse correction. Their four high bits - // are all zero, so share that factor instead of building all 128 weights. - let high_zero = - r[3..].iter().fold(F128::ONE, |acc, &rt| acc * (F128::ONE + rt)); - let low_eq = build_eq(&r[..3]); - let correction_weights: [F128; LOG_PACKING] = - std::array::from_fn(|i| high_zero * low_eq[i]); - - let mut c = [F128::ZERO; 128]; - for (ck, orbit) in c.iter_mut().zip(eq_linearized_orbit_constants()) { - let mut main = orbit[0]; - for t in 0..LOG_PACKING { - main *= F128::ONE + r[t] * orbit[1 + t]; - } - let mut correction = F128::ZERO; - for i in 0..LOG_PACKING { - correction += correction_weights[i] * orbit[1 + LOG_PACKING + i]; - } - *ck = main + correction; - } - c -} - -/// `⟨tensor_algebra_transpose(s_hat_v), w⟩` without the transpose: -/// `Σ_j B_j·L_w(s_hat_v[j])` (see [`linearized_eq_coeffs`]). -pub fn transposed_claim_linearized(s_hat_v: &[F128], c: &[F128; 128]) -> F128 { - assert_eq!(s_hat_v.len(), 128); - let basis = |j: usize| { - if j < 64 { F128::new(1u64 << j, 0) } else { F128::new(0, 1u64 << (j - 64)) } - }; - let mut acc = F128::ZERO; - for (j, &y) in s_hat_v.iter().enumerate() { - let (mut lw, mut p) = (F128::ZERO, y); - for (k, &ck) in c.iter().enumerate() { - lw += ck * p; - if k + 1 < c.len() { - p *= p; - } - } - acc += basis(j) * lw; - } - acc -} +/// Per group of 8 words: build two 16-entry subset-sum tables over the 8 +/// suffix weights (low nibble = words 0..4, high = words 4..8, 30 adds +/// total); then for each byte position `r_byte` gather that byte of all 8 +/// words into a u64 (word `e` in byte slot `e`) and 8x8 bit-transpose it, +/// so transposed byte `p`, bit `e` is bit `r_byte*8 + p` of word `e`: an +/// 8-bit mask over the group for output position `r = r_byte*8 + p`. Each +/// output position then costs two table lookups + one in-register add + one +/// accumulator RMW, regardless of bit density: a constant ~12 adds + 8 RMWs +/// per word vs the scalar path's ~32 data-dependent conditional adds. +/// Per-thread accumulators via rayon fold/reduce (no shared cache lines). +fn fold_1b_rows_mfr_8wide(packed_witness: &[F64], suffix_tensor: &[F192]) -> Vec { + use rayon::prelude::*; + let n = PACKING_WIDTH; + assert_eq!(packed_witness.len(), suffix_tensor.len()); + assert!(packed_witness.len().is_multiple_of(8)); + let zero_acc = || vec![F192::ZERO; n]; -/// [`eval_rs_eq`] via the telescoped product formula. The tensor element is -/// `Π_j (z_j⊗1 + 1⊗(1+q_j))`; its rank-1 subset expansion re-sums per -/// Frobenius power into `Σ_k c_k·Π_j (z_j^{2^k} + 1 + q_j)`. -pub fn eval_rs_eq_from_coeffs(z_vals: &[F128], query: &[F128], c: &[F128; 128]) -> F128 { - assert_eq!(z_vals.len(), query.len()); - let mut zp: Vec = z_vals.to_vec(); - let mut acc = F128::ZERO; - for (k, &ck) in c.iter().enumerate() { - let mut prod = F128::ONE; - for (zpj, &qj) in zp.iter_mut().zip(query.iter()) { - prod *= *zpj + F128::ONE + qj; - if k + 1 < c.len() { - *zpj *= *zpj; + packed_witness + .par_chunks(8) + .zip(suffix_tensor.par_chunks(8)) + .fold(zero_acc, |mut acc, (m_chunk, t_chunk)| { + let lo_tbl = subset_sums_4_ext([t_chunk[0], t_chunk[1], t_chunk[2], t_chunk[3]]); + let hi_tbl = subset_sums_4_ext([t_chunk[4], t_chunk[5], t_chunk[6], t_chunk[7]]); + + let mut m_bytes = [[0u8; 8]; 8]; + for (e, slot) in m_bytes.iter_mut().enumerate() { + *slot = m_chunk[e].0.to_le_bytes(); } - } - acc += ck * prod; - } - acc -} - -/// Standard inner product `Σ_i a[i] · b[i]` over F_{2^128}. -pub use primitives::multilinear::inner_product; -/// **TensorAlgebra transpose** (a.k.a. "bit transpose" of `s_hat_v`). -/// -/// View `s_hat_v` (length 128) as a 128×128 binary matrix with row `i_skip` = -/// the 128 polynomial-basis bits of `s_hat_v[i_skip]`. Output `s_hat_u` -/// (length 128) is the transposed matrix re-packed: row `b` of `s_hat_u` = -/// column `b` of the input. Equivalently: -/// ```text -/// bit i_skip of s_hat_u[b] == bit b of s_hat_v[i_skip] -/// ``` -/// -/// Used in the DP24 ring-switching: after computing `s_hat_v` (slice MLEs at -/// the suffix point), `s_hat_u = transpose(s_hat_v)` is the data viewed with -/// the "vertical" and "horizontal" dimensions swapped. The opening-sumcheck target is -/// `T = ⟨s_hat_u, eq_ind(r'')⟩`. -/// -/// Naive O(128²) bit-extract implementation. NEON acceleration via bit -/// transpose intrinsics is future work. -pub fn tensor_algebra_transpose(s_hat_v: &[F128]) -> Vec { - assert_eq!(s_hat_v.len(), 1 << LOG_PACKING); - let mut s_hat_u = vec![F128::ZERO; 1 << LOG_PACKING]; - for i_skip in 0..128 { - let elem = s_hat_v[i_skip]; - // Iterate over the 128 bits b of `elem`; deposit into s_hat_u[b]'s bit i_skip. - for b in 0..64 { - if (elem.lo >> b) & 1 == 1 { - if i_skip < 64 { - s_hat_u[b].lo |= 1u64 << i_skip; - } else { - s_hat_u[b].hi |= 1u64 << (i_skip - 64); + for r_byte in 0..8 { + let combined: u64 = (m_bytes[0][r_byte] as u64) + | ((m_bytes[1][r_byte] as u64) << 8) + | ((m_bytes[2][r_byte] as u64) << 16) + | ((m_bytes[3][r_byte] as u64) << 24) + | ((m_bytes[4][r_byte] as u64) << 32) + | ((m_bytes[5][r_byte] as u64) << 40) + | ((m_bytes[6][r_byte] as u64) << 48) + | ((m_bytes[7][r_byte] as u64) << 56); + let tb = transpose_8x8_bits(combined).to_le_bytes(); + let base = r_byte * 8; + for (p, &mask) in tb.iter().enumerate() { + acc[base + p] += lo_tbl[(mask & 0x0F) as usize] + hi_tbl[(mask >> 4) as usize]; } } - } - for b in 0..64 { - if (elem.hi >> b) & 1 == 1 { - if i_skip < 64 { - s_hat_u[64 | b].lo |= 1u64 << i_skip; - } else { - s_hat_u[64 | b].hi |= 1u64 << (i_skip - 64); - } + acc + }) + .reduce(zero_acc, |mut a, b| { + for (av, bv) in a.iter_mut().zip(b.iter()) { + *av += *bv; } - } - } - s_hat_u + a + }) } -/// Compute `rs_eq_ind` (the "ring-switching equality indicator"), a transparent -/// multilinear of length `2^L` over the suffix domain. +/// Compute `rs_eq_ind`, the transparent E-valued weight vector over the +/// suffix domain: `rs_eq_ind[y] = Phi(suffix_tensor[y])` where `Phi` sends +/// E-basis bit w to `coordinate_weights[w]`, i.e. /// -/// `rs_eq_ind[i_rest] = Σ_b (bit b of suffix_tensor[i_rest]) · eq_r_dprime[b]` +/// `rs_eq_ind[y] = sum_w bit_w(suffix_tensor[y]) * coordinate_weights[w]` /// -/// Each `suffix_tensor[i_rest] ∈ F_{2^128}` is treated as 128 F_2-bits in the -/// polynomial basis; the inner product with `eq_r_dprime` (length 128) produces -/// one F_{2^128} value per suffix position. This is the transparent multilinear -/// the opening runs its sumcheck against. -/// -/// O(128 · 2^L) parallelized across positions via rayon. Output positions are -/// independent — direct `par_iter` + `collect`. -pub fn fold_b128_elems_naive(suffix_tensor: &[F128], eq_r_dprime: &[F128]) -> Vec { +/// Naive reference: rayon per-position bit-scan over the three 64-bit limbs. +/// See [`fold_ext_elems`] for the bytewise-table production version. +#[cfg(test)] +pub fn fold_ext_elems_naive(suffix_tensor: &[F192], coordinate_weights: &[F192]) -> Vec { use rayon::prelude::*; - assert_eq!(eq_r_dprime.len(), 1 << LOG_PACKING); + assert_eq!(coordinate_weights.len(), DEGREE_E); suffix_tensor .par_iter() .map(|&elem| { - let mut acc = F128::ZERO; - let mut lo = elem.lo; - while lo != 0 { - let b = lo.trailing_zeros() as usize; - acc += eq_r_dprime[b]; - lo &= lo - 1; + let mut acc = F192::ZERO; + let mut c0 = elem.c0; + while c0 != 0 { + let w = c0.trailing_zeros() as usize; + acc += coordinate_weights[w]; + c0 &= c0 - 1; } - let mut hi = elem.hi; - while hi != 0 { - let b = hi.trailing_zeros() as usize; - acc += eq_r_dprime[64 | b]; - hi &= hi - 1; + let mut c1 = elem.c1; + while c1 != 0 { + let w = c1.trailing_zeros() as usize; + acc += coordinate_weights[64 | w]; + c1 &= c1 - 1; } - acc - }) - .collect() -} - -/// Bit-table accelerated `fold_b128_elems`. Precomputes 16 lookup tables (one -/// per byte position), each with 256 entries: `T[byte_idx][value] = Σ eq_r_dprime[bit]` -/// over set bits in `value` (offset by `byte_idx * 8`). Per element: 16 table -/// lookups + 16 F128 XORs, no data-dependent bit-scan. -/// -/// Tables: 16 × 256 × 16 B = 64 KB (fits in L1+L2). Target speedup ~3× vs the -/// `trailing_zeros` loop in `fold_b128_elems_naive`. -pub fn fold_b128_elems(suffix_tensor: &[F128], eq_r_dprime: &[F128]) -> Vec { - use rayon::prelude::*; - assert_eq!(eq_r_dprime.len(), 1 << LOG_PACKING); - const N_BYTES: usize = 16; // bytes per F128 - const TABLE_SIZE: usize = 256; - - // Build the 16 byte-tables. `tables[byte_idx * 256 + value]` = the F128 - // sum of `eq_r_dprime[byte_idx*8 + bit]` over set bits in `value`. - let mut tables = vec![F128::ZERO; N_BYTES * TABLE_SIZE]; - for byte_idx in 0..N_BYTES { - let bit_base = byte_idx * 8; - for value in 0..TABLE_SIZE { - let mut acc = F128::ZERO; - for bit_in_byte in 0..8 { - if (value >> bit_in_byte) & 1 == 1 { - acc += eq_r_dprime[bit_base + bit_in_byte]; - } + let mut c2 = elem.c2; + while c2 != 0 { + let w = c2.trailing_zeros() as usize; + acc += coordinate_weights[128 | w]; + c2 &= c2 - 1; } - tables[byte_idx * TABLE_SIZE + value] = acc; - } - } - - suffix_tensor - .par_iter() - .map(|&elem| { - let tables_ptr = tables.as_ptr(); - let lo_bytes = elem.lo.to_le_bytes(); - let hi_bytes = elem.hi.to_le_bytes(); - // Tree reduction (depth 4) — see fold_b128_elems_split for the - // pattern. Raw pointer access avoids per-lookup bounds checks - // (max index = 15 * 256 + 255 = 4095 = N_BYTES * TABLE_SIZE - 1, - // in bounds). - let (l0, l1, l2, l3, l4, l5, l6, l7, h0, h1, h2, h3, h4, h5, h6, h7) = unsafe { - ( - *tables_ptr.add(lo_bytes[0] as usize), - *tables_ptr.add(TABLE_SIZE + lo_bytes[1] as usize), - *tables_ptr.add(2 * TABLE_SIZE + lo_bytes[2] as usize), - *tables_ptr.add(3 * TABLE_SIZE + lo_bytes[3] as usize), - *tables_ptr.add(4 * TABLE_SIZE + lo_bytes[4] as usize), - *tables_ptr.add(5 * TABLE_SIZE + lo_bytes[5] as usize), - *tables_ptr.add(6 * TABLE_SIZE + lo_bytes[6] as usize), - *tables_ptr.add(7 * TABLE_SIZE + lo_bytes[7] as usize), - *tables_ptr.add(8 * TABLE_SIZE + hi_bytes[0] as usize), - *tables_ptr.add(9 * TABLE_SIZE + hi_bytes[1] as usize), - *tables_ptr.add(10 * TABLE_SIZE + hi_bytes[2] as usize), - *tables_ptr.add(11 * TABLE_SIZE + hi_bytes[3] as usize), - *tables_ptr.add(12 * TABLE_SIZE + hi_bytes[4] as usize), - *tables_ptr.add(13 * TABLE_SIZE + hi_bytes[5] as usize), - *tables_ptr.add(14 * TABLE_SIZE + hi_bytes[6] as usize), - *tables_ptr.add(15 * TABLE_SIZE + hi_bytes[7] as usize), - ) - }; - let p0 = l0 + l1; - let p1 = l2 + l3; - let p2 = l4 + l5; - let p3 = l6 + l7; - let p4 = h0 + h1; - let p5 = h2 + h3; - let p6 = h4 + h5; - let p7 = h6 + h7; - let q0 = p0 + p1; - let q1 = p2 + p3; - let q2 = p4 + p5; - let q3 = p6 + p7; - let r0 = q0 + q1; - let r1 = q2 + q3; - r0 + r1 + acc }) .collect() } -/// Tensor-split sibling of [`fold_b128_elems`]. Takes the two factors -/// `(eq_lo, eq_hi)` from [`build_eq_split`] instead of the materialized -/// suffix tensor. Each full entry `elem = eq_lo[i_lo] * eq_hi[i_hi]` is -/// reconstructed on the fly (one GF multiply per output position) and fed to -/// the same 16-byte-table lookup — the bit-decomposition the table indexes -/// does **not** factor through the `eq_lo`/`eq_hi` split, so the product must -/// be formed first. -/// -/// Output order matches the materialized tensor: `out[i_hi·B + i_lo]` with -/// `B = eq_lo.len()`, so it is **byte-identical** to -/// `fold_b128_elems(build_eq_parallel(r), eq_r_dprime)` (field multiply is -/// exact, so `eq_lo[i_lo] * eq_hi[i_hi]` has the same bits as the -/// materialized entry). -/// Number of bytes in an `F128` (= lookup tables for the fold). -const FOLD_N_BYTES: usize = 16; +/// Number of bytes in an E element (= lookup tables for the fold). +const FOLD_N_BYTES: usize = 24; /// Entries per byte-lookup table. const FOLD_TABLE_SIZE: usize = 256; -/// Build the 16×256 byte-lookup table the fold indexes: `table[k·256 + v]` = -/// `Σ_{bit b set in v} eq_r_dprime[k·8 + b]`. For the ring-switch fold, -/// `eq_r_dprime` already has γ_k baked in, so the table carries γ too. -fn build_fold_byte_table(eq_r_dprime: &[F128]) -> Vec { - assert_eq!(eq_r_dprime.len(), 1 << LOG_PACKING); - let mut tables = vec![F128::ZERO; FOLD_N_BYTES * FOLD_TABLE_SIZE]; +/// Build the 24x256 byte-lookup table for [`fold_ext_elems`]: +/// `table[k * 256 + v] = sum_{bit b set in v} coordinate_weights[k * 8 + b]`. +/// Byte order: bytes 0..8 are the little-endian bytes of `c0` (bits 0..64), +/// bytes 8..16 those of `c1` (bits 64..128), and bytes 16..24 those of `c2`. +fn build_fold_byte_table_ext(coordinate_weights: &[F192]) -> Vec { + assert_eq!(coordinate_weights.len(), DEGREE_E); + let mut tables = vec![F192::ZERO; FOLD_N_BYTES * FOLD_TABLE_SIZE]; for byte_idx in 0..FOLD_N_BYTES { let bit_base = byte_idx * 8; for value in 0..FOLD_TABLE_SIZE { - let mut acc = F128::ZERO; + let mut acc = F192::ZERO; for bit_in_byte in 0..8 { if (value >> bit_in_byte) & 1 == 1 { - acc += eq_r_dprime[bit_base + bit_in_byte]; + acc += coordinate_weights[bit_base + bit_in_byte]; } } tables[byte_idx * FOLD_TABLE_SIZE + value] = acc; @@ -1412,1245 +505,1068 @@ fn build_fold_byte_table(eq_r_dprime: &[F128]) -> Vec { tables } -/// One folded output slot: `Σ_{k=0..16} tables[k·256 + byte_k(elem)]`, where -/// `byte_k` are the 16 little-endian bytes of `elem`. `tables` MUST be a -/// `build_fold_byte_table` output (length `16·256`). Tree-reduced (depth 4) -/// rather than a length-15 XOR chain so the adds pipeline. +/// One folded output slot: `sum_{k=0..24} tables[k * 256 + byte_k(elem)]`, +/// tree-reduced (depth 4) so the XORs pipeline. `tables` MUST be a +/// [`build_fold_byte_table_ext`] output (length 24 * 256). Mirror of +/// `ring_switch::fold_one_slot` with `(c0, c1)` in place of `(lo, hi)`. #[inline(always)] -pub(crate) fn fold_one_slot(elem: F128, tables: &[F128]) -> F128 { +fn fold_one_slot_ext(elem: F192, tables: &[F192]) -> F192 { debug_assert_eq!(tables.len(), FOLD_N_BYTES * FOLD_TABLE_SIZE); - let lo_bytes = elem.lo.to_le_bytes(); - let hi_bytes = elem.hi.to_le_bytes(); - let tables_ptr = tables.as_ptr(); - // SAFETY: byte values are u8 (0..256); the max offset is - // `15·256 + 255 = 4095 = 16·256 − 1`, in-bounds for the asserted length. - let (l0, l1, l2, l3, l4, l5, l6, l7, h0, h1, h2, h3, h4, h5, h6, h7) = unsafe { - ( - *tables_ptr.add(lo_bytes[0] as usize), - *tables_ptr.add(FOLD_TABLE_SIZE + lo_bytes[1] as usize), - *tables_ptr.add(2 * FOLD_TABLE_SIZE + lo_bytes[2] as usize), - *tables_ptr.add(3 * FOLD_TABLE_SIZE + lo_bytes[3] as usize), - *tables_ptr.add(4 * FOLD_TABLE_SIZE + lo_bytes[4] as usize), - *tables_ptr.add(5 * FOLD_TABLE_SIZE + lo_bytes[5] as usize), - *tables_ptr.add(6 * FOLD_TABLE_SIZE + lo_bytes[6] as usize), - *tables_ptr.add(7 * FOLD_TABLE_SIZE + lo_bytes[7] as usize), - *tables_ptr.add(8 * FOLD_TABLE_SIZE + hi_bytes[0] as usize), - *tables_ptr.add(9 * FOLD_TABLE_SIZE + hi_bytes[1] as usize), - *tables_ptr.add(10 * FOLD_TABLE_SIZE + hi_bytes[2] as usize), - *tables_ptr.add(11 * FOLD_TABLE_SIZE + hi_bytes[3] as usize), - *tables_ptr.add(12 * FOLD_TABLE_SIZE + hi_bytes[4] as usize), - *tables_ptr.add(13 * FOLD_TABLE_SIZE + hi_bytes[5] as usize), - *tables_ptr.add(14 * FOLD_TABLE_SIZE + hi_bytes[6] as usize), - *tables_ptr.add(15 * FOLD_TABLE_SIZE + hi_bytes[7] as usize), - ) - }; - // Level 1: 8 pair sums. - let p0 = l0 + l1; - let p1 = l2 + l3; - let p2 = l4 + l5; - let p3 = l6 + l7; - let p4 = h0 + h1; - let p5 = h2 + h3; - let p6 = h4 + h5; - let p7 = h6 + h7; - // Level 2. - let q0 = p0 + p1; - let q1 = p2 + p3; - let q2 = p4 + p5; - let q3 = p6 + p7; - // Level 3. - let r0 = q0 + q1; - let r1 = q2 + q3; - // Level 4. - r0 + r1 -} - -/// Per-output-index value of a [`RsEqInd::DeferredDense`] fold (the value the -/// materialized `fold_b128_elems_split` would store at position `j`): -/// `fold_one_slot(eq_lo[j & (B−1)] · eq_hi[j >> log2 B], table)`, `B = eq_lo.len()`. -#[inline(always)] -pub(crate) fn deferred_dense_value( - eq_lo: &[F128], - eq_hi: &[F128], - table: &[F128], - log_b: usize, - j: usize, -) -> F128 { - let mask = (1usize << log_b) - 1; - fold_one_slot(eq_lo[j & mask] * eq_hi[j >> log_b], table) -} - -pub fn fold_b128_elems_split(eq_lo: &[F128], eq_hi: &[F128], eq_r_dprime: &[F128]) -> Vec { - let tables = build_fold_byte_table(eq_r_dprime); - fold_b128_from_table(eq_lo, eq_hi, &tables) + let bytes = [elem.c0.to_le_bytes(), elem.c1.to_le_bytes(), elem.c2.to_le_bytes()]; + let mut acc = F192::ZERO; + for (word, word_bytes) in bytes.iter().enumerate() { + for (byte, &value) in word_bytes.iter().enumerate() { + acc += tables[(8 * word + byte) * FOLD_TABLE_SIZE + value as usize]; + } + } + acc } -/// Materialize a split-tensor fold from a prebuilt byte `tables` -/// (`build_fold_byte_table` output). Block-parallel over `eq_hi`: each rayon -/// task sweeps one `e_hi` over all of `eq_lo` (so `e_hi` is hoisted once per -/// block). Used to un-defer a [`RsEqInd::DeferredDense`] in the pcs combine's -/// general (mixed/sparse/packed-direct) fallback path. -pub(crate) fn fold_b128_from_table(eq_lo: &[F128], eq_hi: &[F128], tables: &[F128]) -> Vec { - use rayon::prelude::*; - let b = eq_lo.len(); - // Each slot is written exactly once (`*slot = acc`) before any read. - let mut out = primitives::scratch::take_f128(b * eq_hi.len()); - out.par_chunks_mut(b) - .zip(eq_hi.par_iter()) - .for_each(|(out_block, &e_hi)| { - for (i_lo, slot) in out_block.iter_mut().enumerate() { - *slot = fold_one_slot(eq_lo[i_lo] * e_hi, tables); - } - }); - out +/// Deferred, gamma-baked ring-switch output used by the stacked opener. +/// +/// Keeping the split eq factors and the tiny byte table avoids materializing +/// one full `rs_eq_ind` vector per claim. The table already contains the +/// claim's batching scalar, so combining several claims needs only additions. +pub(crate) struct DeferredRingSwitchOutput { + pub(crate) batched_sumcheck_claim: F192, + eq_lo: Vec, + eq_hi: Vec, + table: Vec, } -// --------------------------------------------------------------------------- -// Sparse-tensor fast path. -// -// When the suffix `x_outer[1..]` has `k` coords exactly equal to `F128::ZERO` -// (as is the case for the hash-chain ẑ-opening, whose `x_inner_rest` is padded -// with trailing zeros), `build_eq` zeros out half the table per zero coord — -// so `1 − 2^{-k}` of the suffix tensor is zero and contributes nothing to -// `s_hat_v` (in `fold_1b_rows`) or `rs_eq_ind` (in `fold_b128_elems`). The -// sparse kernels touch only the `2^{-k}` support and produce byte-identical -// outputs to the dense kernels. -// -// Claims with fewer than `SPARSE_ZERO_THRESHOLD` zero coords stay on the dense -// (MFR / 8-wide) path; the crossover threshold of 3 is conservative — at 3 -// zeros the support is 1/8 of the suffix length, plenty to amortize the -// sparse fold's per-entry overhead. -// --------------------------------------------------------------------------- - -/// Minimum number of exactly-zero suffix coords for a claim to be routed -/// through the sparse kernels instead of the dense MFR fold. -const SPARSE_ZERO_THRESHOLD: usize = 3; - -/// Sparse representation of `build_eq(coords)` when `coords` contains exact -/// `F128::ZERO` entries: stores values at the compact (live) tensor positions -/// and a `live_positions` table that maps compact bit `j` → original coord -/// position. Avoids materializing the scattered `(full_idx, val)` pairs — -/// consumers compute the scattered idx on-the-fly via [`Self::scatter_idx`] -/// (a bit-deposit / pdep operation) at the point of use. -#[derive(Clone, Debug)] -pub struct SparseEqTensor { - /// `build_eq(live_coords)` — length `2^live_positions.len()`. - pub live_tensor: Vec, - /// Original-coord positions of each live coord, ascending. So compact bit - /// `j` of an enumeration index maps to bit `live_positions[j]` of the full - /// scattered index. - pub live_positions: Vec, +/// Finish a ring-switch claim without materializing its dense weight vector. +/// The batching scalar is baked into both the target and the byte table. +pub(crate) fn prove_finish_deferred( + state: RingSwitchProveState, + coordinate_weights: &[F192], + gamma: F192, +) -> DeferredRingSwitchOutput { + let s_hat_u = transpose_s_hat(&state.s_hat_v); + let sumcheck_claim = inner_product_base_ext(&s_hat_u, coordinate_weights); + let scaled_weights: Vec = coordinate_weights.iter().map(|&x| gamma * x).collect(); + DeferredRingSwitchOutput { + batched_sumcheck_claim: gamma * sumcheck_claim, + eq_lo: state.eq_lo, + eq_hi: state.eq_hi, + table: build_fold_byte_table_ext(&scaled_weights), + } } -impl SparseEqTensor { - /// Compact-to-scattered index translation: deposit the live bits of `c` - /// into the original-coord positions. Inline so consumers' hot loops fuse - /// this with their own per-entry work. - /// - /// (Tried backing this with per-byte 256-entry LUTs to reduce the - /// 19-iteration loop to 3 LUT reads + ORs at chain scale. Measured wash - /// on the keccak chain m=30 bench — LLVM auto-pipelines the iterative - /// bit-deposit so aggressively that the per-entry scatter is already at - /// the noise floor.) - #[inline(always)] - pub fn scatter_idx(&self, c: usize) -> usize { - let mut full = 0usize; - for (j, &pos) in self.live_positions.iter().enumerate() { - full |= ((c >> j) & 1) << pos; - } - full - } +/// Fold several deferred claims directly into their final combined dense +/// basis. Every output slot is written exactly once; no per-claim dense +/// vectors are allocated or read back. +pub(crate) fn combine_deferred_into(outputs: &[DeferredRingSwitchOutput], out: &mut [F192]) { + use rayon::prelude::*; - /// Materialize the scattered `(idx, val)` pairs. Test-oracle / external - /// consumers that genuinely need the materialized form should call this; - /// the prover hot path leaves the entries deferred via `scatter_idx`. - pub fn materialize(&self) -> Vec<(usize, F128)> { - self.live_tensor + assert!(!outputs.is_empty()); + let block_len = outputs[0].eq_lo.len(); + assert!(block_len.is_power_of_two()); + assert!( + outputs .iter() - .enumerate() - .map(|(c, &v)| (self.scatter_idx(c), v)) - .collect() - } + .all(|o| { o.eq_lo.len() == block_len && o.eq_lo.len() * o.eq_hi.len() == out.len() }) + ); + out.par_chunks_mut(block_len).enumerate().for_each(|(hi, out_block)| { + for (claim_idx, claim) in outputs.iter().enumerate() { + let e_hi = claim.eq_hi[hi]; + if claim_idx == 0 { + for (slot, &e_lo) in out_block.iter_mut().zip(&claim.eq_lo) { + *slot = fold_one_slot_ext(e_lo * e_hi, &claim.table); + } + } else { + for (slot, &e_lo) in out_block.iter_mut().zip(&claim.eq_lo) { + *slot += fold_one_slot_ext(e_lo * e_hi, &claim.table); + } + } + } + }); } -/// Build the sparse `build_eq(coords)` representation, skipping the zero-coord -/// halvings. The output's `live_tensor` is the `build_eq` table over only the -/// nonzero coords (length `2^live_count`); the scattered (full) index for -/// compact entry `c` is reconstructed lazily via [`SparseEqTensor::scatter_idx`]. -/// -/// O(2^live_count) time and memory, vs the dense `build_eq`'s `O(2^coords.len())`. -pub fn build_eq_sparse(coords: &[F128]) -> SparseEqTensor { - let live_positions: Vec = coords - .iter() - .enumerate() - .filter_map(|(i, &c)| if c == F128::ZERO { None } else { Some(i) }) - .collect(); - let live_coords: Vec = live_positions.iter().map(|&i| coords[i]).collect(); - // Sequential build_eq. `build_eq_parallel` *does* save ~0.4 ms on the build - // itself at 19 live coords, but the downstream `fold_1b_rows_sparse` / - // `fold_b128_elems_sparse_pairs` then pay cross-core L2/L3 traffic to - // consume a tensor that was distributed across worker caches — net wash to - // slight loss at the ring_switch level. Keep the tensor cache-local here. - let live_tensor = build_eq(&live_coords); - SparseEqTensor { - live_tensor, - live_positions, - } +/// Bytewise-table accelerated `fold_ext_elems_naive`: 24 lookup tables of +/// 256 E entries each; +/// per position 24 lookups + 23 XORs, no +/// data-dependent bit-scan. Rayon across positions. +/// Split point for the factored eq build: low half sized ~n/2 (min 4, the +/// point where two factor tables beat one full build). Mirror of the extension-field +/// layer's `ring_switch::split_n_lo`. +pub fn split_n_lo(n: usize) -> usize { + (n / 2).clamp(4.min(n), n) } -/// Sparse counterpart of one column of [`fold_1b_rows_multi`]: scans only the -/// nonzero entries of the suffix tensor. Iterates compact (live-only) tensor -/// indices and computes the scattered `packed_witness` index inline via -/// [`SparseEqTensor::scatter_idx`] — avoids materializing the scattered -/// `(idx, val)` pairs upfront. -/// -/// Produces the same 128-entry `s_hat_v` as -/// `fold_1b_rows_naive(packed_witness, build_eq(coords))`, since `build_eq`'s -/// zero-coord halvings would otherwise contribute zero to every accumulator. -pub fn fold_1b_rows_sparse(packed_witness: &[F128], eq: &SparseEqTensor) -> Vec { - // Tried: MFR fast path via `fold_1b_rows_sparse_mfr_block4` for the chain's - // block-of-4 / stride-128 support pattern. **Measured a regression on - // blake3 m=29** (~2.5 ms slower at chain proof level) and roughly break- - // even on keccak. The subset-sum + transpose overhead doesn't amortize - // over only 4 entries per group when packed_witness reads are scattered - // (stride 128 = 2 KB jumps defeat the prefetcher), so the dispatch is - // scalar. - fold_1b_rows_sparse_scalar(packed_witness, eq) +/// Factored eq tensor: `eq(point, y) = eq_lo[y & (2^n_lo - 1)] * eq_hi[y >> n_lo]` +/// (LSB-first indexing, matching `build_eq_table_ext`). Materializes +/// `2^n_lo + 2^(n - n_lo)` entries instead of `2^n`; field multiplication is +/// exact, so the reconstructed entries are bit-identical to the full build. +/// Mirror of the extension-field layer's `ring_switch::build_eq_split`. +pub fn build_eq_split_ext(point: &[F192]) -> (Vec, Vec) { + let n_lo = split_n_lo(point.len()); + (build_eq_table_ext(&point[..n_lo]), build_eq_table_ext(&point[n_lo..])) } -/// Scalar bit-scan fallback for `fold_1b_rows_sparse`. One bit-scan per support -/// entry — used when the support's index pattern isn't a uniform stride-block. -fn fold_1b_rows_sparse_scalar(packed_witness: &[F128], eq: &SparseEqTensor) -> Vec { +/// [`fold_ext_elems`] over the FACTORED tensor: each entry is reconstructed on +/// the fly (`eq_lo[a] * eq_hi[b]`, one multiply) and folded — the full +/// `2^n`-entry tensor is never materialized. Bit-identical output. +#[cfg(test)] +pub fn fold_ext_elems_split(eq_lo: &[F192], eq_hi: &[F192], coordinate_weights: &[F192]) -> Vec { use rayon::prelude::*; - let n = 1 << LOG_PACKING; - let zero_acc = || vec![F128::ZERO; n]; - - eq.live_tensor - .par_iter() - .enumerate() - .fold(zero_acc, |mut acc, (c, &val)| { - // Scatter compact c → original index (inlined bit-deposit). - let idx = eq.scatter_idx(c); - let elem = packed_witness[idx]; - let mut lo = elem.lo; - while lo != 0 { - let r = lo.trailing_zeros() as usize; - acc[r] += val; - lo &= lo - 1; - } - let mut hi = elem.hi; - while hi != 0 { - let r = hi.trailing_zeros() as usize; - acc[64 | r] += val; - hi &= hi - 1; - } - acc - }) - .reduce(zero_acc, |mut a, b| { - for r in 0..n { - a[r] += b[r]; - } - a - }) + let tables = build_fold_byte_table_ext(coordinate_weights); + let n_lo = eq_lo.len(); + debug_assert!(n_lo.is_power_of_two()); + let mask = n_lo - 1; + let shift = n_lo.trailing_zeros(); + (0..n_lo * eq_hi.len()) + .into_par_iter() + .map(|y| fold_one_slot_ext(eq_lo[y & mask] * eq_hi[y >> shift], &tables)) + .collect() } -/// Sparse counterpart of [`fold_b128_elems`] returning **sparse pairs** instead -/// of a dense vector — skips the O(L) zero-init / scatter entirely. Each pair -/// `(idx, value)` has the same per-element bit-scan over `eq_r_dprime` as the -/// dense kernel computed at that index; positions absent from the output are -/// implicitly `F128::ZERO`. Consumers must handle the sparse representation -/// (see [`RsEqInd::Sparse`]). -/// -/// Iterates compact tensor positions and scatters the index inline only at -/// emission — avoids materializing the scattered `(idx, val)` pairs upfront. -pub fn fold_b128_elems_sparse_pairs( - eq: &SparseEqTensor, - eq_r_dprime: &[F128], -) -> Vec<(usize, F128)> { +#[cfg(test)] +pub fn fold_ext_elems(suffix_tensor: &[F192], coordinate_weights: &[F192]) -> Vec { use rayon::prelude::*; - assert_eq!(eq_r_dprime.len(), 1 << LOG_PACKING); - eq.live_tensor + let tables = build_fold_byte_table_ext(coordinate_weights); + suffix_tensor .par_iter() - .enumerate() - .map(|(c, &tensor_val)| { - let mut acc = F128::ZERO; - let mut lo = tensor_val.lo; - while lo != 0 { - let b = lo.trailing_zeros() as usize; - acc += eq_r_dprime[b]; - lo &= lo - 1; - } - let mut hi = tensor_val.hi; - while hi != 0 { - let b = hi.trailing_zeros() as usize; - acc += eq_r_dprime[64 | b]; - hi &= hi - 1; - } - // Scatter compact c → original index via per-byte LUT (inlined). - (eq.scatter_idx(c), acc) - }) + .map(|&elem| fold_one_slot_ext(elem, &tables)) .collect() } // --------------------------------------------------------------------------- -// Prover / verifier of the ring-switching reduction. +// Prover / verifier of the reduction // --------------------------------------------------------------------------- -// (No RingSwitchProof struct: the prover message — `s_hat_v`, the 128 -// slice-MLEs at the suffix point — rides the shared transcript stream.) - -/// Per-claim output of [`prove_batched_padded_with_precomputed`]: the claim's -/// `rs_eq_ind` weight (dense, deferred-dense, or sparse) and its sumcheck -/// claim. The verifier recomputes the claim via [`verify_bind`] inside the -/// stacked verifier (`pcs::verify_opening_batch_mixed_ligerito_stacked`). -pub struct RingSwitchBatchOutput { - /// For dense claims this is `γ_k · B_k` — γ is baked into the byte - /// table during the fold inside `prove_batched_padded_with_precomputed`, - /// so pcs's combine just adds it without per-slot γ-mul. For sparse - /// claims `γ_k · entries` are baked similarly. - pub rs_eq_ind: RsEqInd, - pub sumcheck_claim: F128, +/// The prover message: the 64 bit-slice MLEs at the suffix point. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct RingSwitchProof { + pub s_hat_v: Vec, } -/// Sparse-or-dense representation of `rs_eq_ind`. All variants here have γ_k -/// pre-multiplied in (see `RingSwitchBatchOutput`). +/// What both prover and (dense) verifier compute as a result of the +/// reduction: the transparent weight vector and the Ligerito target. +#[cfg(test)] #[derive(Clone, Debug)] -pub enum RsEqInd { - Dense(Vec), - /// Deferred dense: the `γ_k·B_k` buffer is **not** materialized. Instead the - /// fold ingredients (`build_eq_split` factors + the γ-baked byte table) are - /// carried so pcs's combine can fold each slot on the fly and accumulate it - /// straight into `b_combined` — avoiding a 2^(m-7) materialize + readback - /// per claim. `value(j) = deferred_dense_value(eq_lo, eq_hi, table, log2(B), j)`, - /// `B = eq_lo.len()`; byte-identical to `Dense(fold_b128_elems_split(..))`. - DeferredDense { - eq_lo: Vec, - eq_hi: Vec, - table: Vec, - }, - Sparse { - len: usize, - entries: Vec<(usize, F128)>, - }, +pub struct RingSwitchOutput { + pub rs_eq_ind: Vec, + pub sumcheck_claim: F192, } -impl RsEqInd { - /// Logical length of the underlying vector. - pub fn dense_len(&self) -> usize { - match self { - Self::Dense(v) => v.len(), - Self::DeferredDense { eq_lo, eq_hi, .. } => eq_lo.len() * eq_hi.len(), - Self::Sparse { len, .. } => *len, - } - } - - /// Materialize the dense view. O(L) regardless of variant; use sparingly. - pub fn to_dense(&self) -> Vec { - match self { - Self::Dense(v) => v.clone(), - Self::DeferredDense { - eq_lo, - eq_hi, - table, - } => { - let log_b = eq_lo.len().trailing_zeros() as usize; - let l = eq_lo.len() * eq_hi.len(); - (0..l) - .map(|j| deferred_dense_value(eq_lo, eq_hi, table, log_b, j)) - .collect() - } - Self::Sparse { len, entries } => { - let mut out = vec![F128::ZERO; *len]; - for &(idx, val) in entries { - out[idx] = val; - } - out - } - } - } - +/// Verifier-side output of [`verify_finish`]: everything needed to drive +/// the Ligerito consistency check without materializing `rs_eq_ind`. +#[derive(Clone, Debug)] +pub struct RingSwitchVerifierOutput { + pub sumcheck_claim: F192, + /// Images of the coordinate basis under the batching map; feed them to + /// [`eval_rs_eq`] at the Ligerito final point. + pub coordinate_weights: Vec, } #[derive(Clone, Debug, PartialEq, Eq)] pub enum VerifyError { ClaimMismatch, - /// The proof stream ran out while reading `s_hat_v`. - Transcript(fiat_shamir::transcript::Error), } -/// THE batched ring-switching prover. Accepts an optional precomputed -/// `s_hat_v` per claim. When `precomputed_s_hat_v[i] = Some(v)` for claim `i`, -/// the prover skips that claim's `fold_1b_rows` work and uses `v` directly as -/// `s_hat_v` for the per-opening tail (sumcheck_claim, rs_eq_ind, transcript -/// observe). The eq tensor (`eq_lo`/`eq_hi` or sparse support) is still built -/// because `fold_b128_elems_split` needs it for `rs_eq_ind`. -/// -/// Use case: AB-claim opening when lincheck's pre-sumcheck `z_vec` is -/// available — see [`s_hat_v_from_z_vec`] and `prover::open_claims`. +/// Prover side of the reduction. /// -/// `precomputed_s_hat_v` must be `&[]` (no precomputes) or have length equal -/// to `x_outers.len()`. Each precomputed slice must be length `2^LOG_PACKING`. +/// Inputs: +/// - `packed_witness`: `2^L` K words (L = m - 6), from +/// [`super::pack::pack_witness`]. +/// - `prefix_weights`: the 64 per-bit-column weights of the consumed claim +/// ([`eq_prefix_weights`] for a plain point; phi_8 Lagrange weights mapped +/// directly in the tower for flock's skip claim). +/// - `suffix_point`: the L outer coords (in E) addressing words. +/// - `claim`: the claimed value `sum_i prefix_weights[i] * s_hat_v[i]`; +/// asserted against the witness (an honest caller always passes a +/// consistent claim, so this is a cheap integration check, 64 E-mults). +/// - `sponge` for sampling the row-batching map challenges. /// -/// Output is **byte-identical** to the no-precompute path when the precomputed -/// `s_hat_v` is honest (matches what `fold_1b_rows` would produce). Transcript -/// observes the same bytes in the same order. -pub fn prove_batched_padded_with_precomputed( - packed_witness: &[F128], - x_outers: &[&[F128]], - precomputed_s_hat_v: &[Option<&[F128]>], - padding: &PaddingSpec, - ps: &mut ProverState, -) -> (Vec, Vec) { - assert!(!x_outers.is_empty()); - let trace = std::env::var("PCS_TRACE").is_ok(); - let n = x_outers.len(); - let l = packed_witness.len(); - for x in x_outers { - assert!(!x.is_empty()); - assert_eq!(l, 1 << (x.len() - 1)); - } - assert!( - precomputed_s_hat_v.is_empty() || precomputed_s_hat_v.len() == n, - "precomputed_s_hat_v: must be empty or length {n}, got {}", - precomputed_s_hat_v.len(), +/// Output: the proof message `s_hat_v` (64 E values) plus the Ligerito +/// inputs `(rs_eq_ind, sumcheck_claim)`; open with +/// `recursive_prover_with_basis(config, packed, rs_eq_ind, sumcheck_claim, ..)`. +#[cfg(test)] +pub fn prove( + packed_witness: &[F64], + prefix_weights: &[F192], + suffix_point: &[F192], + claim: F192, + precomputed_s_hat_v: Option<&[F192]>, + sponge: &mut Sponge, +) -> (RingSwitchProof, RingSwitchOutput) { + assert_eq!(prefix_weights.len(), PACKING_WIDTH); + assert_eq!( + packed_witness.len(), + 1usize << suffix_point.len(), + "packed witness must have 2^|suffix_point| words" ); - let n_packed = 1usize << LOG_PACKING; - for p in precomputed_s_hat_v.iter().flatten() { - assert_eq!( - p.len(), - n_packed, - "precomputed_s_hat_v entry must have length 2^LOG_PACKING" - ); - } - // Per-orig-claim "precomputed?" predicate. Empty precomputed slice → all - // claims need fold (matches the existing behavior bit-for-bit). - let has_precomputed = - |orig: usize| -> bool { precomputed_s_hat_v.get(orig).copied().flatten().is_some() }; - - // 1. Classify each claim. Claims whose suffix `x_outer[1..]` has at least - // `SPARSE_ZERO_THRESHOLD` exactly-zero coords (e.g. the hash-chain - // ẑ-claim) skip the dense kernels entirely; the rest fuse through the - // existing MFR/8-wide multi-fold. Pulling sparse claims out also - // restores k==2 (the MFR fast-path threshold in `fold_1b_rows_multi`) - // when there are exactly two dense claims — the common case. - #[derive(Clone, Copy)] - enum Kind { - Dense(usize), - Sparse(usize), - } - let mut kinds: Vec = Vec::with_capacity(n); - let mut dense_suffixes: Vec<&[F128]> = Vec::new(); - let mut sparse_suffixes: Vec<&[F128]> = Vec::new(); - // Map dense/sparse claim index back to the original `x_outers` index — used - // to look up precomputed slots without recomputing the classification. - let mut dense_to_orig: Vec = Vec::new(); - let mut sparse_to_orig: Vec = Vec::new(); - for (orig, x) in x_outers.iter().enumerate() { - let suffix = &x[1..]; - let n_zeros = suffix.iter().filter(|&&c| c == F128::ZERO).count(); - if n_zeros >= SPARSE_ZERO_THRESHOLD { - kinds.push(Kind::Sparse(sparse_suffixes.len())); - sparse_to_orig.push(orig); - sparse_suffixes.push(suffix); - } else { - kinds.push(Kind::Dense(dense_suffixes.len())); - dense_to_orig.push(orig); - dense_suffixes.push(suffix); - } - } + // Single-claim wrapper: observe s_hat_v, sample its map, finish. The + // STACKED opener instead calls `prove_observe` for every claim, samples one + // shared map after all are observed, then `prove_finish` per claim + // (matching the extension-field opener + the recursion guest). + let (proof, state) = prove_observe( + packed_witness, + prefix_weights, + suffix_point, + claim, + precomputed_s_hat_v, + sponge, + ); + let challenges = sample_map_challenges(sponge); + let coordinate_weights = build_coordinate_weights(&challenges); + let out = prove_finish(&state, &coordinate_weights); + (proof, out) +} - // 2. Build suffix representations. Dense claims use the tensor-split - // factorization (two ~2^(n/2) factors instead of the full 2^n tensor) - // whenever `len` is a whole number of 16-wide MFR chunks — i.e. all - // real workloads. The split keeps `build_eq` off the critical path and - // lets the fold skip streaming the multi-MB tensor (see - // `fold_1b_rows_split`). Tiny test sizes (len not divisible by 16) fall - // back to the materialized tensor + the legacy multi-fold. - let use_split = l.is_multiple_of(16); - let t = std::time::Instant::now(); - let dense_splits: Vec<(Vec, Vec)> = if use_split { - dense_suffixes - .iter() - .map(|s| build_eq_split(s, split_n_lo(s.len()))) - .collect() - } else { - Vec::new() - }; - let dense_tensors: Vec> = if use_split { - Vec::new() - } else { - dense_suffixes - .iter() - .map(|s| build_eq_parallel(s)) - .collect() - }; - let sparse_supports: Vec = - sparse_suffixes.iter().map(|s| build_eq_sparse(s)).collect(); - if trace { - eprintln!( - " [rs::prove_batched] build_eq dense×{} ({}) + sparse×{}: {:6.2} ms", - dense_suffixes.len(), - if use_split { "split" } else { "full" }, - sparse_supports.len(), - t.elapsed().as_secs_f64() * 1e3 - ); - } +/// Prover-side scratch carried from [`prove_observe`] into finalization +/// (the batching-independent data: the slice-MLE vector and the factored eq tensor). +#[derive(Clone)] +pub struct RingSwitchProveState { + s_hat_v: Vec, + eq_lo: Vec, + eq_hi: Vec, +} - // 3. fold_1b_rows: split inner-then-outer fold per dense claim (or the - // legacy fused MFR multi-fold for tiny non-split sizes); per-claim - // sparse scan for the rest. - // - // Precomputed claims skip fold_1b_rows entirely — their s_hat_v is - // supplied by the caller. dense_s_hat_v/sparse_s_hat_v are still - // indexed by classify-time index `d` / `s`; we splice precomputed - // values in at those slots and run the kernel only on the others. - let dense_needs_fold: Vec = (0..dense_suffixes.len()) - .filter(|&d| !has_precomputed(dense_to_orig[d])) - .collect(); - let sparse_needs_fold: Vec = (0..sparse_suffixes.len()) - .filter(|&s| !has_precomputed(sparse_to_orig[s])) - .collect(); - let t = std::time::Instant::now(); - let mut dense_s_hat_v: Vec> = vec![Vec::new(); dense_suffixes.len()]; - let mut sparse_s_hat_v: Vec> = vec![Vec::new(); sparse_suffixes.len()]; - // Fill precomputed slots first. - for d in 0..dense_suffixes.len() { - if let Some(p) = precomputed_s_hat_v.get(dense_to_orig[d]).copied().flatten() { - dense_s_hat_v[d] = p.to_vec(); - } - } - for s in 0..sparse_suffixes.len() { - if let Some(p) = precomputed_s_hat_v - .get(sparse_to_orig[s]) - .copied() - .flatten() - { - sparse_s_hat_v[s] = p.to_vec(); - } - } - // Run the kernel only on claims that genuinely need fold_1b_rows. - if use_split { - match dense_needs_fold.len() { - 0 => {} - 2 => { - // K=2 specialization with stack-allocated inner accumulators — - // one packed_witness streaming pass, shared transposes. - let d0 = dense_needs_fold[0]; - let d1 = dense_needs_fold[1]; - let (lo0, hi0) = (dense_splits[d0].0.as_slice(), dense_splits[d0].1.as_slice()); - let (lo1, hi1) = (dense_splits[d1].0.as_slice(), dense_splits[d1].1.as_slice()); - let (a, b) = fold_1b_rows_split_2way(packed_witness, lo0, hi0, lo1, hi1, padding); - dense_s_hat_v[d0] = a; - dense_s_hat_v[d1] = b; - } - _ => { - for &d in &dense_needs_fold { - let (eq_lo, eq_hi) = (&dense_splits[d].0, &dense_splits[d].1); - dense_s_hat_v[d] = fold_1b_rows_split(packed_witness, eq_lo, eq_hi, padding); - } - } +/// Phase 1 of the ring-switch prover: compute + observe `s_hat_v` (NO domain +/// label — matches the extension-field opener). Returns the proof and the scratch for +/// the finalization step. The caller samples the possibly shared map afterwards. +pub fn prove_observe( + packed_witness: &[F64], + prefix_weights: &[F192], + suffix_point: &[F192], + claim: F192, + precomputed_s_hat_v: Option<&[F192]>, + sponge: &mut Sponge, +) -> (RingSwitchProof, RingSwitchProveState) { + assert_eq!(prefix_weights.len(), PACKING_WIDTH); + assert_eq!( + packed_witness.len(), + 1usize << suffix_point.len(), + "packed witness must have 2^|suffix_point| words" + ); + let (eq_lo, eq_hi) = build_eq_split_ext(suffix_point); + let s_hat_v = match precomputed_s_hat_v { + Some(v) => { + assert_eq!(v.len(), PACKING_WIDTH); + v.to_vec() } - } else if !dense_needs_fold.is_empty() { - let dense_refs: Vec<&[F128]> = dense_needs_fold - .iter() - .map(|&d| dense_tensors[d].as_slice()) - .collect(); - let out = fold_1b_rows_multi_padded(packed_witness, &dense_refs, padding); - for (i, &d) in dense_needs_fold.iter().enumerate() { - dense_s_hat_v[d] = out[i].clone(); + None => { + use rayon::prelude::*; + let mask = eq_lo.len() - 1; + let shift = eq_lo.len().trailing_zeros(); + let full: Vec = (0..packed_witness.len()) + .into_par_iter() + .map(|y| eq_lo[y & mask] * eq_hi[y >> shift]) + .collect(); + fold_1b_rows(packed_witness, &full) } - } - for &s in &sparse_needs_fold { - sparse_s_hat_v[s] = fold_1b_rows_sparse(packed_witness, &sparse_supports[s]); - } - if trace { - eprintln!( - " [rs::prove_batched] fold_1b_rows dense(k={})+sparse(k={}): {:6.2} ms", - dense_s_hat_v.len(), - sparse_s_hat_v.len(), - t.elapsed().as_secs_f64() * 1e3 - ); - } + }; + assert_eq!( + claim_check(prefix_weights, &s_hat_v), + claim, + "ring_switch::prove: supplied claim does not match the witness" + ); + observe_ext_slice(sponge, &s_hat_v); + ( + RingSwitchProof { + s_hat_v: s_hat_v.clone(), + }, + RingSwitchProveState { s_hat_v, eq_lo, eq_hi }, + ) +} - // 4. Per-opening tail. Two phases: - // (a) Per claim: observe(label, s_hat_v), sample r''_i, compute - // sumcheck_claim. Stash factors needed for fold. - // (b) Sample γ_rs after all observations (Schwartz-Zippel-sound). - // (c) Per claim: bake γ_k into eq_r_dprime, fold. Output rs_eq_ind - // already has γ_k baked in — pcs combine just adds. - let t = std::time::Instant::now(); - - struct ClaimWork { - sumcheck_claim: F128, - eq_r_dprime: Vec, - } - // Bind every claim's slice first, then sample ONE r'' shared by all of - // them: each per-slice batching is sound conditioned on r'' being drawn - // after its slice is absorbed, which holds for all slices at once, and a - // recursive verifier then builds a single eq tensor and one linearized - // coefficient table for the whole batch. - let mut slices: Vec> = Vec::with_capacity(n); - for i in 0..n { - let s_hat_v: Vec = match kinds[i] { - Kind::Dense(d) => dense_s_hat_v[d].clone(), - Kind::Sparse(s) => sparse_s_hat_v[s].clone(), - }; - ps.add_scalars(&s_hat_v); - slices.push(s_hat_v); +/// Phase 2 of the ring-switch prover: given the shared coordinate weights, produce +/// the batched sumcheck claim and the transparent weight vector `rs_eq_ind`. +#[cfg(test)] +pub fn prove_finish(state: &RingSwitchProveState, coordinate_weights: &[F192]) -> RingSwitchOutput { + let s_hat_u = transpose_s_hat(&state.s_hat_v); + let sumcheck_claim = inner_product_base_ext(&s_hat_u, coordinate_weights); + let rs_eq_ind = fold_ext_elems_split(&state.eq_lo, &state.eq_hi, coordinate_weights); + RingSwitchOutput { + rs_eq_ind, + sumcheck_claim, } - let r_dprime = ps.sample_vec(LOG_PACKING); - let eq_r_dprime = build_eq(&r_dprime); - let mut work: Vec = Vec::with_capacity(n); - for s_hat_v in slices { - let s_hat_u = tensor_algebra_transpose(&s_hat_v); - let sumcheck_claim = inner_product(&s_hat_u, &eq_r_dprime); - work.push(ClaimWork { - sumcheck_claim, - eq_r_dprime: eq_r_dprime.clone(), - }); +} + +/// Verifier side of the reduction (dense: materializes `rs_eq_ind`). +/// +/// Mirrors [`prove`]'s transcript exactly; returns `ClaimMismatch` if +/// `sum_i prefix_weights[i] * s_hat_v[i] != claim`. +#[cfg(test)] +pub fn verify( + claim: F192, + prefix_weights: &[F192], + suffix_point: &[F192], + proof: &RingSwitchProof, + sponge: &mut Sponge, +) -> Result { + assert_eq!(prefix_weights.len(), PACKING_WIDTH); + assert_eq!(proof.s_hat_v.len(), PACKING_WIDTH); + + // No domain label (matches `prove`'s single-claim wrapper + the extension-field opener). + observe_ext_slice(sponge, &proof.s_hat_v); + + if claim_check(prefix_weights, &proof.s_hat_v) != claim { + return Err(VerifyError::ClaimMismatch); } - // γ_rs sampled after all RS observations — sound. Each γ_rs[k] is then - // baked into eq_r_dprime[k] before building the Φ byte table, so the - // fold output is γ_k · B_k directly. pcs combine just adds. - let gammas_rs: Vec = (0..n).map(|_| ps.sample()).collect(); + let challenges = sample_map_challenges(sponge); + let coordinate_weights = build_coordinate_weights(&challenges); - let results: Vec = work - .into_iter() - .zip(gammas_rs.iter()) - .enumerate() - .map(|(i, (w, &g))| { - let scaled_eq_r_dprime: Vec = w.eq_r_dprime.iter().map(|x| g * *x).collect(); - let rs_eq_ind = match kinds[i] { - Kind::Dense(d) => { - if use_split { - // Defer the fold: carry the split factors + γ-baked byte - // table so pcs's combine folds each slot directly into - // `b_combined` (no 2^(m-7) materialize + readback). The - // table build is the only work done here (16·256 adds). - let (eq_lo, eq_hi) = &dense_splits[d]; - RsEqInd::DeferredDense { - eq_lo: eq_lo.clone(), - eq_hi: eq_hi.clone(), - table: build_fold_byte_table(&scaled_eq_r_dprime), - } - } else { - RsEqInd::Dense(fold_b128_elems(&dense_tensors[d], &scaled_eq_r_dprime)) - } - } - Kind::Sparse(s) => RsEqInd::Sparse { - len: l, - entries: fold_b128_elems_sparse_pairs(&sparse_supports[s], &scaled_eq_r_dprime), - }, - }; - RingSwitchBatchOutput { - rs_eq_ind, - sumcheck_claim: w.sumcheck_claim, - } - }) - .collect(); + let s_hat_u = transpose_s_hat(&proof.s_hat_v); + let sumcheck_claim = inner_product_base_ext(&s_hat_u, &coordinate_weights); - if trace { - eprintln!( - " [rs::prove_batched] per-opening tail ×{}: {:6.2} ms", - n, - t.elapsed().as_secs_f64() * 1e3 - ); - } + let suffix_tensor = build_eq_table_ext(suffix_point); + let rs_eq_ind = fold_ext_elems(&suffix_tensor, &coordinate_weights); + + Ok(RingSwitchOutput { + rs_eq_ind, + sumcheck_claim, + }) +} - (results, gammas_rs) +/// Polylog-cost verifier: same transcript as [`verify`] but does NOT build +/// the dense `rs_eq_ind`. Pair with [`eval_rs_eq`] at the Ligerito final +/// point (e.g. inside `recursive_verifier_with_basis_succinct`'s terminal +/// weight closure). +#[cfg(test)] +pub fn verify_succinct( + claim: F192, + prefix_weights: &[F192], + proof: &RingSwitchProof, + sponge: &mut Sponge, +) -> Result { + // Single-claim wrapper; the STACKED verifier observes every claim, samples + // one shared map, then finishes each claim. + verify_observe(claim, prefix_weights, proof, sponge)?; + let challenges = sample_map_challenges(sponge); + let coordinate_weights = build_coordinate_weights(&challenges); + Ok(verify_finish(proof, &coordinate_weights)) } -/// The bind + claim-check phase of the ring-switch verifier, for batch callers -/// that share one `r''` across claims: absorbs the label and slice, checks -/// the claim, samples nothing. -pub fn verify_bind( - claim: F128, - z_skip: F128, - x_outer: &[F128], - vs: &mut VerifierState<'_>, -) -> Result, VerifyError> { - assert!(!x_outer.is_empty()); - let s_hat_v = vs.next_scalars(1 << LOG_PACKING).map_err(VerifyError::Transcript)?; - let weights = build_claim_weights(z_skip, x_outer[0]); - if claim_check(&weights, &s_hat_v) != claim { +/// Phase 1 of the ring-switch verifier: observe `s_hat_v` (NO domain label — +/// matches the extension-field opener) and check the prefix-weight claim. The caller +/// samples the possibly shared map afterwards. +pub fn verify_observe( + claim: F192, + prefix_weights: &[F192], + proof: &RingSwitchProof, + sponge: &mut Sponge, +) -> Result<(), VerifyError> { + assert_eq!(prefix_weights.len(), PACKING_WIDTH); + assert_eq!(proof.s_hat_v.len(), PACKING_WIDTH); + observe_ext_slice(sponge, &proof.s_hat_v); + if claim_check(prefix_weights, &proof.s_hat_v) != claim { return Err(VerifyError::ClaimMismatch); } - Ok(s_hat_v) + Ok(()) +} + +/// Phase 2 of the ring-switch verifier: given the shared coordinate weights, +/// produce the batched sumcheck claim. +pub fn verify_finish(proof: &RingSwitchProof, coordinate_weights: &[F192]) -> RingSwitchVerifierOutput { + let s_hat_u = transpose_s_hat(&proof.s_hat_v); + let sumcheck_claim = inner_product_base_ext(&s_hat_u, coordinate_weights); + RingSwitchVerifierOutput { + sumcheck_claim, + coordinate_weights: coordinate_weights.to_vec(), + } } -/// Polylog-cost evaluation of `MLE(rs_eq_ind)(query)` at the opening's final -/// challenge point (DP24 §1.3, in linearized trace form). +// --------------------------------------------------------------------------- +// Polylog evaluation of MLE(rs_eq_ind) +// --------------------------------------------------------------------------- + +/// Polylog-cost evaluation of `MLE(rs_eq_ind)(query)` at the Ligerito final +/// challenge point, following DP24 section 1.3 Figure 3 (mirror of +/// `ring_switch::eval_rs_eq` retyped to the tower). /// -/// The dense alternative — `mle_eval(&fold_b128_elems(build_eq(z_vals), -/// eq_r_dprime), query)` — costs `O(2^|z_vals|)` field operations. This -/// function costs `O(|z_vals| · 128)`: the tensor recurrence telescopes to -/// `Σ_k c_k · Π_j (z_j^{2^k} + 1 + q_j)` with `c = linearized_eq_coeffs(w)`. +/// ## Derivation /// -/// ## Arguments +/// `rs_eq_ind[y] = Phi(eq(z, y))` with `z = suffix_point` and `Phi : E -> E` +/// the F_2-linear map sending basis bit w to `coordinate_weights[w]`. So /// -/// * `z_vals` — the suffix-side coords, i.e. `x_outer[1..]` of the claim. -/// Length `ℓ' = m − 7`. -/// * `query` — the opening-sumcheck final challenges, length `ℓ'`. -/// * `eq_r_dprime` — the `eq` tensor over the sampled `r''`, length 128. +/// ```text +/// MLE(rs_eq_ind)(q) = sum_y eq(q, y) * Phi(eq(z, y)) +/// = sum_w coordinate_weights[w] * (sum_y A(y, w) * eq(q, y)) +/// ``` +/// +/// where `A(y, w) = bit_w(eq(z, y))`. The inner sums are the components of +/// the tensor-algebra element `Theta = sum_y eq(q, y) (x) eq(z, y)` in +/// `E (x)_F2 E`, decomposed on the second factor's F_2 basis. Theta builds +/// iteratively because eq factorizes per coordinate: in char 2, +/// `sum_{y_j} eq(q_j, y_j) (x) eq(z_j, y_j) = 1 (x) 1 + q_j (x) 1 + 1 (x) z_j`, +/// so each step is `Theta += q_j * Theta|first + z_j * Theta|second` +/// (`scale_horizontal` / `scale_vertical`). The final `fold_vertical` +/// transposes (so rows are indexed by the z-side basis w) and folds with +/// `coordinate_weights`. +/// +/// The rectangular twist vs. the old module: the fold length is e = 192 +/// (the E-degree over F_2), not the packing width 64; the K side of the +/// reduction never appears here because `rs_eq_ind` is E-valued. /// -/// [DP24]: -pub fn eval_rs_eq(z_vals: &[F128], query: &[F128], eq_r_dprime: &[F128]) -> F128 { +/// ## Arguments +/// +/// * `z_vals`: the ring-switch suffix point, +/// length L = m - 6. +/// * `query`: the Ligerito final challenges, length L, same coordinate order. +/// * `coordinate_weights`: the 192 coordinate batching weights (from +/// [`RingSwitchVerifierOutput`]). +pub fn eval_rs_eq(z_vals: &[F192], query: &[F192], coordinate_weights: &[F192]) -> F192 { assert_eq!( z_vals.len(), query.len(), "eval_rs_eq: z_vals and query must have equal length" ); assert_eq!( - eq_r_dprime.len(), - 1 << LOG_PACKING, - "eval_rs_eq: eq_r_dprime length must be 128" + coordinate_weights.len(), + DEGREE_E, + "eval_rs_eq: coordinate_weights length must be 192" ); - // Linearized form (identical value): the tensor recurrence is the product - // Π_j (z_j⊗1 + 1⊗(1+q_j)) folded against eq_r_dprime, which telescopes to - // Σ_k c_k·Π_j (z_j^{2^k} + 1 + q_j). - let c = linearized_eq_coeffs(eq_r_dprime); - eval_rs_eq_from_coeffs(z_vals, query, &c) + + let mut eval = TensorAlgebraE::from_vertical(F192::ONE); + for (&z_i, &q_i) in z_vals.iter().zip(query.iter()) { + let vert_scaled = eval.clone().scale_vertical(z_i); + let hztl_scaled = eval.clone().scale_horizontal(q_i); + eval += &vert_scaled; + eval += &hztl_scaled; + } + eval.fold_vertical(coordinate_weights) +} + +/// Prefix-only variant of [`eval_rs_eq`]: walks `query_prefix.len()` of +/// the (z, query) pairs and returns the partially-evolved tensor element. +/// Pair with [`eval_rs_eq_finish_from_prefix_binary_q`] to share the +/// prefix across many residual positions (the succinct Ligerito closure). +pub fn eval_rs_eq_prefix(z_vals: &[F192], query_prefix: &[F192]) -> TensorAlgebraE { + assert!(query_prefix.len() <= z_vals.len()); + let mut eval = TensorAlgebraE::from_vertical(F192::ONE); + for (&z_i, &q_i) in z_vals.iter().zip(query_prefix.iter()) { + let vert_scaled = eval.clone().scale_vertical(z_i); + let hztl_scaled = eval.clone().scale_horizontal(q_i); + eval += &vert_scaled; + eval += &hztl_scaled; + } + eval +} + +/// Finish [`eval_rs_eq`] from a precomputed prefix when the query suffix +/// is **binary** (bit j of `y_bits` is the j-th suffix coord). With +/// `q_j in {0, 1}` the general step collapses (char 2) to a single vertical +/// scale: `q_j = 0` gives `(1 + z_j) * eval`, `q_j = 1` gives `z_j * eval`. +/// Mirror of `ring_switch::eval_rs_eq_finish_from_prefix_binary_q`. +pub fn eval_rs_eq_finish_from_prefix_binary_q( + prefix: &TensorAlgebraE, + z_vals_suffix: &[F192], + y_bits: u32, + coordinate_weights: &[F192], +) -> F192 { + assert_eq!(coordinate_weights.len(), DEGREE_E); + debug_assert!(z_vals_suffix.len() <= 32, "y_bits is u32; suffix > 32 not supported"); + let mut eval = prefix.clone(); + for (j, &z_i) in z_vals_suffix.iter().enumerate() { + let scalar = if (y_bits >> j) & 1 == 1 { z_i } else { F192::ONE + z_i }; + for e in eval.elems.iter_mut() { + *e *= scalar; + } + } + eval.fold_vertical(coordinate_weights) } #[cfg(test)] mod tests { use super::*; + use crate::ligerito::{ + LigeritoProof, commit, configs_for, recursive_prover_with_basis, recursive_verifier_with_basis, + recursive_verifier_with_basis_succinct, + }; + use crate::ligerito::{ProverConfig, VerifierConfig, default_config, default_verifier_config}; + use crate::merkle::Hash; use crate::pack::pack_witness; - use crate::test_rng::Rng; - use primitives::multilinear::build_eq; - - /// Reference: directly compute ẑ_skip(z_skip, x_outer) for a Boolean witness `z`. - /// - /// `ẑ_skip(z_skip, x_outer) = Σ_{i_skip ∈ {0,1}^6} ν_φ8(i_skip)(z_skip) - /// · Σ_{i_outer} eq(x_outer, i_outer) - /// · z[i_outer * 64 + i_skip]` - /// - /// This is the polynomial that the zerocheck claims at value `v`. - fn zhat_skip_reference(z: &[bool], m: usize, z_skip: F128, x_outer: &[F128]) -> F128 { - const K_SKIP: usize = 6; - let ell = 1usize << K_SKIP; - assert_eq!(z.len(), 1 << m); - assert_eq!(x_outer.len(), m - K_SKIP); - - let lambda = lagrange_weights_naive(K_SKIP, z_skip); // 64 weights - let eq_outer = build_eq(x_outer); // 2^(m-6) values - - // Index convention: z[i] for i ∈ 0..2^m, with low k_skip bits = i_skip and - // high (m - k_skip) bits = i_outer (matching pack_bits in univariate_skip). - let mut acc = F128::ZERO; - for i_outer in 0..(1usize << (m - K_SKIP)) { - let base = i_outer * ell; - // Inner = Σ_{i_skip} λ[i_skip] · z[base + i_skip], where z bits are 0/1 - // lifted to F_{2^128}. - let mut inner = F128::ZERO; - for i_skip in 0..ell { - if z[base + i_skip] { - inner += lambda[i_skip]; + + fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + #[test] + fn deferred_batch_matches_materialized_weights() { + let mut seed = 0xdec0_de01_2345_6789; + let point = (0..10) + .map(|_| F192::new(splitmix64(&mut seed), splitmix64(&mut seed), splitmix64(&mut seed))) + .collect::>(); + let coordinate_weights = (0..DEGREE_E) + .map(|_| F192::new(splitmix64(&mut seed), splitmix64(&mut seed), splitmix64(&mut seed))) + .collect::>(); + let gammas = [ + F192::new(splitmix64(&mut seed), splitmix64(&mut seed), splitmix64(&mut seed)), + F192::new(splitmix64(&mut seed), splitmix64(&mut seed), splitmix64(&mut seed)), + ]; + let states = (0..2) + .map(|_| { + let (eq_lo, eq_hi) = build_eq_split_ext(&point); + RingSwitchProveState { + s_hat_v: (0..PACKING_WIDTH) + .map(|_| F192::new(splitmix64(&mut seed), splitmix64(&mut seed), splitmix64(&mut seed))) + .collect(), + eq_lo, + eq_hi, } - } - acc += eq_outer[i_outer] * inner; - } - acc + }) + .collect::>(); + + let dense = states + .iter() + .map(|state| prove_finish(state, &coordinate_weights)) + .collect::>(); + let expected_target = dense + .iter() + .zip(gammas) + .fold(F192::ZERO, |acc, (out, gamma)| acc + gamma * out.sumcheck_claim); + let expected_basis = (0..1usize << point.len()) + .map(|i| gammas[0] * dense[0].rs_eq_ind[i] + gammas[1] * dense[1].rs_eq_ind[i]) + .collect::>(); + + let deferred = states + .into_iter() + .zip(gammas) + .map(|(state, gamma)| prove_finish_deferred(state, &coordinate_weights, gamma)) + .collect::>(); + let deferred_target = deferred + .iter() + .fold(F192::ZERO, |acc, out| acc + out.batched_sumcheck_claim); + let mut deferred_basis = vec![F192::ZERO; expected_basis.len()]; + combine_deferred_into(&deferred, &mut deferred_basis); + + assert_eq!(deferred_target, expected_target); + assert_eq!(deferred_basis, expected_basis); } - /// The key identity: with weights and s_hat_v constructed from the right - /// places, the claim-check yields `ẑ_skip(z_skip, x_outer)`. #[test] - fn claim_check_recovers_zhat_skip() { - let mut rng = Rng::new(0xAA7); - // m must satisfy m ≥ LOG_PACKING = 7 AND m ≥ K_SKIP + 1 = 7 (for x_outer - // to have at least one element so x_outer[0] is defined). m ≥ 7 suffices - // when m == K_SKIP only x_outer is length 1, x_outer[0..0] = empty suffix. - // But pack_witness needs m ≥ 7 (LOG_PACKING). Test at m in [8, 9, 10]. - for &m in &[8usize, 9, 10] { - let z = rng.bits(1 << m); - let z_skip = rng.f128(); - let x_outer: Vec = (0..(m - 6)).map(|_| rng.f128()).collect(); - - // Reference value: directly compute ẑ_skip. - let expected = zhat_skip_reference(&z, m, z_skip, &x_outer); - - // Build PCS data: pack, then compute s_hat_v naively from the packed - // witness and the suffix tensor (= eq_ind of x_outer[1..]). - let packed = pack_witness(&z, m); - let suffix_tensor = build_eq(&x_outer[1..]); // length 2^(m-7) - assert_eq!(packed.len(), suffix_tensor.len()); - let s_hat_v = fold_1b_rows_naive(&packed, &suffix_tensor); - - // Build weights and run claim check. - let weights = build_claim_weights(z_skip, x_outer[0]); - let got = claim_check(&weights, &s_hat_v); - - assert_eq!(got, expected, "claim-check mismatch at m={m}"); + fn trace_dual_basis_is_dual() { + // Tr(dual[i]·basis(j)) == δ_ij, and bit_i(y) = Tr(dual[i]·y) recovers + // coordinate bits of a few random elements. + let dual = trace_dual_basis(); + let basis = |j: usize| { + if j < 64 { + F192::new(1u64 << j, 0, 0) + } else if j < 128 { + F192::new(0, 1u64 << (j - 64), 0) + } else { + F192::new(0, 0, 1u64 << (j - 128)) + } + }; + let tr = |x: F192| { + let (mut acc, mut p) = (F192::ZERO, x); + for _ in 0..192 { + acc += p; + p = p * p; + } + acc + }; + for i in 0..DEGREE_E { + for j in 0..DEGREE_E { + let want = if i == j { F192::ONE } else { F192::ZERO }; + assert_eq!(tr(dual[i] * basis(j)), want, "duality fails at i={i}, j={j}"); + } + } + let mut s = 0xDEAD_BEEF_u64; + for _ in 0..8 { + let y = F192::new(splitmix64(&mut s), splitmix64(&mut s), splitmix64(&mut s)); + for i in 0..DEGREE_E { + let bit = if i < 64 { + (y.c0 >> i) & 1 + } else if i < 128 { + (y.c1 >> (i - 64)) & 1 + } else { + (y.c2 >> (i - 128)) & 1 + }; + let want = if bit == 1 { F192::ONE } else { F192::ZERO }; + assert_eq!(tr(dual[i] * y), want, "bit {i} extraction wrong"); + } } } - #[test] - fn weights_have_correct_length() { - let w = build_claim_weights(F128 { lo: 1, hi: 0 }, F128 { lo: 2, hi: 0 }); - assert_eq!(w.len(), 128); + fn rand_ext(s: &mut u64) -> F192 { + F192::new(splitmix64(s), splitmix64(s), splitmix64(s)) } - /// Tensor-algebra transpose is involutive (applying it twice returns the - /// original). - #[test] - fn transpose_is_involution() { - let mut rng = Rng::new(0xDEAD); - let s_hat_v: Vec = (0..128).map(|_| rng.f128()).collect(); - let twice = tensor_algebra_transpose(&tensor_algebra_transpose(&s_hat_v)); - assert_eq!(s_hat_v, twice); + fn rand_bits(m: usize, s: &mut u64) -> Vec { + (0..1usize << m).map(|_| splitmix64(s) & 1 == 1).collect() } + /// The contract between the native opener and every verifier that batches + /// from `Phi`'s coefficients (the recursion guest, the Python reference + /// verifier): weighting the COLUMN view by `build_coordinate_weights` must + /// equal applying `Phi` to the ROW view and combining with `x^j`. If this + /// drifts, the guest computes a different opening target than the prover. #[test] - fn factored_eq_coeffs_match_dense_moore_transform() { - let mut rng = Rng::new(0xFAC7_0EED); - let points = [ - vec![F128::ZERO; LOG_PACKING], - vec![F128::ONE; LOG_PACKING], - (0..LOG_PACKING).map(|_| rng.f128()).collect(), - (0..LOG_PACKING).map(|_| rng.f128()).collect(), - ]; - for r in points { - let weights = build_eq(&r); - assert_eq!( - linearized_eq_coeffs_eq(&r), - linearized_eq_coeffs(&weights), - "factored coefficient mismatch at r={r:?}" - ); + fn column_weights_match_the_row_side_linearized_map() { + let mut s = 0xF00D_BEEF_1234_5678; + for _ in 0..4 { + let challenges = std::array::from_fn(|_| rand_ext(&mut s)); + let s_hat_v: Vec = (0..PACKING_WIDTH).map(|_| rand_ext(&mut s)).collect(); + + // Column side: sum_w weights[w] * t_w over the transposed K columns. + let columns = transpose_s_hat(&s_hat_v); + let lhs = inner_product_base_ext(&columns, &build_coordinate_weights(&challenges)); + + // Row side: sum_j x^j * Phi(y_j), the guest's loop. + let x = F192::new(2, 0, 0); + let mut rhs = F192::ZERO; + let mut x_pow = F192::ONE; + for y in &s_hat_v { + let phi = apply_composed_map(*y, &challenges); + rhs += x_pow * phi; + x_pow *= x; + } + assert_eq!(lhs, rhs, "column weights and row-side Phi disagree"); } } - /// Throughput A/B of the fold_1b_rows variants at m=29 scale. `#[ignore]`d - /// (allocates/folds 64 MB buffers many times); run explicitly with - /// `cargo test --release -- --ignored --nocapture zzz_bench_fold_1b`. - /// **Padding skip is byte-identical to the dense fold.** On a packed - /// witness whose every block has bits `[useful_bits, 2^k_log)` honestly - /// zero, the `_padded` kernels must produce the exact same `(a0, a1)` as - /// the dense kernels — every skipped chunk would have contributed - /// `lookup[0] = 0` to every output position. - /// - /// Covers all three hash padding shapes for both the 8-wide and 4-wide - /// MFR kernels. + /// Expanding the composition must populate every Frobenius exponent exactly + /// once. Distinct support monomials are the property used by soundness; + /// pointwise distinct coordinate weights are neither required nor generally + /// true for every challenge tuple. #[test] - fn fold_1b_padded_matches_dense() { - // (m, k_log, useful_bits) - let cases: &[(usize, usize, usize)] = &[ - // BLAKE3: k_log=14, useful=15409 (boundary not 128-aligned) - (17, 14, 15_409), - // SHA-2: k_log=15, useful=31401 (boundary not 128-aligned) - (18, 15, 31_401), - // Keccak: k_log=16, useful=42560 (128-aligned; 35% of chunks skip) - (19, 16, 42_560), - ]; - for &(m, k_log, useful_bits) in cases { - let mut rng = Rng::new(0xCAFE_FACE_u64.wrapping_add((k_log * 31 + m) as u64)); - let total_bits = 1usize << m; - let block_size = 1usize << k_log; - let n_blocks = 1usize << (m - k_log); - - // Random witness, then zero bits [useful_bits, block_size) of every - // block. - let mut z = rng.bits(total_bits); - for blk in 0..n_blocks { - for j in useful_bits..block_size { - z[blk * block_size + j] = false; + fn composed_map_has_full_frobenius_support() { + let mut monomials = [None; LINEARIZED_TERMS]; + monomials[0] = Some([0u64; COMPOSITION_SHIFTS.len()]); + for (stage, &shift) in COMPOSITION_SHIFTS.iter().enumerate() { + let previous = monomials; + for (i, exponents) in previous.into_iter().enumerate().take(LINEARIZED_TERMS - shift) { + if let Some(mut exponents) = exponents { + for exponent in &mut exponents { + *exponent <<= shift; + } + exponents[stage] += 1; + assert!(monomials[i + shift].replace(exponents).is_none()); } } - let packed = pack_witness(&z, m); - - // Random suffix tensors of the right length. - let len = packed.len(); - let t0: Vec = (0..len).map(|_| rng.f128()).collect(); - let t1: Vec = (0..len).map(|_| rng.f128()).collect(); - let padding = PaddingSpec { - k_log, - useful_bits_per_block: useful_bits, - }; - - // The padding bits of the witness are zero, so the scalar - // bit-scan over the FULL witness is the exact reference. - let want0 = fold_1b_rows_naive(&packed, &t0); - let want1 = fold_1b_rows_naive(&packed, &t1); - if packed.len().is_multiple_of(8) { - let (a0, a1) = fold_1b_rows_2way_mfr_8wide_padded(&packed, &t0, &t1, &padding); - assert_eq!( - (&a0, &a1), - (&want0, &want1), - "8-wide mismatch: m={m}, k_log={k_log}, useful={useful_bits}" - ); - } - if packed.len().is_multiple_of(4) { - let (a0, a1) = fold_1b_rows_2way_mfr_padded(&packed, &t0, &t1, &padding); - assert_eq!( - (&a0, &a1), - (&want0, &want1), - "4-wide mismatch: m={m}, k_log={k_log}, useful={useful_bits}" - ); - } - if packed.len().is_multiple_of(16) { - let a0 = fold_1b_rows_1way_mfr_16wide_padded(&packed, &t0, &padding); - assert_eq!( - a0, want0, - "16-wide mismatch: m={m}, k_log={k_log}, useful={useful_bits}" - ); - } } - } + let monomials: std::collections::HashSet<_> = monomials.into_iter().map(Option::unwrap).collect(); + assert_eq!(monomials.len(), LINEARIZED_TERMS); + assert_eq!( + monomials.iter().map(|exponents| exponents.iter().sum::()).max(), + Some(RING_SWITCH_SOUNDNESS_DEGREE as u64) + ); - /// `build_eq_split` factors `build_eq` exactly: the outer product of the - /// two halves reconstructs every full-tensor entry bit-for-bit. - #[test] - fn build_eq_split_reconstructs_full() { - let mut rng = Rng::new(0x9911); - for &l in &[4usize, 7, 10] { - let r: Vec = (0..l).map(|_| rng.f128()).collect(); - let full = build_eq(&r); - for n_lo in 0..=l { - let (eq_lo, eq_hi) = build_eq_split(&r, n_lo); - assert_eq!(eq_lo.len(), 1 << n_lo); - assert_eq!(eq_hi.len(), 1 << (l - n_lo)); - let mask = (1usize << n_lo) - 1; - for (i, &f) in full.iter().enumerate() { - let recon = eq_lo[i & mask] * eq_hi[i >> n_lo]; - assert_eq!(recon, f, "reconstruct mismatch l={l} n_lo={n_lo} i={i}"); + let mut s = 0x1234_5678_9abc_def0; + let challenges: [F192; COMPOSITION_SHIFTS.len()] = std::array::from_fn(|_| rand_ext(&mut s)); + let mut coefficients = [F192::ZERO; LINEARIZED_TERMS]; + coefficients[0] = F192::ONE; + for (&challenge, &shift) in challenges.iter().zip(COMPOSITION_SHIFTS.iter()) { + let previous = coefficients; + for (i, mut coefficient) in previous.into_iter().enumerate().take(LINEARIZED_TERMS - shift) { + if coefficient == F192::ZERO { + continue; } + for _ in 0..shift { + coefficient = coefficient.square(); + } + coefficients[i + shift] = challenge * coefficient; } } + assert!(coefficients.iter().all(|coefficient| *coefficient != F192::ZERO)); + + let value = rand_ext(&mut s); + let mut expanded = F192::ZERO; + let mut frobenius = value; + for coefficient in coefficients { + expanded += coefficient * frobenius; + frobenius = frobenius.square(); + } + assert_eq!(apply_composed_map(value, &challenges), expanded); + } + + /// Reference s_hat_v: brute-force partial evaluation of each bit-column + /// MLE at the suffix point (direct bit-extract loop, no fold kernel). + fn s_hat_v_reference(packed: &[F64], suffix_point: &[F192]) -> Vec { + let eq_suffix = build_eq_table_ext(suffix_point); + (0..PACKING_WIDTH) + .map(|i| { + let mut acc = F192::ZERO; + for (word, &w) in packed.iter().zip(eq_suffix.iter()) { + if (word.0 >> i) & 1 == 1 { + acc += w; + } + } + acc + }) + .collect() } - /// `fold_1b_rows_split` is byte-identical to the materialized 16-wide - /// kernel for every split width, including padded (skip-engaging) shapes - /// and split blocks both smaller and larger than the padding block. + /// s_hat_v[i] must equal the MLE of the i-th bit-slice at the suffix + /// point; cross-check the fold kernel against a from-the-bits brute + /// force over the full (prefix + suffix) hypercube. #[test] - fn fold_1b_rows_split_matches_16wide() { - // (m, k_log, useful_bits): same padding shapes as - // `fold_1b_padded_matches_dense`, so chunk-skip actually engages. - let cases: &[(usize, usize, usize)] = - &[(17, 14, 15_409), (18, 15, 31_401), (19, 16, 42_560)]; - for &(m, k_log, useful_bits) in cases { - let l = m - LOG_PACKING; - let len = 1usize << l; - let mut rng = Rng::new(0x5757_u64.wrapping_add((m * 131 + k_log) as u64)); - let w: Vec = (0..len).map(|_| rng.f128()).collect(); - let r: Vec = (0..l).map(|_| rng.f128()).collect(); - let full_eq = build_eq(&r); - let padding = PaddingSpec { - k_log, - useful_bits_per_block: useful_bits, - }; - - let reference = fold_1b_rows_1way_mfr_16wide_padded(&w, &full_eq, &padding); - // Sweep n_lo across, below, and equal to the padding block width so - // the split-block vs padding-block alignment is exercised both ways. - for n_lo in 4..=l { - let (eq_lo, eq_hi) = build_eq_split(&r, n_lo); - let got = fold_1b_rows_split(&w, &eq_lo, &eq_hi, &padding); - assert_eq!( - got, reference, - "fold_1b_rows_split mismatch: m={m}, k_log={k_log}, n_lo={n_lo}" - ); + fn s_hat_v_matches_bruteforce() { + let m = 9; + let mut s = 1u64; + let bits = rand_bits(m, &mut s); + let packed = pack_witness(&bits, m); + let suffix_point: Vec = (0..m - LOG_PACKING).map(|_| rand_ext(&mut s)).collect(); + let eq_suffix = build_eq_table_ext(&suffix_point); + + let s_hat_v = fold_1b_rows(&packed, &eq_suffix); + assert_eq!(s_hat_v.len(), PACKING_WIDTH); + + // From the flat bit layout: column i is z[y * 64 + i]. + for i in 0..PACKING_WIDTH { + let mut expected = F192::ZERO; + for (y, &w) in eq_suffix.iter().enumerate() { + if bits[(y << LOG_PACKING) | i] { + expected += w; + } } - // The production chooser. - let (eq_lo, eq_hi) = build_eq_split(&r, split_n_lo(l)); - assert_eq!( - fold_1b_rows_split(&w, &eq_lo, &eq_hi, &padding), - reference, - "fold_1b_rows_split mismatch at split_n_lo: m={m}" - ); + assert_eq!(s_hat_v[i], expected, "bit column {i}"); } + assert_eq!(s_hat_v, s_hat_v_reference(&packed, &suffix_point)); } - /// `fold_1b_rows_split_2way` matches two separate `fold_1b_rows_split` - /// calls byte-for-byte across the padded/skip shapes. + /// The MFR kernel must equal the scalar bit-scan (same XOR-sums, only + /// regrouped) on random data, and the dispatcher must route both regimes + /// correctly (multiple-of-8 lengths to MFR, smaller powers of two to the + /// scalar path). #[test] - fn fold_1b_rows_split_2way_matches_per_claim() { - let cases: &[(usize, usize, usize)] = - &[(17, 14, 15_409), (18, 15, 31_401), (19, 16, 42_560)]; - for &(m, k_log, useful_bits) in cases { - let l = m - LOG_PACKING; - let len = 1usize << l; - let mut rng = Rng::new(0xBEEF_u64.wrapping_add((m * 131 + k_log) as u64)); - let w: Vec = (0..len).map(|_| rng.f128()).collect(); - let padding = PaddingSpec { - k_log, - useful_bits_per_block: useful_bits, - }; - let n_lo = split_n_lo(l); - let r0: Vec = (0..l).map(|_| rng.f128()).collect(); - let r1: Vec = (0..l).map(|_| rng.f128()).collect(); - let (lo0, hi0) = build_eq_split(&r0, n_lo); - let (lo1, hi1) = build_eq_split(&r1, n_lo); - let (got0, got1) = fold_1b_rows_split_2way(&w, &lo0, &hi0, &lo1, &hi1, &padding); - let want0 = fold_1b_rows_split(&w, &lo0, &hi0, &padding); - let want1 = fold_1b_rows_split(&w, &lo1, &hi1, &padding); - assert_eq!( - got0, want0, - "fold_1b_rows_split_2way mismatch (claim 0) m={m}" - ); + fn fold_1b_rows_mfr_matches_scalar() { + let mut s = 31u64; + for log_len in [3usize, 4, 7, 11] { + let len = 1usize << log_len; + let packed: Vec = (0..len).map(|_| F64(splitmix64(&mut s))).collect(); + let tensor: Vec = (0..len).map(|_| rand_ext(&mut s)).collect(); + let mfr = fold_1b_rows_mfr_8wide(&packed, &tensor); + let scalar = fold_1b_rows_scalar(&packed, &tensor); + assert_eq!(mfr, scalar, "MFR/scalar split at len={len}"); + assert_eq!(fold_1b_rows(&packed, &tensor), mfr, "dispatcher at len={len}"); + } + for len in [1usize, 2, 4] { + let packed: Vec = (0..len).map(|_| F64(splitmix64(&mut s))).collect(); + let tensor: Vec = (0..len).map(|_| rand_ext(&mut s)).collect(); assert_eq!( - got1, want1, - "fold_1b_rows_split_2way mismatch (claim 1) m={m}" + fold_1b_rows(&packed, &tensor), + fold_1b_rows_scalar(&packed, &tensor), + "scalar fallback at len={len}" ); } } - /// `fold_b128_elems_split` reconstructs the suffix entry on the fly and - /// matches the materialized `fold_b128_elems` for every split width. + /// Claim-check completeness (a plain point claim verifies) and soundness + /// (a wrong claim value or a tampered s_hat_v is rejected). #[test] - fn fold_b128_elems_split_matches_dense() { - let mut rng = Rng::new(0xB0B0); - for &l in &[4usize, 8, 10] { - let r: Vec = (0..l).map(|_| rng.f128()).collect(); - let full_eq = build_eq(&r); - let eq_r: Vec = (0..128).map(|_| rng.f128()).collect(); - let reference = fold_b128_elems(&full_eq, &eq_r); - for n_lo in 4..=l { - let (eq_lo, eq_hi) = build_eq_split(&r, n_lo); - let got = fold_b128_elems_split(&eq_lo, &eq_hi, &eq_r); - assert_eq!( - got, reference, - "fold_b128_elems_split mismatch l={l} n_lo={n_lo}" - ); + fn claim_check_completeness_and_soundness() { + let m = 10; + let mut s = 2u64; + let bits = rand_bits(m, &mut s); + let packed = pack_witness(&bits, m); + let point: Vec = (0..m).map(|_| rand_ext(&mut s)).collect(); + let prefix_weights = eq_prefix_weights(&point[..LOG_PACKING]); + let suffix_point = &point[LOG_PACKING..]; + + // Honest claim from the reference partials; sanity: it equals the + // full bit-MLE evaluated with the full eq table. + let s_ref = s_hat_v_reference(&packed, suffix_point); + let claim = claim_check(&prefix_weights, &s_ref); + let eq_full = build_eq_table_ext(&point); + let mut direct = F192::ZERO; + for (x, &w) in eq_full.iter().enumerate() { + if bits[x] { + direct += w; } } + assert_eq!(claim, direct, "prefix x suffix split must factor the MLE"); + + let mut ch = Sponge::new(b"rs-claim-test", &[]); + let (proof, _out) = prove(&packed, &prefix_weights, suffix_point, claim, None, &mut ch); + + let mut ch = Sponge::new(b"rs-claim-test", &[]); + assert!(verify(claim, &prefix_weights, suffix_point, &proof, &mut ch).is_ok()); + + // Wrong claim value. + let bad_claim = claim + F192::ONE; + let mut ch = Sponge::new(b"rs-claim-test", &[]); + assert_eq!( + verify(bad_claim, &prefix_weights, suffix_point, &proof, &mut ch).unwrap_err(), + VerifyError::ClaimMismatch + ); + let mut ch = Sponge::new(b"rs-claim-test", &[]); + assert_eq!( + verify_succinct(bad_claim, &prefix_weights, &proof, &mut ch).unwrap_err(), + VerifyError::ClaimMismatch + ); + + // Tampered s_hat_v. + let mut bad = proof.clone(); + bad.s_hat_v[17].c0 ^= 1; + let mut ch = Sponge::new(b"rs-claim-test", &[]); + assert_eq!( + verify(claim, &prefix_weights, suffix_point, &bad, &mut ch).unwrap_err(), + VerifyError::ClaimMismatch + ); } + /// The bytewise-table rs_eq_ind fold must match the naive bit-scan on + /// arbitrary (not necessarily eq-structured) input. + #[test] + fn rs_eq_ind_fast_matches_naive() { + let mut s = 3u64; + let tensor: Vec = (0..1usize << 8).map(|_| rand_ext(&mut s)).collect(); + let coordinate_weights: Vec = (0..DEGREE_E).map(|_| rand_ext(&mut s)).collect(); + assert_eq!( + fold_ext_elems(&tensor, &coordinate_weights), + fold_ext_elems_naive(&tensor, &coordinate_weights) + ); + } - /// `prove_batched_padded_with_precomputed` is byte-identical to the - /// no-precompute path when the supplied precomputed `s_hat_v` matches - /// what `fold_1b_rows` would have produced. Exercises every claim being - /// precomputed, the first being precomputed, and the second being - /// precomputed — covers the K=0, K=1 (claim 0 only), K=1 (claim 1 only), - /// and K=2 (no precompute) fold_1b_rows dispatch branches. + /// eval_rs_eq must agree with the dense evaluation: materialize + /// rs_eq_ind, evaluate its MLE at a random query with the eq table. + /// Also pins the prefix + binary-q variant against the full path. #[test] - fn prove_batched_with_precomputed_matches_unprecomputed() { - - let mut rng = Rng::new(0xF00D); - for &m in &[8usize, 9, 10, 11] { - let z = rng.bits(1 << m); - let x_a: Vec = (0..(m - 6)).map(|_| rng.f128()).collect(); - let x_b: Vec = (0..(m - 6)).map(|_| rng.f128()).collect(); - let packed = pack_witness(&z, m); - - // Baseline: no precomputes. The streamed proof is the two - // s_hat_v slices (128 words each). - let mut ch_base = crate::ProverState::new(b"flock-test-v0", &[]); - let (base, _) = prove_batched_padded_with_precomputed( - &packed, &[&x_a, &x_b], &[None, None], &PaddingSpec::dense(m), &mut ch_base, - ); - let base_stream = ch_base.into_proof().stream; - let s_hat_v_a = base_stream[0..128].to_vec(); - let s_hat_v_b = base_stream[128..256].to_vec(); - - let padding = PaddingSpec::dense(m); - - for &(pre_a, pre_b) in &[ - (false, false), // K=2 path (no precompute) - (true, false), // K=1 path: only claim 1 needs fold - (false, true), // K=1 path: only claim 0 needs fold - (true, true), // K=0 path: both precomputed - ] { - let pa: Option<&[F128]> = if pre_a { Some(&s_hat_v_a) } else { None }; - let pb: Option<&[F128]> = if pre_b { Some(&s_hat_v_b) } else { None }; - let mut ch = crate::ProverState::new(b"flock-test-v0", &[]); - let (got, _) = prove_batched_padded_with_precomputed( - &packed, - &[&x_a, &x_b], - &[pa, pb], - &padding, - &mut ch, - ); - assert_eq!( - ch.into_proof().stream, - base_stream, - "streamed proof mismatch (pre_a={pre_a}, pre_b={pre_b}, m={m})" - ); - assert_eq!(got[0].sumcheck_claim, base[0].sumcheck_claim); - assert_eq!(got[1].sumcheck_claim, base[1].sumcheck_claim); - assert_eq!(got[0].rs_eq_ind.to_dense(), base[0].rs_eq_ind.to_dense()); - assert_eq!(got[1].rs_eq_ind.to_dense(), base[1].rs_eq_ind.to_dense()); + fn eval_rs_eq_matches_dense() { + let l = 6; + let mut s = 4u64; + let z: Vec = (0..l).map(|_| rand_ext(&mut s)).collect(); + let challenges = std::array::from_fn(|_| rand_ext(&mut s)); + let coordinate_weights = build_coordinate_weights(&challenges); + let rs_eq_ind = fold_ext_elems(&build_eq_table_ext(&z), &coordinate_weights); + + let query: Vec = (0..l).map(|_| rand_ext(&mut s)).collect(); + let eq_query = build_eq_table_ext(&query); + let dense = inner_product_ext(&rs_eq_ind, &eq_query); + + assert_eq!(eval_rs_eq(&z, &query, &coordinate_weights), dense); + + // Prefix + binary-q path: replace the last 3 query coords by the + // bits of y and compare against the general path. + let split = l - 3; + let prefix = eval_rs_eq_prefix(&z, &query[..split]); + for y in 0..8u32 { + let mut q_bin = query[..split].to_vec(); + for j in 0..3 { + q_bin.push(if (y >> j) & 1 == 1 { F192::ONE } else { F192::ZERO }); } + assert_eq!( + eval_rs_eq_finish_from_prefix_binary_q(&prefix, &z[split..], y, &coordinate_weights), + eval_rs_eq(&z, &q_bin, &coordinate_weights), + "binary-q finish mismatch at y={y}" + ); } } - /// Degenerate path: when k_log == LOG_PACKING (so x_inner_rest is just - /// the single prefix0 coord), the kernel returns z_vec untouched. + /// The core algebraic identity of the reduction: the honest packed + /// witness satisfies the output claim, + /// `sum_y rs_eq_ind[y] * packed[y] == sumcheck_claim`. #[test] - fn s_hat_v_from_z_vec_degenerate_tail() { - let mut rng = Rng::new(0xDEAD); - let z_vec: Vec = (0..(1 << LOG_PACKING)).map(|_| rng.f128()).collect(); - let got = s_hat_v_from_z_vec(&z_vec, &[]); - assert_eq!(got, z_vec); + fn sumcheck_claim_matches_inner_product() { + let m = 12; + let mut s = 5u64; + let bits = rand_bits(m, &mut s); + let packed = pack_witness(&bits, m); + let point: Vec = (0..m).map(|_| rand_ext(&mut s)).collect(); + let prefix_weights = eq_prefix_weights(&point[..LOG_PACKING]); + let suffix_point = &point[LOG_PACKING..]; + let claim = claim_check(&prefix_weights, &s_hat_v_reference(&packed, suffix_point)); + + let mut ch = Sponge::new(b"rs-identity-test", &[]); + let (_proof, out) = prove(&packed, &prefix_weights, suffix_point, claim, None, &mut ch); + assert_eq!( + inner_product_base_ext(&packed, &out.rs_eq_ind), + out.sumcheck_claim, + "reduction output claim must hold for the honest witness" + ); } - /// `subset_sums_4` matches the obvious specification. - #[test] - fn subset_sums_4_correctness() { - let mut rng = Rng::new(0xABCD); - let elems: [F128; 4] = [rng.f128(), rng.f128(), rng.f128(), rng.f128()]; - let sums = subset_sums_4(elems); - assert_eq!(sums[0], F128::ZERO); - for mask in 0..16 { - let mut expected = F128::ZERO; - for k in 0..4 { - if (mask >> k) & 1 == 1 { - expected += elems[k]; + // -- end-to-end: reduction + ligerito opening -------------------------- + + /// Configs for a K-witness of `2^log_n` words: prefer the production + /// Secure-profile derivation; fall back to the ad-hoc default_config + /// shape at test sizes below its feasibility floor (same fallback the + /// ligerito tests use). + fn test_configs_for(log_n: usize) -> (ProverConfig, VerifierConfig) { + if let Ok(pv) = configs_for(log_n) { + return pv; + } + for bs in (1..=5).rev() { + for rate in 1..=4 { + if let (Ok(pc), Ok(vc)) = ( + default_config(log_n, bs, rate), + default_verifier_config(log_n, bs, rate), + ) { + return (pc, vc); } } - assert_eq!(sums[mask], expected, "mask={mask:04b}"); } + panic!("no feasible ligerito config at log_n = {log_n}"); } - #[test] - fn fold_b128_elems_matches_naive() { - let mut rng = Rng::new(0xF00D); - for &l in &[1usize, 4, 8, 12] { - let len = 1usize << l; - let suffix: Vec = (0..len).map(|_| rng.f128()).collect(); - let eq_r: Vec = (0..128).map(|_| rng.f128()).collect(); - let a = fold_b128_elems_naive(&suffix, &eq_r); - let b = fold_b128_elems(&suffix, &eq_r); - assert_eq!(a, b, "fold_b128_elems mismatch at L={l}"); + struct E2e { + vc: VerifierConfig, + log_n: usize, + prefix_weights: Vec, + suffix_point: Vec, + claim: F192, + root: Hash, + rs_proof: RingSwitchProof, + lig_proof: LigeritoProof, + } + + const E2E_DOMAIN: &[u8] = b"ring-switch-e2e-test"; + + /// Full prover pipeline: random bit witness, pack, commit, ring switch + /// (plain-point eq weights or a caller-supplied generalized weight + /// vector), then the ligerito opening on (rs_eq_ind, sumcheck_claim), + /// all over one continuous transcript. + fn prove_e2e(m: usize, seed: u64, generalized_weights: bool) -> E2e { + let mut s = seed; + let bits = rand_bits(m, &mut s); + let packed = pack_witness(&bits, m); + let log_n = m - LOG_PACKING; + let (pc, vc) = test_configs_for(log_n); + let (cm, pd) = commit(&packed, pc.initial_k, pc.log_inv_rates[0]); + + let suffix_point: Vec = (0..log_n).map(|_| rand_ext(&mut s)).collect(); + let prefix_weights: Vec = if generalized_weights { + // Synthetic non-eq weights (e.g. standing in for phi_8 Lagrange + // weights): any 64 E-values work. + (0..PACKING_WIDTH).map(|_| rand_ext(&mut s)).collect() + } else { + let r_prefix: Vec = (0..LOG_PACKING).map(|_| rand_ext(&mut s)).collect(); + eq_prefix_weights(&r_prefix) + }; + let claim = claim_check(&prefix_weights, &s_hat_v_reference(&packed, &suffix_point)); + + let mut ch = Sponge::new(E2E_DOMAIN, &[]); + let (rs_proof, out) = prove(&packed, &prefix_weights, &suffix_point, claim, None, &mut ch); + assert_eq!(inner_product_base_ext(&packed, &out.rs_eq_ind), out.sumcheck_claim); + let lig_proof = recursive_prover_with_basis( + &pc, + &packed, + out.rs_eq_ind, + out.sumcheck_claim, + &pd.codeword, + &pd.merkle_tree, + &mut ch, + ); + E2e { + vc, + log_n, + prefix_weights, + suffix_point, + claim, + root: cm.root, + rs_proof, + lig_proof, } } - // ----------------------------------------------------------------------- - // Sparse-tensor fast path: each sparse kernel must produce byte-identical - // output to its dense counterpart for any coord vector that mixes nonzero - // and exactly-zero entries. - // ----------------------------------------------------------------------- + /// Dense verification: ring-switch verify (rebuilds rs_eq_ind), then the + /// dense ligerito verifier with b_initial = rs_eq_ind. + fn verify_e2e_dense(e: &E2e) -> bool { + let mut ch = Sponge::new(E2E_DOMAIN, &[]); + let out = match verify(e.claim, &e.prefix_weights, &e.suffix_point, &e.rs_proof, &mut ch) { + Ok(o) => o, + Err(_) => return false, + }; + recursive_verifier_with_basis( + &e.vc, + &e.lig_proof, + &out.rs_eq_ind, + out.sumcheck_claim, + &e.root, + &mut ch, + ) + } - /// Build a coord vector with `n_zeros` exact-zero entries at the requested - /// positions and random F128s elsewhere. - fn mk_coords(rng: &mut Rng, n: usize, zero_positions: &[usize]) -> Vec { - (0..n) - .map(|i| { - if zero_positions.contains(&i) { - F128::ZERO - } else { - rng.f128() - } - }) - .collect() + /// Succinct verification: verify_succinct (no rs_eq_ind), then the + /// succinct Ligerito verifier whose terminal closure evaluates + /// MLE(rs_eq_ind) once via `eval_rs_eq`. + fn verify_e2e_succinct(e: &E2e) -> bool { + let mut ch = Sponge::new(E2E_DOMAIN, &[]); + let out = match verify_succinct(e.claim, &e.prefix_weights, &e.rs_proof, &mut ch) { + Ok(o) => o, + Err(_) => return false, + }; + let z = e.suffix_point.clone(); + let coordinate_weights = out.coordinate_weights.clone(); + recursive_verifier_with_basis_succinct( + &e.vc, + &e.lig_proof, + e.log_n, + out.sumcheck_claim, + &e.root, + |point| eval_rs_eq(&z, point, &coordinate_weights), + &mut ch, + ) } #[test] - fn build_eq_sparse_matches_dense() { - let mut rng = Rng::new(0xCAFE_F00D); - let cases: &[(usize, &[usize])] = &[ - (1, &[0]), - (4, &[1, 3]), - (6, &[0, 1, 2, 3, 4]), - (8, &[2, 3, 4, 5, 6]), - (10, &[]), - (10, &[0, 5, 9]), - ]; - for &(n_coords, zero_pos) in cases { - let coords = mk_coords(&mut rng, n_coords, zero_pos); - let dense = build_eq(&coords); - let sparse_eq = build_eq_sparse(&coords); - let materialized = sparse_eq.materialize(); - - // Sparse entries match the dense table; dense entries off the - // sparse support are exactly zero. - let mut covered = vec![false; dense.len()]; - for &(idx, val) in &materialized { - assert_eq!( - val, dense[idx], - "sparse value mismatch at idx={idx} (n={n_coords}, zeros={zero_pos:?})" - ); - assert_ne!( - val, - F128::ZERO, - "sparse entry is zero — should have been skipped" - ); - covered[idx] = true; - } - for (i, &c) in covered.iter().enumerate() { - if !c { - assert_eq!( - dense[i], - F128::ZERO, - "dense[{i}] nonzero but absent from sparse (n={n_coords}, zeros={zero_pos:?})" - ); - } - } - // Support is in ascending index order. - for w in materialized.windows(2) { - assert!(w[0].0 < w[1].0, "support not strictly ascending"); - } - // Support size = 2^live_count. - let live_count = n_coords - zero_pos.len(); - assert_eq!(sparse_eq.live_tensor.len(), 1usize << live_count); + fn end_to_end_plain_point() { + for (m, seed) in [(13usize, 10u64), (17, 11)] { + let e = prove_e2e(m, seed, false); + assert!(verify_e2e_dense(&e), "dense e2e rejected at m={m}"); + assert!(verify_e2e_succinct(&e), "succinct e2e rejected at m={m}"); } } #[test] - fn fold_1b_rows_sparse_matches_naive() { - let mut rng = Rng::new(0x5EED_DEAD); - // m large enough that the suffix has multiple coords, with a few zeros. - for &m in &[9usize, 11, 13] { - let l = m - 7; - let pw_len = 1usize << l; - let pw: Vec = (0..pw_len).map(|_| rng.f128()).collect(); - // Suffix length = l. Pin some coords to zero. - let zero_pos: Vec = (0..l.min(3)).collect(); - let suffix = mk_coords(&mut rng, l, &zero_pos); - - let dense_tensor = build_eq(&suffix); - let sparse_eq = build_eq_sparse(&suffix); - - let dense_s = fold_1b_rows_naive(&pw, &dense_tensor); - let sparse_s = fold_1b_rows_sparse(&pw, &sparse_eq); - - assert_eq!(dense_s, sparse_s, "s_hat_v mismatch at m={m}"); - } + fn end_to_end_generalized_weights() { + let e = prove_e2e(13, 12, true); + assert!(verify_e2e_dense(&e), "dense e2e (generalized) rejected"); + assert!(verify_e2e_succinct(&e), "succinct e2e (generalized) rejected"); } - - /// Cross-check `eval_rs_eq` against the dense `mle_eval(fold_b128_elems(build_eq(z_vals)), query)` - /// path at several `ℓ' = |z_vals|` values. The two must agree bit-for-bit. + /// Tampering: a bit-flip in s_hat_v breaks the claim check; a + /// claim-preserving forgery (two entries adjusted so the weighted sum is + /// unchanged) passes the claim check but diverges the FS transcript, so + /// the ligerito opening must reject it. A tampered claim value is + /// rejected outright. Dense and succinct paths must agree throughout. #[test] - fn eval_rs_eq_matches_dense() { - fn mle_eval_naive(values: &[F128], r: &[F128]) -> F128 { - assert_eq!(values.len(), 1 << r.len()); - let mut buf = values.to_vec(); - for &r_i in r.iter().rev() { - let half = buf.len() / 2; - for i in 0..half { - let lo = buf[i]; - let hi = buf[i + half]; - buf[i] = lo + r_i * (lo + hi); - } - buf.truncate(half); - } - buf[0] - } + fn end_to_end_rejects_tampering() { + let e = prove_e2e(13, 13, false); + + // Plain bit flip: caught by the claim check. + let mut bad = E2e { + rs_proof: e.rs_proof.clone(), + lig_proof: e.lig_proof.clone(), + vc: e.vc.clone(), + log_n: e.log_n, + prefix_weights: e.prefix_weights.clone(), + suffix_point: e.suffix_point.clone(), + claim: e.claim, + root: e.root, + }; + bad.rs_proof.s_hat_v[5].c1 ^= 1; + assert!(!verify_e2e_dense(&bad), "bit-flipped s_hat_v accepted"); + assert!(!verify_e2e_succinct(&bad), "bit-flipped s_hat_v accepted (succinct)"); + + // Claim-preserving forgery: s'_1 = s_1 + d, s'_0 = s_0 + w_1*d/w_0 + // keeps sum_i w_i s'_i = claim, so the claim check passes; the + // downstream opening must still reject (the batching weights and + // target diverge from what the ligerito proof was built for). + let mut s = 99u64; + let d = rand_ext(&mut s); + let w0 = e.prefix_weights[0]; + let w1 = e.prefix_weights[1]; + assert!(!w0.is_zero() && !d.is_zero()); + bad.rs_proof = e.rs_proof.clone(); + bad.rs_proof.s_hat_v[1] += d; + bad.rs_proof.s_hat_v[0] += w1 * d * w0.inv(); + assert_eq!( + claim_check(&bad.prefix_weights, &bad.rs_proof.s_hat_v), + e.claim, + "forgery must be claim-preserving for this test to bite" + ); + assert!(!verify_e2e_dense(&bad), "claim-preserving forgery accepted (dense)"); + assert!( + !verify_e2e_succinct(&bad), + "claim-preserving forgery accepted (succinct)" + ); - let mut rng = Rng::new(0xDEADBEEF); - for &l_prime in &[3usize, 6, 10, 14] { - for _trial in 0..3 { - let z_vals: Vec = (0..l_prime).map(|_| rng.f128()).collect(); - let query: Vec = (0..l_prime).map(|_| rng.f128()).collect(); - let r_dprime: Vec = (0..LOG_PACKING).map(|_| rng.f128()).collect(); - let eq_r_dprime = build_eq(&r_dprime); - - // Dense path: build_eq(z_vals) → fold_b128_elems → mle_eval at query. - let suffix_tensor = build_eq(&z_vals); - let rs_eq_ind_dense = fold_b128_elems(&suffix_tensor, &eq_r_dprime); - let dense_eval = mle_eval_naive(&rs_eq_ind_dense, &query); - - // Succinct path. - let succinct_eval = eval_rs_eq(&z_vals, &query, &eq_r_dprime); - - assert_eq!( - succinct_eval, dense_eval, - "eval_rs_eq mismatch at l_prime={l_prime}" - ); - } - } + // Tampered claim value. + bad.rs_proof = e.rs_proof.clone(); + bad.claim = e.claim + F192::ONE; + assert!(!verify_e2e_dense(&bad), "tampered claim accepted"); + assert!(!verify_e2e_succinct(&bad), "tampered claim accepted (succinct)"); } } diff --git a/crates/pcs/src/stack_open.rs b/crates/pcs/src/stack_open.rs new file mode 100644 index 00000000..03f1bff1 --- /dev/null +++ b/crates/pcs/src/stack_open.rs @@ -0,0 +1,912 @@ +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +//! Stacked batch-mixed opening for the F64-committed PCS. +//! +//! The committed witness is a stack of `2^log_n` [`F64`] words (committed via +//! [`super::ligerito::commit`]), and one Ligerito run discharges +//! +//! - **point claims** ([`StackClaim`]): plain multilinear evaluations of +//! aligned sub-slices of the stack (a `Point` claim's weight is +//! `eq(low_point, .)` supported on `[offset, offset + 2^|low_point|)`; a +//! `Strided` claim freezes the low `stride_log` in-block coords to `slot`'s +//! bits, so its weight is nonzero only at `offset + slot + j * 2^stride_log`), +//! - **ring-switched claims** ([`RingSwitchOpen`]): bit-MLE evaluation claims +//! on the packed sub-block `q_pkd = stack[offset .. offset + 2^qpkd_vars]`, +//! reduced per claim by [`super::ring_switch::prove_observe`] and the +//! deferred finish path to an inner-product +//! claim ` = sumcheck_claim` against the transparent +//! E-valued weight `rs_eq_ind`. +//! +//! All claims are gamma-folded into ONE combined weight `b_stack` over the +//! whole stack plus one `target`, then proved by +//! [`super::ligerito::recursive_prover_with_basis`]. The verifier replays +//! the ring-switch reductions succinctly ([`super::ring_switch::verify_observe`] +//! and [`super::ring_switch::verify_finish`], with no dense `rs_eq_ind`) and drives +//! [`super::ligerito::recursive_verifier_with_basis_succinct`] with a +//! terminal evaluator that reconstructs `MLE(b_stack)` once, at the final fold +//! point, using closed-form eq / stride selectors and +//! [`super::ring_switch::eval_rs_eq`]. +//! +//! ## Transcript order (identical on both sides) +//! +//! label -> per ring-switched claim ([`super::ring_switch`]'s own label + +//! `s_hat_v_i` observed + shared linear map sampled) -> gamma_rs (one per claim) -> +//! per point claim (label + value observed) -> gamma_pd (one per claim) -> +//! Ligerito, with domain-separated labels for every phase. +//! +//! ## The combined weight +//! +//! With `sel = offset >> qpkd_vars` the selector coords of the q_pkd slice, +//! the lifted weight at a full-stack point `x = (x_lo, x_hi)` (split at +//! `qpkd_vars`, LSB-first) is +//! +//! ```text +//! b(x) = eq(sel, x_hi) * sum_i gamma_rs_i * MLE(rs_eq_ind_i)(x_lo) +//! + sum_j gamma_pd_j * eq(claim_j, x) +//! ``` +//! +//! which is exactly what the dense `b_stack` scatter produces (each claim's +//! weight lives on its aligned slice, so scattering the low-dimensional eq / +//! rs_eq_ind tensor at the slice offset IS multiplying by the boolean +//! selector eq). + +use crate::merkle::Hash; +use fiat_shamir::Sponge; +use primitives::field::{F64, F192}; +use serde::{Deserialize, Serialize}; + +use super::ligerito::{ + LigeritoProof, ProverData, build_eq_table_ext, recursive_prover_with_basis, + recursive_verifier_with_basis_succinct_with_squeezes, +}; +use super::ligerito::{ProverConfig, VerifierConfig}; +use super::pack::PACKING_WIDTH; +use super::ring_switch::{self, RingSwitchProof}; + +// --------------------------------------------------------------------------- +// Sponge helpers (same convention as ligerito): E-scalars straight off +// the shared Fiat-Shamir sponge. +// Sponge scalars ARE E-elements; the helpers keep call sites uniform. Every +// 24-byte pattern is a valid F192, and observing ferries all three limbs +// through the transcript. +// --------------------------------------------------------------------------- + +fn sample_ext_vec(sponge: &mut Sponge, n: usize) -> Vec { + sponge.sample_vec(n) +} + +#[inline] +fn observe_ext(sponge: &mut Sponge, e: F192) { + sponge.observe(e); +} + +/// Multilinear eq at two E-points (char 2: each factor is `1 + r_i + x_i`). +/// Mirror of `zerocheck::multilinear::eq_eval` retyped to the tower. +fn eq_eval_ext(r: &[F192], x: &[F192]) -> F192 { + assert_eq!(r.len(), x.len()); + let mut acc = F192::ONE; + for (&a, &b) in r.iter().zip(x.iter()) { + acc *= F192::ONE + a + b; + } + acc +} + +// --------------------------------------------------------------------------- +// Claim types +// --------------------------------------------------------------------------- + +/// An owning point claim folded into the stacked mixed opening. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum StackClaim { + /// `eq(low_point, .)` on the aligned slice + /// `[offset, offset + 2^|low_point|)`; `offset` must be a multiple of + /// `2^|low_point|`. + Point { + offset: usize, + low_point: Vec, + value: F192, + }, + /// A boolean-selector claim on a packed column: the low `stride_log` + /// in-block coords are frozen to `slot`'s bits (so the weight is nonzero + /// only at `offset + slot + j * 2^stride_log`) and `point` is the high + /// part. Equivalent to a `Point` with `low_point = slot_bits ++ point`, + /// folded in `O(2^|point|)` instead of `O(2^(stride_log + |point|))`. + /// `offset` must be a multiple of `2^(stride_log + |point|)` and + /// `slot < 2^stride_log`. + Strided { + offset: usize, + slot: usize, + stride_log: usize, + point: Vec, + value: F192, + }, +} + +impl StackClaim { + #[inline] + pub fn value(&self) -> F192 { + match self { + StackClaim::Point { value, .. } | StackClaim::Strided { value, .. } => *value, + } + } +} + +/// One ring-switched evaluation claim on the q_pkd sub-block: the consumed +/// claim is `value == sum_i prefix_weights[i] * s_hat_v[i]` where `s_hat_v` +/// are the 64 bit-slice MLEs of q_pkd at `suffix_point` (see +/// [`super::ring_switch`]). `prefix_weights` has [`PACKING_WIDTH`] = 64 +/// entries ([`super::ring_switch::eq_prefix_weights`] for a plain point +/// claim; phi_8 Lagrange weights for flock's +/// univariate-skip claim); `suffix_point` has `qpkd_vars` coords. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RingSwitchClaim { + pub prefix_weights: Vec, + pub suffix_point: Vec, + pub value: F192, + /// Prover-side optional precomputed `s_hat_v` (the 64 bit-slice MLE + /// values at `suffix_point`, e.g. captured inside flock's reduction). + /// When present, [`super::ring_switch::prove_observe`] skips its + /// `fold_1b_rows` recomputation; the values are checked against the + /// claim (`claim_check`) and the transcript is identical either way. + /// Verifier-side bundles leave it `None`. + pub s_hat_v: Option>, +} + +/// Prover-side bundle of the ring-switched claims discharged in the same +/// stacked opening as the [`StackClaim`]s. Each claim may carry its +/// precomputed `s_hat_v`. +#[derive(Clone, Debug)] +pub struct RingSwitchOpen { + /// q_pkd's offset inside the committed stack; must be a multiple of + /// `2^qpkd_vars` (an aligned slice). + pub offset: usize, + /// log2 of q_pkd's length in F64 words; the opener slices + /// `q_pkd = stack[offset .. offset + 2^qpkd_vars]` (no separate copy). + pub qpkd_vars: usize, + pub claims: Vec, +} + +/// Verifier counterpart of [`RingSwitchOpen`]: identical statement data +/// (the proof travels separately as [`BatchOpeningProof`]). +#[derive(Clone, Debug)] +pub struct RingSwitchVerify { + /// q_pkd's offset inside the committed stack. + pub offset: usize, + /// log2 of q_pkd's length in F64 words. + pub qpkd_vars: usize, + pub claims: Vec, +} + +/// Batched stacked opening proof: one ring-switch message per ring-switched +/// claim plus one Ligerito proof over the combined claim. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct BatchOpeningProof { + pub ring_switches: Vec, + pub ligerito: LigeritoProof, +} + +/// What the stacked-opening verifier hands back on accept—the recursion +/// harness's hook for the Ligerito fold/query data. +#[derive(Clone, Debug, Default)] +pub struct StackedOpeningSummary { + pub lig: LigVerifierSummary, +} + +/// See [`StackedOpeningSummary`]. +#[derive(Clone, Debug, Default)] +pub struct LigVerifierSummary { + /// The raw query-sampling squeezes, per level in transcript order. + pub query_squeezes: Vec>, +} + +// --------------------------------------------------------------------------- +// Shared claim folding / evaluation +// --------------------------------------------------------------------------- + +/// Fold the gamma-weighted point claims into the stack weight `b_stack` and +/// running `target` (pure: the caller has already observed the claim values +/// and sampled `gammas` in transcript order). Mirror of the extension-field +/// `fold_stacked_point_claims`: a `Point` builds eq over ONLY its aligned +/// slice, a `Strided` scatters the eq of its high coords at the slot's +/// stride. Both scatter with `+=`, so overlapping slices accumulate +/// correctly; the OUTER loop therefore stays serial (several bus claims can +/// land on one column region), and parallelism lives inside each claim: the +/// gamma-seeded eq build ([`build_eq_table_ext_seeded_into`], parallel above +/// its level floor, into one scratch buffer reused across claims) and the +/// slice add. Small slices stay fully serial (with many tiny point claims, +/// rayon dispatch would cost more than the fold itself). The gamma seeding +/// and the serial/parallel splits are exact-field/order-preserving, so +/// `b_stack`'s bytes (and hence the proof) are unchanged relative to the +/// build-then-multiply form. +fn fold_stacked_point_claims(b_stack: &mut [F192], target: &mut F192, claims: &[StackClaim], gammas: &[F192]) { + use rayon::prelude::*; + const PAR_FOLD_THRESHOLD: usize = 1 << 14; + // One reusable eq scratch sized to the largest Point claim: a fresh + // multi-MB allocation per claim would pay the first-touch page faults anew. + let max_len = claims + .iter() + .map(|c| match c { + StackClaim::Point { low_point, .. } => 1usize << low_point.len(), + StackClaim::Strided { .. } => 0, + }) + .max() + .unwrap_or(0); + let mut scratch = primitives::alloc_uninit(max_len); + for (claim, g) in claims.iter().zip(gammas.iter()) { + let g = *g; + match claim { + StackClaim::Point { + offset, + low_point, + value, + } => { + let len = 1usize << low_point.len(); + assert!( + offset % len == 0, + "StackClaim::Point: offset must be 2^|low_point|-aligned" + ); + super::ligerito::build_eq_table_ext_seeded_uninit(low_point, g, &mut scratch[..len]); + // SAFETY: the seeded eq build initialized this entire prefix. + let eq = unsafe { std::slice::from_raw_parts(scratch.as_ptr().cast::(), len) }; + let dst = &mut b_stack[*offset..*offset + len]; + if len < PAR_FOLD_THRESHOLD { + for (bi, ei) in dst.iter_mut().zip(eq.iter()) { + *bi += *ei; + } + } else { + dst.par_iter_mut().zip(eq.par_iter()).for_each(|(bi, ei)| *bi += *ei); + } + *target += g * *value; + } + StackClaim::Strided { + offset, + slot, + stride_log, + point, + value, + } => { + // Sparse: eq over the instance `point` (2^|point| entries), + // scattered at stride 2^stride_log from the slot's position. + // Identical b_stack contribution to the dense Point with + // low_point = slot_bits ++ point, at ~2^stride_log x less work. + let stride = 1usize << stride_log; + let block = 1usize << (stride_log + point.len()); + assert!(*slot < stride, "StackClaim::Strided: slot must fit the stride"); + assert!( + offset % block == 0, + "StackClaim::Strided: offset must be 2^(stride_log + |point|)-aligned" + ); + let base = *offset + *slot; + let eq = build_eq_table_ext(point); + for (j, &ej) in eq.iter().enumerate() { + b_stack[base + j * stride] += g * ej; + } + *target += g * *value; + } + } + } +} + +/// The claim's weight `eq(full claim point, x)` at an arbitrary point `x` of +/// the full stack cube. A `Point`'s full point is `[low_point, sel_bits]`, a +/// `Strided`'s is `[slot_bits, point, sel_bits]`; neither is materialized. +/// Mirror of the extension-field `stack_claim_eq_at`. +fn stack_claim_eq_at(claim: &StackClaim, x: &[F192]) -> F192 { + match claim { + StackClaim::Point { offset, low_point, .. } => { + let n = low_point.len(); + let mut e = eq_eval_ext(low_point, &x[..n]); + let sel = offset >> n; + for (k, &xi) in x[n..].iter().enumerate() { + e *= if (sel >> k) & 1 == 1 { xi } else { F192::ONE + xi }; + } + e + } + StackClaim::Strided { + offset, + slot, + stride_log, + point, + .. + } => { + let mut e = F192::ONE; + for (k, &xi) in x[..*stride_log].iter().enumerate() { + e *= if (slot >> k) & 1 == 1 { xi } else { F192::ONE + xi }; + } + let block_vars = stride_log + point.len(); + e *= eq_eval_ext(point, &x[*stride_log..block_vars]); + let sel = offset >> block_vars; + for (k, &xi) in x[block_vars..].iter().enumerate() { + e *= if (sel >> k) & 1 == 1 { xi } else { F192::ONE + xi }; + } + e + } + } +} + +// --------------------------------------------------------------------------- +// Prover +// --------------------------------------------------------------------------- + +/// Open the committed `F64` stack: discharge every `point_claims` slice +/// evaluation AND the ring-switched q_pkd claims (`ring`) in ONE Ligerito +/// run, reusing the caller's [`super::ligerito::commit`] output as L0. +/// +/// `stack` is the committed message (the caller retains it; it is not stored +/// in [`ProverData`]); `config.initial_k` / `config.log_inv_rates[0]` must +/// match the commit's `log_batch_size` / `log_inv_rate` (enforced by shape +/// asserts inside the Ligerito prover). +pub fn open_batch_mixed_ligerito_stacked( + sponge: &mut Sponge, + stack: &[F64], + prover_data: &ProverData, + config: &ProverConfig, + point_claims: &[StackClaim], + ring: &RingSwitchOpen, +) -> BatchOpeningProof { + let qpkd_len = 1usize << ring.qpkd_vars; + assert!( + ring.offset.is_multiple_of(qpkd_len), + "q_pkd offset must be 2^qpkd_vars-aligned" + ); + assert!( + ring.offset + qpkd_len <= stack.len(), + "q_pkd slice must fit inside the stack" + ); + assert!( + !ring.claims.is_empty(), + "stacked PCS opening carries at least one ring-switched claim" + ); + // Optional phase timing, answering to the same env var as the Ligerito + // prover/commit tracing (one env lookup per open, no work when unset). + let trace = std::env::var_os("LIGERITO_TRACE").is_some(); + let mut t = std::time::Instant::now(); + let mark = |label: &str, t: &mut std::time::Instant| { + if trace { + eprintln!("[stack-open-k] {label}: {:7.2} ms", t.elapsed().as_secs_f64() * 1e3); + } + *t = std::time::Instant::now(); + }; + + // 1. Ring-switch reduction: observe every claim's s_hat_v, sample one + // shared linear map, then finish each claim against that map. + let qpkd = &stack[ring.offset..ring.offset + qpkd_len]; + let mut rs_proofs = Vec::with_capacity(ring.claims.len()); + let mut rs_states = Vec::with_capacity(ring.claims.len()); + for claim in &ring.claims { + assert_eq!( + claim.suffix_point.len(), + ring.qpkd_vars, + "ring-switch suffix point must have qpkd_vars coords" + ); + let (proof, state) = ring_switch::prove_observe( + qpkd, + &claim.prefix_weights, + &claim.suffix_point, + claim.value, + claim.s_hat_v.as_deref(), + sponge, + ); + rs_proofs.push(proof); + rs_states.push(state); + } + let map_challenges = ring_switch::sample_map_challenges(sponge); + let coordinate_weights = ring_switch::build_coordinate_weights(&map_challenges); + // Per-claim batching gammas, sampled AFTER all ring-switch messages are + // bound (mirror of the extension-field layer's gamma_rs pattern). + let gammas_rs = sample_ext_vec(sponge, ring.claims.len()); + let rs_outputs: Vec<_> = rs_states + .into_iter() + .zip(gammas_rs) + .map(|(state, gamma)| ring_switch::prove_finish_deferred(state, &coordinate_weights, gamma)) + .collect(); + mark("ring-switch proves", &mut t); + + // 2. Observe point-claim values + sample their gammas (Schwartz-Zippel + // sound: every gamma_pd is sampled after all values are observed). + for claim in point_claims { + observe_ext(sponge, claim.value()); + } + let gammas_pd = sample_ext_vec(sponge, point_claims.len()); + + // 3. Combined target and lifted stack weight b_stack: the gamma-weighted + // rs_eq_ind sum scattered at the q_pkd slice, plus the point-claim + // eq tensors scattered at their offsets. + let mut target = rs_outputs + .iter() + .fold(F192::ZERO, |acc, out| acc + out.batched_sumcheck_claim); + // Parallel first-touch wins for the tower stack: its many scattered point + // claims otherwise fault pages one claim at a time. + let mut b_stack = primitives::alloc_uninit(stack.len()); + { + use rayon::prelude::*; + const ZERO_CHUNK: usize = 1 << 16; + b_stack.par_chunks_mut(ZERO_CHUNK).for_each(|chunk| { + for value in chunk { + value.write(F192::ZERO); + } + }); + } + // SAFETY: the parallel fill initializes every stack weight to zero. + let mut b_stack = unsafe { primitives::assume_init(b_stack) }; + mark("b_stack zero fill", &mut t); + ring_switch::combine_deferred_into(&rs_outputs, &mut b_stack[ring.offset..ring.offset + qpkd_len]); + mark("rs_eq_ind scatter", &mut t); + fold_stacked_point_claims(&mut b_stack, &mut target, point_claims, &gammas_pd); + mark("point-claim folds", &mut t); + + // 4. One Ligerito over the full stack against the combined claim (the + // stack is borrowed by the prover; no copy). + let ligerito = recursive_prover_with_basis( + config, + stack, + b_stack, + target, + &prover_data.codeword, + &prover_data.merkle_tree, + sponge, + ); + BatchOpeningProof { + ring_switches: rs_proofs, + ligerito, + } +} + +// --------------------------------------------------------------------------- +// Verifier +// --------------------------------------------------------------------------- + +/// Verifier mirror of [`open_batch_mixed_ligerito_stacked`]: replay the +/// ring-switch reductions succinctly, recompute the combined target, then +/// drive the succinct Ligerito verifier with one terminal evaluation of the +/// lifted weight. `log_n` is the committed stack's log size in F64 words and +/// `root` the L0 commitment root ([`super::ligerito::Commitment::root`]). +pub fn verify_opening_batch_mixed_ligerito_stacked( + sponge: &mut Sponge, + config: &VerifierConfig, + log_n: usize, + root: &Hash, + point_claims: &[StackClaim], + ring: &RingSwitchVerify, + proof: &BatchOpeningProof, +) -> Option { + let n_rs = ring.claims.len(); + let qpkd_vars = ring.qpkd_vars; + // Caller (statement) invariants: panic on misuse, like the extension-field layer. + assert!(qpkd_vars <= log_n); + assert!( + ring.offset.is_multiple_of(1usize << qpkd_vars), + "q_pkd offset must be 2^qpkd_vars-aligned" + ); + assert!(n_rs > 0, "stacked PCS opening carries at least one ring-switched claim"); + for claim in &ring.claims { + assert_eq!(claim.prefix_weights.len(), PACKING_WIDTH); + assert_eq!(claim.suffix_point.len(), qpkd_vars); + } + // `proof` is attacker-controlled (deserialized): validate its shape and + // reject rather than panicking (`verify_succinct` asserts the + // s_hat_v length internally). + if proof.ring_switches.len() != n_rs || proof.ring_switches.iter().any(|rs| rs.s_hat_v.len() != PACKING_WIDTH) { + return None; + } + + // 1. Ring-switch succinct verify: observe every claim's s_hat_v, sample one + // shared linear map, then finish each claim (mirrors the prover and guest). + for (claim, rs_proof) in ring.claims.iter().zip(proof.ring_switches.iter()) { + if ring_switch::verify_observe(claim.value, &claim.prefix_weights, rs_proof, sponge).is_err() { + return None; + } + } + let map_challenges = ring_switch::sample_map_challenges(sponge); + let coordinate_weights = ring_switch::build_coordinate_weights(&map_challenges); + let rs_outputs: Vec<_> = proof + .ring_switches + .iter() + .map(|rs_proof| ring_switch::verify_finish(rs_proof, &coordinate_weights)) + .collect(); + let gammas_rs = sample_ext_vec(sponge, n_rs); + let mut target = F192::ZERO; + for (out, g) in rs_outputs.iter().zip(gammas_rs.iter()) { + target += *g * out.sumcheck_claim; + } + + // 2. Point-claim values + gammas; fold into the target. + for claim in point_claims { + observe_ext(sponge, claim.value()); + } + let gammas_pd = sample_ext_vec(sponge, point_claims.len()); + for (claim, g) in point_claims.iter().zip(gammas_pd.iter()) { + target += *g * claim.value(); + } + + // 3. Evaluate the lifted weight once, at the terminal sumcheck point. + let sel = ring.offset >> qpkd_vars; + let eval_b_at = |x: &[F192]| -> F192 { + let (x_lo, x_hi) = x.split_at(qpkd_vars); + let mut sel_eq = F192::ONE; + for (k, &xi) in x_hi.iter().enumerate() { + sel_eq *= if (sel >> k) & 1 == 1 { xi } else { F192::ONE + xi }; + } + let mut rs_part = F192::ZERO; + for ((claim, g), out) in ring.claims.iter().zip(gammas_rs.iter()).zip(rs_outputs.iter()) { + rs_part += *g * ring_switch::eval_rs_eq(&claim.suffix_point, x_lo, &out.coordinate_weights); + } + let mut acc = rs_part * sel_eq; + for (claim, g) in point_claims.iter().zip(gammas_pd.iter()) { + acc += *g * stack_claim_eq_at(claim, x); + } + acc + }; + + let mut query_squeezes: Vec> = Vec::new(); + let ok = recursive_verifier_with_basis_succinct_with_squeezes( + config, + &proof.ligerito, + log_n, + target, + root, + eval_b_at, + sponge, + &mut query_squeezes, + ); + ok.then_some(StackedOpeningSummary { + lig: LigVerifierSummary { query_squeezes }, + }) +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use crate::ligerito::{commit, configs_for, inner_product_base_ext}; + use crate::ligerito::{default_config, default_verifier_config}; + use crate::pack::{LOG_PACKING, pack_witness}; + use crate::ring_switch::{claim_check, eq_prefix_weights, fold_1b_rows}; + + fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn rand_ext(s: &mut u64) -> F192 { + F192::new(splitmix64(s), splitmix64(s), splitmix64(s)) + } + + /// Configs for a K-stack of `2^log_n` words: prefer the production + /// Secure-profile derivation; fall back to the ad-hoc default_config + /// shape below its feasibility floor (same fallback the sibling K test + /// modules use). + fn test_configs_for(log_n: usize) -> (ProverConfig, VerifierConfig) { + match configs_for(log_n) { + Ok(pv) => pv, + Err(_) => { + let pc = default_config(log_n, 5, 1).unwrap(); + let vc = default_verifier_config(log_n, 5, 1).unwrap(); + (pc, vc) + } + } + } + + const DOMAIN: &[u8] = b"stack-open-test"; + + struct Instance { + vc: VerifierConfig, + log_n: usize, + root: Hash, + point_claims: Vec, + ring: RingSwitchOpen, + proof: BatchOpeningProof, + } + + /// Synthetic stack of 2^14 F64 words: three aligned 2^12-word columns + /// plus a q_pkd region (a random bit-witness packed by pack) at the + /// top slice, padded with random filler. Pool: one point claim per + /// column at a random E point, one strided claim into q_pkd, one + /// ring-switched claim with plain eq prefix weights. + /// + /// q_pkd is kept SMALL (2^8 words) so the succinct verifier's residual + /// cube sits entirely above the q_pkd coords (the production regime: + /// shared tensor prefix folded once, y coords all selector-indicator, + /// nonempty E-valued selector prefix from ris); the crossing regime is + /// exercised by `stacked_open_residual_crosses_qpkd`. + fn build_instance(seed: u64) -> Instance { + let log_n = 14usize; + let col_vars = 12usize; + let col_len = 1usize << col_vars; + let qpkd_vars = 8usize; + let qpkd_offset = 3 * col_len; + let mut s = seed; + + // Three random columns, the packed bit-witness region, then filler. + let mut stack: Vec = (0..3 * col_len).map(|_| F64(splitmix64(&mut s))).collect(); + let bits: Vec = (0..1usize << (qpkd_vars + LOG_PACKING)) + .map(|_| splitmix64(&mut s) & 1 == 1) + .collect(); + stack.extend(pack_witness(&bits, qpkd_vars + LOG_PACKING)); + while stack.len() < 1 << log_n { + stack.push(F64(splitmix64(&mut s))); + } + assert_eq!(stack.len(), 1 << log_n); + + // One point claim per column, at a random E point. + let mut point_claims: Vec = (0..3) + .map(|c| { + let offset = c * col_len; + let low_point: Vec = (0..col_vars).map(|_| rand_ext(&mut s)).collect(); + let eq = build_eq_table_ext(&low_point); + let value = inner_product_base_ext(&stack[offset..offset + col_len], &eq); + StackClaim::Point { + offset, + low_point, + value, + } + }) + .collect(); + + // One strided claim into the q_pkd region: freeze the low 3 in-block + // coords to slot 5, eq over the remaining coords of the slice. + { + let stride_log = 3usize; + let slot = 5usize; + let point: Vec = (0..qpkd_vars - stride_log).map(|_| rand_ext(&mut s)).collect(); + let eq = build_eq_table_ext(&point); + let mut value = F192::ZERO; + for (j, &ej) in eq.iter().enumerate() { + value += ej.mul_base(stack[qpkd_offset + slot + (j << stride_log)]); + } + point_claims.push(StackClaim::Strided { + offset: qpkd_offset, + slot, + stride_log, + point, + value, + }); + } + + // One ring-switched claim on q_pkd (plain eq prefix weights). + let qpkd = &stack[qpkd_offset..qpkd_offset + (1 << qpkd_vars)]; + let r_prefix: Vec = (0..LOG_PACKING).map(|_| rand_ext(&mut s)).collect(); + let prefix_weights = eq_prefix_weights(&r_prefix); + let suffix_point: Vec = (0..qpkd_vars).map(|_| rand_ext(&mut s)).collect(); + let s_hat_v = fold_1b_rows(qpkd, &build_eq_table_ext(&suffix_point)); + let value = claim_check(&prefix_weights, &s_hat_v); + let ring = RingSwitchOpen { + offset: qpkd_offset, + qpkd_vars, + claims: vec![RingSwitchClaim { + prefix_weights, + suffix_point, + value, + // Exercise the fold path (no precompute). + s_hat_v: None, + }], + }; + + let (pc, vc) = test_configs_for(log_n); + // Pin the intended residual regime: the residual cube must sit + // entirely above the q_pkd coords, with at least one selector coord + // covered by ris (the E-valued sel prefix) and the rest by y bits. + let yr_log_n = log_n - pc.initial_k - pc.level_ks.iter().sum::(); + assert!( + qpkd_vars < log_n - yr_log_n, + "test shape must keep the residual cube above q_pkd (yr_log_n = {yr_log_n})" + ); + let (cm, pd) = commit(&stack, pc.initial_k, pc.log_inv_rates[0]); + let mut ch = Sponge::new(DOMAIN, &[]); + let proof = open_batch_mixed_ligerito_stacked(&mut ch, &stack, &pd, &pc, &point_claims, &ring); + + Instance { + vc, + log_n, + root: cm.root, + point_claims, + ring, + proof, + } + } + + fn verify_instance( + inst: &Instance, + point_claims: &[StackClaim], + ring_claims: &[RingSwitchClaim], + proof: &BatchOpeningProof, + ) -> bool { + let ring = RingSwitchVerify { + offset: inst.ring.offset, + qpkd_vars: inst.ring.qpkd_vars, + claims: ring_claims.to_vec(), + }; + let mut ch = Sponge::new(DOMAIN, &[]); + verify_opening_batch_mixed_ligerito_stacked( + &mut ch, + &inst.vc, + inst.log_n, + &inst.root, + point_claims, + &ring, + proof, + ) + .is_some() + } + + #[test] + fn stacked_open_roundtrip_and_tampering() { + let inst = build_instance(1); + assert!( + verify_instance(&inst, &inst.point_claims, &inst.ring.claims, &inst.proof), + "honest stacked opening rejected" + ); + + // Wrong point-claim value (dense column claim). + let mut bad_points = inst.point_claims.clone(); + if let StackClaim::Point { value, .. } = &mut bad_points[0] { + *value += F192::ONE; + } else { + unreachable!() + } + assert!( + !verify_instance(&inst, &bad_points, &inst.ring.claims, &inst.proof), + "tampered Point value accepted" + ); + + // Wrong strided-claim value. + let mut bad_points = inst.point_claims.clone(); + if let StackClaim::Strided { value, .. } = &mut bad_points[3] { + *value += F192::ONE; + } else { + unreachable!() + } + assert!( + !verify_instance(&inst, &bad_points, &inst.ring.claims, &inst.proof), + "tampered Strided value accepted" + ); + + // Wrong ring-switched claim value: rejected by the claim check. + let mut bad_ring = inst.ring.claims.clone(); + bad_ring[0].value += F192::ONE; + assert!( + !verify_instance(&inst, &inst.point_claims, &bad_ring, &inst.proof), + "tampered ring-switch value accepted" + ); + + // Tampered s_hat_v: breaks the claim check. + let mut bad_proof = inst.proof.clone(); + bad_proof.ring_switches[0].s_hat_v[17].c0 ^= 1; + assert!( + !verify_instance(&inst, &inst.point_claims, &inst.ring.claims, &bad_proof), + "tampered s_hat_v accepted" + ); + + // Tampered Ligerito proof scalars. + let mut bad_proof = inst.proof.clone(); + bad_proof.ligerito.sumcheck_transcript[0].u_0.c0 ^= 1; + assert!( + !verify_instance(&inst, &inst.point_claims, &inst.ring.claims, &bad_proof), + "tampered sumcheck u_0 accepted" + ); + let mut bad_proof = inst.proof.clone(); + bad_proof.ligerito.final_proof.yr[0].c1 ^= 1; + assert!( + !verify_instance(&inst, &inst.point_claims, &inst.ring.claims, &bad_proof), + "tampered final yr accepted" + ); + + // Proof-shape tamper: dropping the ring-switch message must return + // false (not panic). + let mut bad_proof = inst.proof.clone(); + bad_proof.ring_switches[0].s_hat_v.pop(); + assert!( + !verify_instance(&inst, &inst.point_claims, &inst.ring.claims, &bad_proof), + "short s_hat_v accepted" + ); + } + + #[test] + fn stacked_open_proof_is_deterministic() { + let a = build_instance(2); + let b = build_instance(2); + assert_eq!(a.proof, b.proof, "same inputs must yield identical proofs"); + let bytes_a = bincode::serialize(&a.proof).unwrap(); + let bytes_b = bincode::serialize(&b.proof).unwrap(); + assert_eq!(bytes_a, bytes_b, "proof bytes must be deterministic"); + } + + /// Residual cube crossing INTO the q_pkd slice (case split = n_ris in the + /// verifier closure): q_pkd occupies half a 2^14 stack (qpkd_vars = 13), + /// and the fallback config's residual cube (yr_log_n = 3) is wider than + /// the single selector coordinate, so some q_pkd coords are covered by + /// binary y bits and the tensor finish runs with a nonempty suffix. + #[test] + fn stacked_open_residual_crosses_qpkd() { + let log_n = 14usize; + let qpkd_vars = 13usize; + let qpkd_offset = 1usize << 13; + let mut s = 3u64; + + let mut stack: Vec = (0..1usize << 13).map(|_| F64(splitmix64(&mut s))).collect(); + let bits: Vec = (0..1usize << (qpkd_vars + LOG_PACKING)) + .map(|_| splitmix64(&mut s) & 1 == 1) + .collect(); + stack.extend(pack_witness(&bits, qpkd_vars + LOG_PACKING)); + assert_eq!(stack.len(), 1 << log_n); + + // One point claim on the low column. + let low_point: Vec = (0..12).map(|_| rand_ext(&mut s)).collect(); + let eq = build_eq_table_ext(&low_point); + let value = inner_product_base_ext(&stack[..1 << 12], &eq); + let point_claims = vec![StackClaim::Point { + offset: 0, + low_point, + value, + }]; + + // One ring-switched claim on the wide q_pkd. + let qpkd = &stack[qpkd_offset..]; + let r_prefix: Vec = (0..LOG_PACKING).map(|_| rand_ext(&mut s)).collect(); + let prefix_weights = eq_prefix_weights(&r_prefix); + let suffix_point: Vec = (0..qpkd_vars).map(|_| rand_ext(&mut s)).collect(); + let s_hat_v = fold_1b_rows(qpkd, &build_eq_table_ext(&suffix_point)); + let rs_value = claim_check(&prefix_weights, &s_hat_v); + let claims = vec![RingSwitchClaim { + prefix_weights, + suffix_point, + value: rs_value, + // Exercise the precomputed path (transcript must be identical). + s_hat_v: Some(s_hat_v.clone()), + }]; + + // Fixed fallback config so the residual cube size is known: the + // crossing regime needs qpkd_vars > log_n - yr_log_n. + let pc = default_config(log_n, 5, 1).unwrap(); + let vc = default_verifier_config(log_n, 5, 1).unwrap(); + let yr_log_n = log_n - pc.initial_k - pc.level_ks.iter().sum::(); + assert!( + qpkd_vars > log_n - yr_log_n, + "test shape must exercise the crossing regime (yr_log_n = {yr_log_n})" + ); + + let (cm, pd) = commit(&stack, pc.initial_k, pc.log_inv_rates[0]); + let ring = RingSwitchOpen { + offset: qpkd_offset, + qpkd_vars, + claims, + }; + let mut ch = Sponge::new(DOMAIN, &[]); + let proof = open_batch_mixed_ligerito_stacked(&mut ch, &stack, &pd, &pc, &point_claims, &ring); + + let ring_v = RingSwitchVerify { + offset: qpkd_offset, + qpkd_vars, + claims: ring.claims.clone(), + }; + let mut ch = Sponge::new(DOMAIN, &[]); + assert!( + verify_opening_batch_mixed_ligerito_stacked(&mut ch, &vc, log_n, &cm.root, &point_claims, &ring_v, &proof,) + .is_some(), + "honest crossing-regime opening rejected" + ); + + // And the crossing-regime ring claim is still bound: flip its value. + let mut bad_ring = ring_v; + bad_ring.claims[0].value += F192::ONE; + let mut ch = Sponge::new(DOMAIN, &[]); + assert!( + verify_opening_batch_mixed_ligerito_stacked( + &mut ch, + &vc, + log_n, + &cm.root, + &point_claims, + &bad_ring, + &proof, + ) + .is_none(), + "tampered crossing-regime ring value accepted" + ); + } +} diff --git a/crates/pcs/src/tensor_algebra.rs b/crates/pcs/src/tensor_algebra.rs new file mode 100644 index 00000000..594d24cf --- /dev/null +++ b/crates/pcs/src/tensor_algebra.rs @@ -0,0 +1,293 @@ +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/binius-zk/binius64, Apache-2.0. +// Copyright 2025 The Binius Developers +// Copyright 2025 Irreducible, Inc. +// Modifications copyright 2026 Succinct Labs, Benedikt Bunz, William Wang +// SPDX-License-Identifier: Apache-2.0 OR MIT +// +// Adapted from [`super::tensor_algebra`] (itself a port of binius64's +// `crates/math/src/tensor_algebra.rs`) for the 64-bit transition: +// K = F_{2^64} packing, E = GF(2^192) tower opening field. + +//! Tensor-algebra helpers for the rectangular (f = 64, e = 192) ring switch. +//! +//! Two pieces live here: +//! +//! 1. **The rectangular transpose** [`transpose_s_hat`]: an element of +//! `K (x)_F2 E` is a 64x192 F_2 matrix. The prover's message `s_hat_v` is +//! its row view: 64 E-elements, `s_hat_v[i]` = the MLE of the i-th +//! bit-slice of the witness at the suffix point. The transpose re-packs +//! the columns: 192 K-elements `s_hat_u`, where +//! ```text +//! bit i of s_hat_u[w] == bit w of s_hat_v[i] +//! ``` +//! `s_hat_u[w]` is `t_w` in the ring-switching-generalized note: the +//! packed-polynomial claim attached to the w-th E-basis coordinate, +//! `t_w = sum_y A(y, w) * P_packed(y)` with `A(y, w)` = the w-th F_2 +//! coordinate of `eq(r_suffix, y)`. +//! +//! 2. **The square E-tensor algebra** [`TensorAlgebraE`] over +//! `E (x)_F2 E`: the verifier's polylog `eval_rs_eq` accumulates +//! `sum_y eq(query, y) (x) eq(r_suffix, y)`, which is E-valued on BOTH +//! factors (the K packing never appears there because `rs_eq_ind` is +//! E-valued). The F_2 coordinates of an E element are the bits of its +//! `(c0, c1, c2)` representation. +//! +//! "Bit w" of an E element means: bit w of `c0` for `w < 64`, bit `w - 64` +//! of `c1` for `w < 128`, and bit `w - 128` of `c2` otherwise. + +use core::ops::{Add, AddAssign}; +use primitives::field::{F64, F192}; + +/// The degree of K = F_{2^64} over F_2 (the packing degree f). +pub const DEGREE: usize = 64; + +/// The degree of E = GF(2^192) over F_2 (the opening degree e). +pub const DEGREE_E: usize = 192; + +/// Bit w of an E element in the tower basis (w in 0..192). +#[inline(always)] +fn ext_bit(e: F192, w: usize) -> u64 { + if w < 64 { + (e.c0 >> w) & 1 + } else if w < 128 { + (e.c1 >> (w - 64)) & 1 + } else { + (e.c2 >> (w - 128)) & 1 + } +} + +/// Rectangular tensor-algebra transpose: `s_hat_v` (64 E-elements, the row +/// view of a `K (x)_F2 E` element) to `s_hat_u` (192 K-elements, the column +/// view). +/// +/// ```text +/// bit i of s_hat_u[w] == bit w of s_hat_v[i], i in 0..64, w in 0..192 +/// ``` +/// +/// `s_hat_u[w] = t_w` in the ring-switching-generalized note. Mirror of +/// the legacy tensor-algebra transpose, generalized to the 64x192 shape. +/// Naive O(64 * 192) bit-scan; the input is a fixed 1.5 KiB, so this is never +/// on a hot path. +pub fn transpose_s_hat(s_hat_v: &[F192]) -> Vec { + assert_eq!( + s_hat_v.len(), + DEGREE, + "transpose_s_hat: s_hat_v must have one entry per packing bit (64)" + ); + let mut s_hat_u = vec![F64::ZERO; DEGREE_E]; + for (i, elem) in s_hat_v.iter().enumerate() { + // Deposit bit w of elem into bit i of s_hat_u[w]; scan set bits only. + let mut c0 = elem.c0; + while c0 != 0 { + let w = c0.trailing_zeros() as usize; + s_hat_u[w].0 |= 1u64 << i; + c0 &= c0 - 1; + } + let mut c1 = elem.c1; + while c1 != 0 { + let w = c1.trailing_zeros() as usize; + s_hat_u[64 | w].0 |= 1u64 << i; + c1 &= c1 - 1; + } + let mut c2 = elem.c2; + while c2 != 0 { + let w = c2.trailing_zeros() as usize; + s_hat_u[128 | w].0 |= 1u64 << i; + c2 &= c2 - 1; + } + } + s_hat_u +} + +/// An element of `E (x)_F2 E` (E = the tower GF(2^192)), stored as 192 +/// `F192` elements: `elems[i]` is the second-factor component attached to +/// the i-th F_2-basis element of the first factor, i.e. +/// `bit_j(elems[i])` = the coefficient of `b_i (x) b_j`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TensorAlgebraE { + /// Length-192 vector; see the struct docs for the indexing convention. + pub elems: Vec, +} + +impl TensorAlgebraE { + /// All-zero element. + pub fn zero() -> Self { + Self { + elems: vec![F192::ZERO; DEGREE_E], + } + } + + /// Multiplicative identity: `1 (x) 1`. + pub fn one() -> Self { + let mut elems = vec![F192::ZERO; DEGREE_E]; + elems[0] = F192::ONE; + Self { elems } + } + + /// Embed `x` into the vertical subring: returns `1 (x) x`. + pub fn from_vertical(x: F192) -> Self { + let mut elems = vec![F192::ZERO; DEGREE_E]; + elems[0] = x; + Self { elems } + } + + /// Multiply by an element of the vertical subring (`1 (x) scalar`): each + /// `elems[i]` is scaled by `scalar` in E. + pub fn scale_vertical(mut self, scalar: F192) -> Self { + for e in self.elems.iter_mut() { + *e *= scalar; + } + self + } + + /// Multiply by an element of the horizontal subring (`scalar (x) 1`). + /// Implemented as `transpose . scale_vertical . transpose`. + pub fn scale_horizontal(self, scalar: F192) -> Self { + self.transpose().scale_vertical(scalar).transpose() + } + + /// Transpose: swap the two tensor factors. Concretely, after transpose, + /// `bit_j(elems'[i]) = bit_i(elems[j])` for all `i, j in [0, 192)`. + pub fn transpose(mut self) -> Self { + square_transpose_ext(&mut self.elems); + self + } + + /// Fold to a single E element: transpose, then scale row `w` by + /// `coeffs[w]` and sum. + /// + /// Computes `sum_w coeffs[w] * transpose(self).elems[w]`. With `self = + /// sum_y eq(query, y) (x) eq(z, y)` and `coeffs = eq(r'')` this is the + /// MLE of `rs_eq_ind` at `query` (see `ring_switch::eval_rs_eq`). + pub fn fold_vertical(self, coeffs: &[F192]) -> F192 { + assert_eq!(coeffs.len(), DEGREE_E, "fold_vertical: coeffs.len() must be 192"); + let transposed = self.transpose(); + let mut acc = F192::ZERO; + for (e, c) in transposed.elems.iter().zip(coeffs.iter()) { + acc += *e * *c; + } + acc + } +} + +impl Add<&TensorAlgebraE> for TensorAlgebraE { + type Output = TensorAlgebraE; + fn add(mut self, rhs: &TensorAlgebraE) -> TensorAlgebraE { + self += rhs; + self + } +} + +impl AddAssign<&TensorAlgebraE> for TensorAlgebraE { + fn add_assign(&mut self, rhs: &TensorAlgebraE) { + for (a, b) in self.elems.iter_mut().zip(rhs.elems.iter()) { + *a = *a + *b; + } + } +} + +/// In-place 192x192 F_2 matrix transpose of the F192 coefficient table. +/// +/// On input: `elems[i]` viewed as a 192-bit row; bit `j` (tower basis) is the +/// F_2 coefficient at position `(i, j)`. On output: bit `j` of `elems[i]` +/// becomes the old bit `i` of `elems[j]`. Mirror of +/// `tensor_algebra::square_transpose`, extended to `(c0, c1, c2)`. +fn square_transpose_ext(elems: &mut [F192]) { + assert_eq!(elems.len(), DEGREE_E, "square_transpose_ext: input must be length 192"); + + let mut out = [F192::ZERO; DEGREE_E]; + for (j, o) in out.iter_mut().enumerate() { + let mut c0: u64 = 0; + let mut c1: u64 = 0; + let mut c2: u64 = 0; + for i in 0..64 { + c0 |= ext_bit(elems[i], j) << i; + } + for i in 64..128 { + c1 |= ext_bit(elems[i], j) << (i - 64); + } + for i in 128..192 { + c2 |= ext_bit(elems[i], j) << (i - 128); + } + *o = F192::new(c0, c1, c2); + } + elems.copy_from_slice(&out); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn rand_ext(s: &mut u64) -> F192 { + F192::new(splitmix64(s), splitmix64(s), splitmix64(s)) + } + + #[test] + fn rect_transpose_bit_relation() { + let mut s = 1u64; + let s_hat_v: Vec = (0..DEGREE).map(|_| rand_ext(&mut s)).collect(); + let s_hat_u = transpose_s_hat(&s_hat_v); + assert_eq!(s_hat_u.len(), DEGREE_E); + for i in 0..DEGREE { + for w in 0..DEGREE_E { + assert_eq!( + (s_hat_u[w].0 >> i) & 1, + ext_bit(s_hat_v[i], w), + "bit ({i}, {w}) not transposed" + ); + } + } + } + + #[test] + fn square_transpose_is_involution() { + let mut s = 2u64; + let orig: Vec = (0..DEGREE_E).map(|_| rand_ext(&mut s)).collect(); + let t = TensorAlgebraE { elems: orig.clone() }; + let tt = t.clone().transpose(); + // Bit relation on a spot-check diagonal band plus full involution. + for i in 0..DEGREE_E { + for w in [0usize, 1, 63, 64, 65, 127] { + assert_eq!(ext_bit(tt.elems[i], w), ext_bit(orig[w], i)); + } + } + assert_eq!(tt.transpose().elems, orig, "transpose twice must be id"); + } + + /// `fold_vertical(from_vertical(x), coeffs)` is exactly the F_2-linear + /// map Phi sending the w-th E-basis bit to coeffs[w], applied to x. This + /// is the map the ring switch uses to define `rs_eq_ind`. + #[test] + fn fold_vertical_is_phi() { + let mut s = 3u64; + let coeffs: Vec = (0..DEGREE_E).map(|_| rand_ext(&mut s)).collect(); + let x = rand_ext(&mut s); + let folded = TensorAlgebraE::from_vertical(x).fold_vertical(&coeffs); + let mut expected = F192::ZERO; + for (w, &c) in coeffs.iter().enumerate() { + if ext_bit(x, w) == 1 { + expected += c; + } + } + assert_eq!(folded, expected); + } + + /// `scale_horizontal(1 (x) 1, s) == s (x) 1 == transpose(1 (x) s)`. + #[test] + fn scale_horizontal_semantics() { + let mut s = 4u64; + let x = rand_ext(&mut s); + let lhs = TensorAlgebraE::one().scale_horizontal(x); + let rhs = TensorAlgebraE::from_vertical(x).transpose(); + assert_eq!(lhs, rhs); + } +} diff --git a/crates/pcs/src/test_rng.rs b/crates/pcs/src/test_rng.rs index aad0672f..500aa327 100644 --- a/crates/pcs/src/test_rng.rs +++ b/crates/pcs/src/test_rng.rs @@ -1,5 +1,3 @@ -use primitives::field::F128; - /// Deterministic SplitMix64 generator shared by the crate's unit tests. pub(crate) struct Rng(u64); @@ -15,12 +13,4 @@ impl Rng { z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); z ^ (z >> 31) } - - pub(crate) fn bits(&mut self, n: usize) -> Vec { - (0..n).map(|_| self.next_u64() & 1 == 1).collect() - } - - pub(crate) fn f128(&mut self) -> F128 { - F128::new(self.next_u64(), self.next_u64()) - } } diff --git a/crates/pcs/tests/pcs_throughput.rs b/crates/pcs/tests/pcs_throughput.rs new file mode 100644 index 00000000..1af39330 --- /dev/null +++ b/crates/pcs/tests/pcs_throughput.rs @@ -0,0 +1,150 @@ +//! Dedicated PCS throughput benchmark (manual; `#[ignore]`d so it never runs +//! in a normal `cargo test`). +//! +//! Commits and opens a random witness of `2^PCS_LOG_N` GF(2^64) elements at +//! inverse-rate `1/2^PCS_LOG_INV_RATE`, times each phase, and reports GiB/s +//! over the committed data. Env knobs (all optional): +//! +//! PCS_LOG_N number of variables = log2(witness length) [default 22] +//! PCS_LOG_INV_RATE log2 of the inverse RS rate (rate = 1/2^r) [default: profile] +//! PCS_SAMPLES timed repetitions; the median is reported [default 5] +//! +//! Run: +//! PCS_LOG_N=24 PCS_LOG_INV_RATE=1 cargo test --release -p pcs --test pcs_throughput -- --ignored --nocapture +//! +//! Hierarchical tracing is enabled automatically (`RUST_LOG` adjusts its +//! verbosity). Set `LIGERITO_TRACE=1` as well for the legacy textual per-phase +//! breakdown. Large `PCS_LOG_N` needs substantial memory (the RS codeword is +//! `2^log_inv_rate`× the witness, and the open clones the basis table each +//! sample). + +use std::hint::black_box; +use std::time::Instant; + +use fiat_shamir::Sponge; +use pcs::ligerito::{ + build_eq_table_ext, commit, configs_for, configs_for_rate, inner_product_base_ext, recursive_prover_with_basis, +}; +use primitives::{ + field::{F64, F192}, + pretty_integer, +}; + +fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) +} + +fn env_usize(key: &str) -> Option { + std::env::var(key).ok().map(|s| { + s.parse() + .unwrap_or_else(|_| panic!("{key} must be a non-negative integer")) + }) +} + +fn median(mut xs: Vec) -> f64 { + xs.sort_by(|a, b| a.partial_cmp(b).unwrap()); + xs[xs.len() / 2] +} + +#[test] +#[ignore = "manual release benchmark; drive with PCS_LOG_N / PCS_LOG_INV_RATE"] +fn pcs_throughput() { + primitives::init_tracing(); + + let log_n = env_usize("PCS_LOG_N").unwrap_or(22); + let samples = env_usize("PCS_SAMPLES").unwrap_or(5).max(1); + + // Honour PCS_LOG_INV_RATE if set, else the production profile's L0 rate. + let (pc, _vc) = match env_usize("PCS_LOG_INV_RATE") { + Some(r) => configs_for_rate(log_n, r), + None => configs_for(log_n), + } + .expect("Ligerito config feasible (try a larger PCS_LOG_N, e.g. >= 16)"); + let log_inv_rate = pc.log_inv_rates[0]; + let trace_span = tracing::info_span!("PCS throughput", log_n, log_inv_rate, samples).entered(); + + // Random F64 witness (the committed polynomial) and a random E evaluation point. + let mut s = 0x0192_0000u64 ^ log_n as u64; + let n = 1usize << log_n; + let witness: Vec = (0..n).map(|_| F64(splitmix64(&mut s))).collect(); + let point: Vec = (0..log_n) + .map(|_| F192::new(splitmix64(&mut s), splitmix64(&mut s), splitmix64(&mut s))) + .collect(); + let b_initial = build_eq_table_ext(&point); + let target = inner_product_base_ext(&witness, &b_initial); + + let mut commit_times = Vec::with_capacity(samples); + let mut open_times = Vec::with_capacity(samples); + for sample in 0..samples { + tracing::info_span!("Sample", sample).in_scope(|| { + let ((cm, pd), elapsed) = tracing::info_span!("Commit").in_scope(|| { + let t = Instant::now(); + let committed = commit(&witness, pc.initial_k, log_inv_rate); + (committed, t.elapsed().as_secs_f64()) + }); + commit_times.push(elapsed); + + let mut ch = Sponge::new(b"pcs-throughput", &[]); + let (proof, elapsed) = tracing::info_span!("PCS open").in_scope(|| { + let t = Instant::now(); + let proof = recursive_prover_with_basis( + &pc, + &witness, + b_initial.clone(), + target, + &pd.codeword, + &pd.merkle_tree, + &mut ch, + ); + (proof, t.elapsed().as_secs_f64()) + }); + open_times.push(elapsed); + black_box((cm, proof)); + }); + } + + let commit_s = median(commit_times); + let open_s = median(open_times); + + // Throughput is over the committed data: 2^log_n F64 = 2^log_n * 8 bytes. + let data_bytes = (n as f64) * 8.0; + let mib = |bytes: f64| bytes / (1u64 << 20) as f64; + let gibps = |secs: f64| (data_bytes / (1u64 << 30) as f64) / secs; + let codeword_bytes = data_bytes * (1u64 << log_inv_rate) as f64; + + // tracing-forest renders the tree when its root span closes. Close it + // before printing the throughput report so the complete trace appears first. + drop(trace_span); + + println!( + "\nPCS throughput — 2^{log_n} variables, rate 1/2^{log_inv_rate}, median of {}", + pretty_integer(samples) + ); + println!( + " committed data : {:>8.1} MiB ({:>13} F64)", + mib(data_bytes), + pretty_integer(n) + ); + println!(" RS codeword (encoded) : {:>8.1} MiB", mib(codeword_bytes)); + println!(" ------------------------------------------------------------"); + println!( + " commit : {:>8.1} ms ({:>6.2} GiB/s)", + commit_s * 1e3, + gibps(commit_s) + ); + println!( + " open : {:>8.1} ms ({:>6.2} GiB/s)", + open_s * 1e3, + gibps(open_s) + ); + println!(" ------------------------------------------------------------"); + println!( + " commit + open : {:>8.1} ms ({:>6.2} GiB/s)", + (commit_s + open_s) * 1e3, + gibps(commit_s + open_s), + ); +} diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index f425b0a9..84c93f6a 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -11,3 +11,6 @@ rayon.workspace = true serde.workspace = true tracing-forest.workspace = true tracing-subscriber.workspace = true + +[dev-dependencies] +bincode.workspace = true diff --git a/crates/primitives/src/bits.rs b/crates/primitives/src/bits.rs index 2ca8f044..9a7e8d03 100644 --- a/crates/primitives/src/bits.rs +++ b/crates/primitives/src/bits.rs @@ -1,4 +1,4 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. //! Small bit-manipulation primitives shared across modules. /// Hacker's Delight (Sec. 7-3) 8×8 bit-matrix transpose stored in a `u64`. @@ -69,6 +69,7 @@ fn bit_transpose_64bytes_scalar(input: &[u8; 64], output: &mut [u8; 64]) { /// `output[b*8..]`. Bit `(x, t)` of the gathered word lands at `(t, x)` — /// exactly `output[b*8 + t] bit x = input[x*8 + b] bit t`. #[cfg(not(target_arch = "aarch64"))] +#[cfg_attr(all(target_arch = "x86_64", target_feature = "avx512vbmi"), allow(dead_code))] #[inline] fn bit_transpose_64bytes_u64(input: &[u8; 64], output: &mut [u8; 64]) { for b_chunk in 0..8 { @@ -92,6 +93,43 @@ fn bit_transpose_64bytes_u64(input: &[u8; 64], output: &mut [u8; 64]) { } } +/// AVX-512VBMI 64-byte bit-transpose. `vpermb` first gathers the eight +/// strided byte-columns into eight `u64` lanes; three lane-wise masked-swap +/// rounds then transpose each gathered 8x8 bit matrix in parallel. +#[cfg(all(target_arch = "x86_64", target_feature = "avx512vbmi"))] +#[target_feature(enable = "avx512vbmi", enable = "avx512f")] +unsafe fn bit_transpose_64bytes_avx512vbmi(input: &[u8; 64], output: &mut [u8; 64]) { + use core::arch::x86_64::*; + + // Each consecutive eight indices select byte-column b from input rows + // x=0..7, producing one gathered u64 lane per column. + const IDX: [u8; 64] = [ + 0, 8, 16, 24, 32, 40, 48, 56, 1, 9, 17, 25, 33, 41, 49, 57, 2, 10, 18, 26, 34, 42, 50, 58, 3, 11, 19, 27, 35, + 43, 51, 59, 4, 12, 20, 28, 36, 44, 52, 60, 5, 13, 21, 29, 37, 45, 53, 61, 6, 14, 22, 30, 38, 46, 54, 62, 7, 15, + 23, 31, 39, 47, 55, 63, + ]; + + unsafe { + let bytes = _mm512_loadu_si512(input.as_ptr().cast()); + let indices = _mm512_loadu_si512(IDX.as_ptr().cast()); + let mut y = _mm512_permutexvar_epi8(indices, bytes); + + let mask1 = _mm512_set1_epi64(0x00AA00AA00AA00AA); + let t = _mm512_and_si512(_mm512_xor_si512(y, _mm512_srli_epi64::<7>(y)), mask1); + y = _mm512_xor_si512(y, _mm512_xor_si512(t, _mm512_slli_epi64::<7>(t))); + + let mask2 = _mm512_set1_epi64(0x0000CCCC0000CCCC); + let t = _mm512_and_si512(_mm512_xor_si512(y, _mm512_srli_epi64::<14>(y)), mask2); + y = _mm512_xor_si512(y, _mm512_xor_si512(t, _mm512_slli_epi64::<14>(t))); + + let mask3 = _mm512_set1_epi64(0x00000000F0F0F0F0); + let t = _mm512_and_si512(_mm512_xor_si512(y, _mm512_srli_epi64::<28>(y)), mask3); + y = _mm512_xor_si512(y, _mm512_xor_si512(t, _mm512_slli_epi64::<28>(t))); + + _mm512_storeu_si512(output.as_mut_ptr().cast(), y); + } +} + /// NEON 64-byte bit-transpose. Two-stage: /// 1. `vqtbl4q_u8` reorders the 64 input bytes so each 8-byte group within /// the output is one byte-chunk's worth of `x_small=0..8` bytes. @@ -171,7 +209,12 @@ pub fn bit_transpose_64bytes(input: &[u8; 64], output: &mut [u8; 64]) { unsafe { bit_transpose_64bytes_neon(input, output) } - #[cfg(not(target_arch = "aarch64"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx512vbmi"))] + // SAFETY: AVX-512VBMI is statically enabled at compile time. + unsafe { + bit_transpose_64bytes_avx512vbmi(input, output) + } + #[cfg(not(any(target_arch = "aarch64", all(target_arch = "x86_64", target_feature = "avx512vbmi"))))] bit_transpose_64bytes_u64(input, output); } @@ -179,7 +222,7 @@ pub fn bit_transpose_64bytes(input: &[u8; 64], output: &mut [u8; 64]) { mod tests { use super::*; - /// splitmix64 test PRNG (same helper as the gf2_128/gf2_8 test modules). + /// splitmix64 test PRNG (same helper as the former_field_module/gf2_8 test modules). #[cfg(target_arch = "aarch64")] struct Rng(u64); #[cfg(target_arch = "aarch64")] @@ -293,6 +336,26 @@ mod tests { } } + #[cfg(all(target_arch = "x86_64", target_feature = "avx512vbmi"))] + #[test] + fn avx512vbmi_bit_transpose_matches_scalar() { + let mut seed = 0xB17_BB17u64; + let mut next = || { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (seed >> 33) as u8 + }; + for _ in 0..64 { + let mut input = [0u8; 64]; + input.iter_mut().for_each(|b| *b = next()); + let mut out_scalar = [0u8; 64]; + let mut out_avx512vbmi = [0u8; 64]; + bit_transpose_64bytes_scalar(&input, &mut out_scalar); + // SAFETY: this test is compiled only when AVX-512VBMI is enabled. + unsafe { bit_transpose_64bytes_avx512vbmi(&input, &mut out_avx512vbmi) }; + assert_eq!(out_scalar, out_avx512vbmi, "bit_transpose disagreement"); + } + } + #[cfg(target_arch = "aarch64")] #[test] fn neon_bit_transpose_matches_scalar() { diff --git a/crates/primitives/src/epool.rs b/crates/primitives/src/epool.rs index 2751a1a7..49a8413d 100644 --- a/crates/primitives/src/epool.rs +++ b/crates/primitives/src/epool.rs @@ -176,10 +176,7 @@ where // splits down to single indices so every main worker can pick one up; // under nesting fewer run, which the queue tolerates by construction. let drain_main = || { - (0..main_threads) - .into_par_iter() - .with_max_len(1) - .for_each(|_| worker()); + (0..main_threads).into_par_iter().with_max_len(1).for_each(|_| worker()); }; std::thread::scope(|s| { // The scoped thread parks inside `broadcast` while the E-workers @@ -218,10 +215,7 @@ mod tests { /// Every chunk runs exactly once, whichever pool claims it. #[test] fn helper_queue_runs_each_chunk_exactly_once() { - let helper = rayon::ThreadPoolBuilder::new() - .num_threads(2) - .build() - .unwrap(); + let helper = rayon::ThreadPoolBuilder::new().num_threads(2).build().unwrap(); for n in [0usize, 1, 15, 16, 257] { let counts: Vec = (0..n).map(|_| AtomicU32::new(0)).collect(); run_chunks_with_helper( diff --git a/crates/primitives/src/field.rs b/crates/primitives/src/field.rs deleted file mode 100644 index 058f3310..00000000 --- a/crates/primitives/src/field.rs +++ /dev/null @@ -1,78 +0,0 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. -//! Binary field arithmetic. -//! -//! - [`F8`] — GF(2^8) with AES polynomial x^8 + x^4 + x^3 + x + 1 -//! - [`F128`] — GF(2^128) in GHASH form, polynomial x^128 + x^7 + x^2 + x + 1 -//! - [`F256Unreduced`] — 256-bit unreduced GHASH products, for deferred reduction - -pub mod gf2_128; -pub mod gf2_8; -#[cfg(target_arch = "aarch64")] -pub mod neon; -pub mod phi8; - -pub use gf2_8::F8; -pub use gf2_128::{F128, F256Unreduced, mul_by_x}; -pub use phi8::{PHI_8_TABLE, phi8}; - -// --------------------------------------------------------------------------- -// leanVM g-power helpers: domain separators / opcodes as x^k, and the g-power -// index encoding. -// --------------------------------------------------------------------------- - -use rayon::prelude::*; - -/// `[g^0, g^1, …, g^{n-1}]`, built in parallel: each chunk seeds with one g-power -/// (`x_pow`, `O(log)`) and fills by `mul_by_x`, breaking the serial prefix chain -/// across cores. -pub fn g_powers(n: usize) -> Vec { - const CHUNK: usize = 1 << 12; - let mut v = vec![F128::ZERO; n]; - v.par_chunks_mut(CHUNK).enumerate().for_each(|(ci, chunk)| { - let mut acc = x_pow(ci * CHUNK); - for slot in chunk.iter_mut() { - *slot = acc; - acc = mul_by_x(acc); - } - }); - v -} - -/// `x^k` in the GHASH monomial basis by square-and-multiply (`O(log k)`). Used for -/// domain separators, opcodes, and the g-power index encoding. -pub fn x_pow(k: usize) -> F128 { - let mut result = F128::ONE; - let mut base = F128::generator(); // x = g - let mut e = k; - while e > 0 { - if e & 1 == 1 { - result *= base; - } - base = base * base; - e >>= 1; - } - result -} - -/// The fixed generator `g = x`, of multiplicative order `2^128 − 1` (`x` is -/// primitive) — larger than every index used. For `k < 128`, `g^k` is the monomial -/// `x^k` (bit `k`), which the XMSS encoding check relies on. -pub const G: F128 = F128::generator(); - -/// `g^i`, the g-power encoding of index `i` (§1). -#[inline] -pub fn g_pow(i: usize) -> F128 { - x_pow(i) -} - -/// MLE of the index column `[g^0, …, g^{2^n−1}]` over the `n`-variable cube: -/// `∏_k (1 + ζ_k·(1 + g^{2^k}))`, evaluated in `O(n)` (§5.3). -pub fn index_mle(zeta: &[F128]) -> F128 { - let mut acc = F128::ONE; - let mut g2k = G; // g^{2^0} = g - for &z in zeta { - acc *= F128::ONE + z * (F128::ONE + g2k); - g2k = g2k * g2k; - } - acc -} diff --git a/crates/primitives/src/field/gf2_128.rs b/crates/primitives/src/field/gf2_128.rs deleted file mode 100644 index e1b0d869..00000000 --- a/crates/primitives/src/field/gf2_128.rs +++ /dev/null @@ -1,1160 +0,0 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. -// Copyright 2025 The Binius Developers -// Copyright 2025 Irreducible, Inc. -// Modifications copyright 2026 Succinct Labs, Benedikt Bunz, William Wang -// SPDX-License-Identifier: Apache-2.0 OR MIT -// -// The default `Mul` implementation (`ghash_mul_binius`) is a port of -// `mul_clmul` from binius64 -// (https://github.com/binius-zk/binius64, `crates/field/src/arch/shared/ghash.rs`). - -//! GF(2^128) in GHASH form: irreducible polynomial x^128 + x^7 + x^2 + x + 1. -//! -//! Layout: `lo` holds coefficients x^0..x^63, `hi` holds x^64..x^127. -//! Hardware: `vmull_p64` (ARM PMULL, AES extension) does a 64×64 carry-less mul -//! in one instruction. Default `Mul` impl uses the binius64 reduction variant -//! (4 PMULL schoolbook + 2-stage recursive reduction, 2 extra PMULL), which -//! benchmarked as the fastest of four variants tried. - -use core::ops::{Add, AddAssign, BitXor, BitXorAssign, Mul, MulAssign}; - -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[repr(C, align(16))] -pub struct F128 { - pub lo: u64, - pub hi: u64, -} - -impl F128 { - pub const ZERO: Self = Self { lo: 0, hi: 0 }; - pub const ONE: Self = Self { lo: 1, hi: 0 }; - - #[inline] - pub const fn new(lo: u64, hi: u64) -> Self { - Self { lo, hi } - } - - /// Decode the polynomial-basis bits from their canonical little-endian bytes. - #[inline] - pub const fn from_le_bytes(bytes: [u8; 16]) -> Self { - let value = u128::from_le_bytes(bytes); - Self::new(value as u64, (value >> 64) as u64) - } - - /// Encode the polynomial-basis bits as canonical little-endian bytes. - #[inline] - pub const fn to_le_bytes(self) -> [u8; 16] { - (self.lo as u128 | (self.hi as u128) << 64).to_le_bytes() - } - - /// The generator γ (i.e. the element `x`). `mul_by_x` is a fast shift+fold. - #[inline] - pub const fn generator() -> Self { - Self { lo: 2, hi: 0 } - } - - #[inline] - pub const fn is_zero(self) -> bool { - self.lo == 0 && self.hi == 0 - } - - /// 256-bit unreduced product `(self · rhs)`. Caller XORs many of these into - /// an `F256Unreduced` accumulator and calls `.reduce()` once at the end. - /// Reduction commutes with XOR, so Σ (aᵢ·bᵢ) mod p = (Σ aᵢ·bᵢ) mod p. - #[inline] - pub fn mul_unreduced(self, rhs: Self) -> F256Unreduced { - ghash_mul_unreduced(self, rhs) - } - - /// Multiplicative inverse via Fermat: x^{2^128 − 2}. - /// Used in one-time setup (Lagrange weight computation), not in hot paths. - pub fn inv(self) -> Self { - // x^{2^128 - 2} = ∏_{i=1..127} x^{2^i} - let mut r = Self::ONE; - let mut cur = self * self; // x^2 - for _ in 1..128 { - r *= cur; - cur = cur * cur; - } - r - } -} - -impl Add for F128 { - type Output = Self; - #[inline] - fn add(self, rhs: Self) -> Self { - Self { - lo: self.lo ^ rhs.lo, - hi: self.hi ^ rhs.hi, - } - } -} - -impl AddAssign for F128 { - #[inline] - fn add_assign(&mut self, rhs: Self) { - self.lo ^= rhs.lo; - self.hi ^= rhs.hi; - } -} - -impl Mul for F128 { - type Output = Self; - #[inline] - fn mul(self, rhs: Self) -> Self { - #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] - { - // SAFETY: aes target feature is enabled at compile time. - unsafe { aarch64::ghash_mul_binius(self, rhs) } - } - #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] - { - // SAFETY: pclmulqdq target feature is enabled at compile time. - unsafe { x86_64::ghash_mul_clmul(self, rhs) } - } - #[cfg(not(any( - all(target_arch = "aarch64", target_feature = "aes"), - all(target_arch = "x86_64", target_feature = "pclmulqdq") - )))] - { - software::ghash_mul(self, rhs) - } - } -} - -impl MulAssign for F128 { - #[inline] - fn mul_assign(&mut self, rhs: Self) { - *self = *self * rhs; - } -} - -/// Multiply by x (the generator). One shift + conditional XOR with 0x87, no PMULL. -/// Used by the sumcheck round when the fixed evaluation point is the generator. -#[inline] -pub const fn mul_by_x(z: F128) -> F128 { - let carry = z.hi >> 63; - let mask = 0u64.wrapping_sub(carry); // 0 or all-ones - F128 { - lo: (z.lo << 1) ^ (0x87 & mask), - hi: (z.hi << 1) | (z.lo >> 63), - } -} - -// --------------------------------------------------------------------------- -// Deferred reduction: 256-bit unreduced products that can be XOR-accumulated. -// --------------------------------------------------------------------------- - -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct F256Unreduced { - pub r0: u64, - pub r1: u64, - pub r2: u64, - pub r3: u64, -} - -impl F256Unreduced { - pub const ZERO: Self = Self { - r0: 0, - r1: 0, - r2: 0, - r3: 0, - }; - - #[inline] - pub fn reduce(self) -> F128 { - ghash_reduce(self.r0, self.r1, self.r2, self.r3) - } -} - -impl BitXor for F256Unreduced { - type Output = Self; - #[inline] - fn bitxor(self, rhs: Self) -> Self { - Self { - r0: self.r0 ^ rhs.r0, - r1: self.r1 ^ rhs.r1, - r2: self.r2 ^ rhs.r2, - r3: self.r3 ^ rhs.r3, - } - } -} - -impl BitXorAssign for F256Unreduced { - #[inline] - fn bitxor_assign(&mut self, rhs: Self) { - self.r0 ^= rhs.r0; - self.r1 ^= rhs.r1; - self.r2 ^= rhs.r2; - self.r3 ^= rhs.r3; - } -} - -// --------------------------------------------------------------------------- -// Reduction mod p = x^128 + x^7 + x^2 + x + 1. Works on any target. -// --------------------------------------------------------------------------- - -/// Fold the upper 128 bits (r2:r3) into the lower 128 bits (r0:r1) mod p. -/// x^128 ≡ x^7 + x^2 + x + 1, so U·x^128 ≡ U ^ (U<<1) ^ (U<<2) ^ (U<<7). -#[inline] -pub fn ghash_reduce(r0: u64, r1: u64, r2: u64, r3: u64) -> F128 { - let s1_lo = r2 << 1; - let s1_hi = (r3 << 1) | (r2 >> 63); - let s2_lo = r2 << 2; - let s2_hi = (r3 << 2) | (r2 >> 62); - let s7_lo = r2 << 7; - let s7_hi = (r3 << 7) | (r2 >> 57); - - let t_lo = r2 ^ s1_lo ^ s2_lo ^ s7_lo; - let t_hi = r3 ^ s1_hi ^ s2_hi ^ s7_hi; - - // Bits of r3 that shifted past position 127 (top 7 bits, in 3 shifts). - let ov = (r3 >> 63) ^ (r3 >> 62) ^ (r3 >> 57); - let corr = ov ^ (ov << 1) ^ (ov << 2) ^ (ov << 7); - - F128 { - lo: r0 ^ t_lo ^ corr, - hi: r1 ^ t_hi, - } -} - -// --------------------------------------------------------------------------- -// aarch64 + AES: PMULL-based multiplication variants. -// --------------------------------------------------------------------------- - -#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] -pub mod aarch64 { - use super::{F128, F256Unreduced, ghash_reduce}; - use core::arch::aarch64::*; - use core::mem::transmute; - - /// 64×64 carry-less product, returned as a 128-bit vector. - /// - /// # Safety - /// Caller must ensure the `aes` target feature is enabled (statically - /// satisfied here because every caller is itself `#[target_feature(enable = "aes")]`). - #[inline] - #[target_feature(enable = "aes")] - unsafe fn pmull(a: u64, b: u64) -> uint64x2_t { - let prod = vmull_p64(a, b); - // SAFETY: u128 and uint64x2_t are both 128-bit, 16-byte-aligned values; - // transmute is a bit-level reinterpret with no UB. - unsafe { transmute::(prod) } - } - - /// Schoolbook 4 PMULL — fully independent products, then scalar reduction. - /// - /// # Safety - /// Requires the `aes` target feature (compiles to PMULL); only call where - /// `aes` is statically enabled or has been runtime-detected. - #[target_feature(enable = "aes")] - pub unsafe fn ghash_mul_schoolbook(a: F128, b: F128) -> F128 { - // SAFETY: function carries the aes target feature; helper calls below - // require that and nothing else. - unsafe { - let p_ll = pmull(a.lo, b.lo); - let p_lh = pmull(a.lo, b.hi); - let p_hl = pmull(a.hi, b.lo); - let p_hh = pmull(a.hi, b.hi); - - let ll_lo = vgetq_lane_u64::<0>(p_ll); - let ll_hi = vgetq_lane_u64::<1>(p_ll); - let hh_lo = vgetq_lane_u64::<0>(p_hh); - let hh_hi = vgetq_lane_u64::<1>(p_hh); - let cross = veorq_u64(p_lh, p_hl); - let cr_lo = vgetq_lane_u64::<0>(cross); - let cr_hi = vgetq_lane_u64::<1>(cross); - - ghash_reduce(ll_lo, ll_hi ^ cr_lo, hh_lo ^ cr_hi, hh_hi) - } - } - - /// Karatsuba 3 PMULL — middle term depends on XOR of inputs (one stall on - /// CPUs with 2 PMULL units). - /// - /// # Safety - /// Requires the `aes` target feature (compiles to PMULL); only call where - /// `aes` is statically enabled or has been runtime-detected. - #[target_feature(enable = "aes")] - pub unsafe fn ghash_mul_karatsuba(a: F128, b: F128) -> F128 { - // SAFETY: function carries the aes target feature. - unsafe { - let p0 = pmull(a.lo, b.lo); - let p1 = pmull(a.hi, b.hi); - let pm = pmull(a.lo ^ a.hi, b.lo ^ b.hi); - - let p0_lo = vgetq_lane_u64::<0>(p0); - let p0_hi = vgetq_lane_u64::<1>(p0); - let p1_lo = vgetq_lane_u64::<0>(p1); - let p1_hi = vgetq_lane_u64::<1>(p1); - let pm_lo = vgetq_lane_u64::<0>(pm); - let pm_hi = vgetq_lane_u64::<1>(pm); - - let cross_lo = pm_lo ^ p0_lo ^ p1_lo; - let cross_hi = pm_hi ^ p0_hi ^ p1_hi; - - ghash_reduce(p0_lo, p0_hi ^ cross_lo, p1_lo ^ cross_hi, p1_hi) - } - } - - /// Karatsuba 3 PMULL + Barrett 2 PMULL = 5 PMULL total. - /// `r_hi = hi_hi · 0x87` depends only on `d2`, not `d1`, so it can issue - /// in parallel with the cross-term computation. - /// - /// # Safety - /// Requires the `aes` target feature (compiles to PMULL); only call where - /// `aes` is statically enabled or has been runtime-detected. - #[target_feature(enable = "aes")] - pub unsafe fn ghash_mul_karatsuba_barrett(a: F128, b: F128) -> F128 { - // SAFETY: function carries the aes target feature. - unsafe { - let d0 = pmull(a.lo, b.lo); - let d2 = pmull(a.hi, b.hi); - let dm = pmull(a.lo ^ a.hi, b.lo ^ b.hi); - let d1 = veorq_u64(veorq_u64(dm, d0), d2); - - let d0_lo = vgetq_lane_u64::<0>(d0); - let d0_hi = vgetq_lane_u64::<1>(d0); - let d1_lo = vgetq_lane_u64::<0>(d1); - let d1_hi = vgetq_lane_u64::<1>(d1); - let d2_lo = vgetq_lane_u64::<0>(d2); - let d2_hi = vgetq_lane_u64::<1>(d2); - - let lo_lo = d0_lo; - let lo_hi = d0_hi ^ d1_lo; - let hi_lo = d2_lo ^ d1_hi; - let hi_hi = d2_hi; - - let r_hi = pmull(hi_hi, 0x87); - let r_lo = pmull(hi_lo, 0x87); - - let r_lo_lo = vgetq_lane_u64::<0>(r_lo); - let r_lo_hi = vgetq_lane_u64::<1>(r_lo); - let r_hi_lo = vgetq_lane_u64::<0>(r_hi); - let r_hi_hi = vgetq_lane_u64::<1>(r_hi); - - // hi_hi · 0x87 has degree ≤ 70, so r_hi_hi has at most 7 bits. - let ov = r_hi_hi; - let corr = ov ^ (ov << 1) ^ (ov << 2) ^ (ov << 7); - - F128 { - lo: lo_lo ^ r_lo_lo ^ corr, - hi: lo_hi ^ r_lo_hi ^ r_hi_lo, - } - } - } - - /// Binius-style: schoolbook 4 PMULL + recursive 2-stage reduction (2 PMULL). - /// Each stage keeps the intermediate ≤128 bits — no separate 7-bit overflow - /// term required. Total 6 PMULL but fewer scalar shifts in the dep chain. - /// Memory recorded this as the best of the four variants on M-series. - /// - /// # Safety - /// Requires the `aes` target feature (compiles to PMULL); only call where - /// `aes` is statically enabled or has been runtime-detected. - #[target_feature(enable = "aes")] - pub unsafe fn ghash_mul_binius(a: F128, b: F128) -> F128 { - // SAFETY: function carries the aes target feature. - unsafe { - let zero = vdupq_n_u64(0); - - let t0 = pmull(a.lo, b.lo); - let t1a = pmull(a.lo, b.hi); - let t1b = pmull(a.hi, b.lo); - let t2 = pmull(a.hi, b.hi); - let mut t1 = veorq_u64(t1a, t1b); - - // First reduce: t1 = t1 + x^64 · t2 (mod p). - // vextq_u64::<1>(zero, t2) = {0, t2.lo} — places t2.lo into t1.hi. - let t2_shifted = vextq_u64::<1>(zero, t2); - t1 = veorq_u64(t1, t2_shifted); - let t2_hi_s = vgetq_lane_u64::<1>(t2); - let t2_red = pmull(t2_hi_s, 0x87); - t1 = veorq_u64(t1, t2_red); - - // Second reduce: t0 = t0 + x^64 · t1 (mod p). - let mut t0 = t0; - let t1_shifted = vextq_u64::<1>(zero, t1); - t0 = veorq_u64(t0, t1_shifted); - let t1_hi_s = vgetq_lane_u64::<1>(t1); - let t1_red = pmull(t1_hi_s, 0x87); - t0 = veorq_u64(t0, t1_red); - - F128 { - lo: vgetq_lane_u64::<0>(t0), - hi: vgetq_lane_u64::<1>(t0), - } - } - } - - /// Batch multiply 2× F128 in parallel. - /// - /// Strategy: 8 schoolbook PMULLs (4 per mul, all independent), repack the - /// four unreduced 64-bit words `(r0, r1, r2, r3)` of each product into - /// lane-paired `uint64x2_t` registers, then run the GHASH shift-XOR - /// reduction once with each NEON op handling both muls' lanes. Trades - /// the binius variant's 4 reduction-stage PMULLs (2 per mul × 2 muls) - /// for a vectorised XOR-based reduction. Worth it because PMULL is the - /// scarce resource on M-class (2 units, 1/cycle each). - /// - /// # Safety - /// Requires the `aes` target feature (compiles to PMULL); only call where - /// `aes` is statically enabled or has been runtime-detected. - #[target_feature(enable = "aes")] - pub unsafe fn ghash_mul_vec2_neon(a: [F128; 2], b: [F128; 2]) -> [F128; 2] { - // SAFETY: function carries the aes target feature; pmull requires it. - unsafe { - // 8 independent schoolbook PMULLs. - let p0_ll = pmull(a[0].lo, b[0].lo); - let p0_lh = pmull(a[0].lo, b[0].hi); - let p0_hl = pmull(a[0].hi, b[0].lo); - let p0_hh = pmull(a[0].hi, b[0].hi); - let p1_ll = pmull(a[1].lo, b[1].lo); - let p1_lh = pmull(a[1].lo, b[1].hi); - let p1_hl = pmull(a[1].hi, b[1].lo); - let p1_hh = pmull(a[1].hi, b[1].hi); - - // Per-mul cross terms (lh + hl). - let c0 = veorq_u64(p0_lh, p0_hl); - let c1 = veorq_u64(p1_lh, p1_hl); - - // Lane-paired (mul0, mul1) layout for each word position. - // r0 = ll_lo - // r1 = ll_hi ^ cross_lo - // r2 = hh_lo ^ cross_hi - // r3 = hh_hi - let r0 = vzip1q_u64(p0_ll, p1_ll); - let ll_hi = vzip2q_u64(p0_ll, p1_ll); - let c_lo = vzip1q_u64(c0, c1); - let r1 = veorq_u64(ll_hi, c_lo); - let hh_lo = vzip1q_u64(p0_hh, p1_hh); - let c_hi = vzip2q_u64(c0, c1); - let r2 = veorq_u64(hh_lo, c_hi); - let r3 = vzip2q_u64(p0_hh, p1_hh); - - // Vectorised GHASH reduction: fold (r2, r3) into (r0, r1) mod p, - // where p = x^128 + x^7 + x^2 + x + 1. r(x) = x^7 + x^2 + x + 1. - // Each shift produces (lo_part, overflow); the overflow goes into - // the next-higher word. - let s1_lo = vshlq_n_u64::<1>(r2); - let s1_hi = veorq_u64(vshlq_n_u64::<1>(r3), vshrq_n_u64::<63>(r2)); - let s2_lo = vshlq_n_u64::<2>(r2); - let s2_hi = veorq_u64(vshlq_n_u64::<2>(r3), vshrq_n_u64::<62>(r2)); - let s7_lo = vshlq_n_u64::<7>(r2); - let s7_hi = veorq_u64(vshlq_n_u64::<7>(r3), vshrq_n_u64::<57>(r2)); - - let t_lo = veorq_u64(veorq_u64(r2, s1_lo), veorq_u64(s2_lo, s7_lo)); - let t_hi = veorq_u64(veorq_u64(r3, s1_hi), veorq_u64(s2_hi, s7_hi)); - - // Bits of r3 that overflowed past position 127 in the three shifts. - let ov = veorq_u64( - veorq_u64(vshrq_n_u64::<63>(r3), vshrq_n_u64::<62>(r3)), - vshrq_n_u64::<57>(r3), - ); - let corr = veorq_u64( - veorq_u64(ov, vshlq_n_u64::<1>(ov)), - veorq_u64(vshlq_n_u64::<2>(ov), vshlq_n_u64::<7>(ov)), - ); - - let final_lo = veorq_u64(veorq_u64(r0, t_lo), corr); - let final_hi = veorq_u64(r1, t_hi); - - // Unpack: lane 0 → mul0, lane 1 → mul1. - [ - F128 { - lo: vgetq_lane_u64::<0>(final_lo), - hi: vgetq_lane_u64::<0>(final_hi), - }, - F128 { - lo: vgetq_lane_u64::<1>(final_lo), - hi: vgetq_lane_u64::<1>(final_hi), - }, - ] - } - } - - /// Full 256-bit carry-less product `a · b`, no mod-p reduction. The standard - /// middle-cross fold is baked in: r1 = ll_hi ^ cross_lo, r2 = hh_lo ^ cross_hi. - /// - /// # Safety - /// Requires the `aes` target feature (compiles to PMULL); only call where - /// `aes` is statically enabled or has been runtime-detected. - #[target_feature(enable = "aes")] - pub unsafe fn ghash_mul_unreduced_neon(a: F128, b: F128) -> F256Unreduced { - // SAFETY: function carries the aes target feature. - unsafe { - let p_ll = pmull(a.lo, b.lo); - let p_lh = pmull(a.lo, b.hi); - let p_hl = pmull(a.hi, b.lo); - let p_hh = pmull(a.hi, b.hi); - - let ll_lo = vgetq_lane_u64::<0>(p_ll); - let ll_hi = vgetq_lane_u64::<1>(p_ll); - let hh_lo = vgetq_lane_u64::<0>(p_hh); - let hh_hi = vgetq_lane_u64::<1>(p_hh); - let cross = veorq_u64(p_lh, p_hl); - let cr_lo = vgetq_lane_u64::<0>(cross); - let cr_hi = vgetq_lane_u64::<1>(cross); - - F256Unreduced { - r0: ll_lo, - r1: ll_hi ^ cr_lo, - r2: hh_lo ^ cr_hi, - r3: hh_hi, - } - } - } - - /// Two full carry-less products, without reduction. The independent PMULLs - /// are issued together so the NEON path has the same batched structure as - /// the AVX2 and AVX-512 radix-four kernels. - /// - /// # Safety - /// Requires the `aes` target feature (which includes PMULL). - #[target_feature(enable = "aes")] - pub unsafe fn ghash_mul_unreduced_vec2_neon(a: [F128; 2], b: [F128; 2]) -> [F256Unreduced; 2] { - // SAFETY: this function carries the required target feature. - unsafe { - let p0_ll = pmull(a[0].lo, b[0].lo); - let p0_lh = pmull(a[0].lo, b[0].hi); - let p0_hl = pmull(a[0].hi, b[0].lo); - let p0_hh = pmull(a[0].hi, b[0].hi); - let p1_ll = pmull(a[1].lo, b[1].lo); - let p1_lh = pmull(a[1].lo, b[1].hi); - let p1_hl = pmull(a[1].hi, b[1].lo); - let p1_hh = pmull(a[1].hi, b[1].hi); - let c0 = veorq_u64(p0_lh, p0_hl); - let c1 = veorq_u64(p1_lh, p1_hl); - [ - F256Unreduced { - r0: vgetq_lane_u64::<0>(p0_ll), - r1: vgetq_lane_u64::<1>(p0_ll) ^ vgetq_lane_u64::<0>(c0), - r2: vgetq_lane_u64::<0>(p0_hh) ^ vgetq_lane_u64::<1>(c0), - r3: vgetq_lane_u64::<1>(p0_hh), - }, - F256Unreduced { - r0: vgetq_lane_u64::<0>(p1_ll), - r1: vgetq_lane_u64::<1>(p1_ll) ^ vgetq_lane_u64::<0>(c1), - r2: vgetq_lane_u64::<0>(p1_hh) ^ vgetq_lane_u64::<1>(c1), - r3: vgetq_lane_u64::<1>(p1_hh), - }, - ] - } - } -} - -// --------------------------------------------------------------------------- -// x86_64 + PCLMULQDQ: CLMUL-based multiplication (the x86 twin of the NEON -// PMULL paths above). `_mm_clmulepi64_si128` is one 64×64 carry-less mul per -// instruction; the immediate selects which 64-bit half of each operand. -// --------------------------------------------------------------------------- - -#[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] -pub mod x86_64 { - use super::{F128, F256Unreduced}; - use core::arch::x86_64::*; - - /// The reduction polynomial r(x) = x^7 + x^2 + x + 1 in the low lane. - #[inline] - #[target_feature(enable = "sse2")] - unsafe fn poly() -> __m128i { - _mm_set_epi64x(0, 0x87) - } - - #[inline] - #[target_feature(enable = "sse2")] - unsafe fn load(a: F128) -> __m128i { - _mm_set_epi64x(a.hi as i64, a.lo as i64) - } - - #[inline] - #[target_feature(enable = "sse2")] - unsafe fn lane0(v: __m128i) -> u64 { - _mm_cvtsi128_si64(v) as u64 - } - - #[inline] - #[target_feature(enable = "sse2")] - unsafe fn lane1(v: __m128i) -> u64 { - _mm_cvtsi128_si64(_mm_unpackhi_epi64(v, v)) as u64 - } - - /// Binius-style: schoolbook 4 CLMUL + recursive 2-stage reduction (2 CLMUL). - /// Mirrors `aarch64::ghash_mul_binius`; each stage keeps the intermediate - /// ≤128 bits, so no separate overflow-correction term is needed. - /// - /// # Safety - /// Requires the `pclmulqdq` target feature; only call where it is - /// statically enabled or has been runtime-detected. - #[target_feature(enable = "pclmulqdq", enable = "sse2")] - pub unsafe fn ghash_mul_clmul(a: F128, b: F128) -> F128 { - // SAFETY: function carries the pclmulqdq+sse2 target features. - unsafe { - let x = load(a); - let y = load(b); - - let t0 = _mm_clmulepi64_si128::<0x00>(x, y); // a.lo · b.lo - let t1a = _mm_clmulepi64_si128::<0x10>(x, y); // a.lo · b.hi - let t1b = _mm_clmulepi64_si128::<0x01>(x, y); // a.hi · b.lo - let t2 = _mm_clmulepi64_si128::<0x11>(x, y); // a.hi · b.hi - let mut t1 = _mm_xor_si128(t1a, t1b); - - // First reduce: t1 = t1 + x^64 · t2 (mod p). - // _mm_slli_si128::<8> places t2.lo into the high lane (t1.hi). - t1 = _mm_xor_si128(t1, _mm_slli_si128::<8>(t2)); - t1 = _mm_xor_si128(t1, _mm_clmulepi64_si128::<0x01>(t2, poly())); - - // Second reduce: t0 = t0 + x^64 · t1 (mod p). - let mut t0 = t0; - t0 = _mm_xor_si128(t0, _mm_slli_si128::<8>(t1)); - t0 = _mm_xor_si128(t0, _mm_clmulepi64_si128::<0x01>(t1, poly())); - - F128 { - lo: lane0(t0), - hi: lane1(t0), - } - } - } - - /// Batch multiply 2× F128 with one 256-bit VPCLMULQDQ per product pair — - /// the x86 twin of `aarch64::ghash_mul_vec2_neon`. Each 128-bit lane of a - /// ymm register carries one multiplication; the binius-style 2-stage - /// reduction runs per-lane. 6 vpclmul (3 per mul, same count as the scalar - /// path) but half the instructions, doubling CLMUL throughput on cores - /// with a full-width VPCLMULQDQ unit (Zen 4+, Ice Lake+). - /// - /// # Safety - /// Requires the `vpclmulqdq` + `avx2` target features; only call where - /// they are statically enabled or have been runtime-detected. - #[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx2"))] - #[target_feature(enable = "vpclmulqdq", enable = "avx2")] - pub unsafe fn ghash_mul_vec2_clmul(a: [F128; 2], b: [F128; 2]) -> [F128; 2] { - // SAFETY: function carries the vpclmulqdq+avx2 target features; the - // loads/stores are 32-byte unaligned on [F128; 2] (32 bytes, repr(C)). - unsafe { - let x = _mm256_loadu_si256(a.as_ptr() as *const __m256i); - let y = _mm256_loadu_si256(b.as_ptr() as *const __m256i); - let poly = _mm256_set_epi64x(0, 0x87, 0, 0x87); - - let t0 = _mm256_clmulepi64_epi128::<0x00>(x, y); // lo·lo per lane - let t1a = _mm256_clmulepi64_epi128::<0x10>(x, y); // lo·hi - let t1b = _mm256_clmulepi64_epi128::<0x01>(x, y); // hi·lo - let t2 = _mm256_clmulepi64_epi128::<0x11>(x, y); // hi·hi - let mut t1 = _mm256_xor_si256(t1a, t1b); - - // First reduce: t1 = t1 + x^64 · t2 (mod p), per 128-bit lane. - t1 = _mm256_xor_si256(t1, _mm256_bslli_epi128::<8>(t2)); - t1 = _mm256_xor_si256(t1, _mm256_clmulepi64_epi128::<0x01>(t2, poly)); - - // Second reduce: t0 = t0 + x^64 · t1 (mod p). - let mut t0 = t0; - t0 = _mm256_xor_si256(t0, _mm256_bslli_epi128::<8>(t1)); - t0 = _mm256_xor_si256(t0, _mm256_clmulepi64_epi128::<0x01>(t1, poly)); - - let mut out = [F128::ZERO; 2]; - _mm256_storeu_si256(out.as_mut_ptr() as *mut __m256i, t0); - out - } - } - - /// Two full carry-less products in parallel, without reduction. - /// - /// # Safety - /// Requires `vpclmulqdq` and `avx2`. - #[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx2"))] - #[target_feature(enable = "vpclmulqdq", enable = "avx2")] - pub unsafe fn ghash_mul_unreduced_vec2_clmul(a: [F128; 2], b: [F128; 2]) -> [F256Unreduced; 2] { - // SAFETY: this function carries the required target features and each - // unaligned load spans exactly two repr(C) field elements. - unsafe { - let x = _mm256_loadu_si256(a.as_ptr().cast()); - let y = _mm256_loadu_si256(b.as_ptr().cast()); - let ll = _mm256_clmulepi64_epi128::<0x00>(x, y); - let lh = _mm256_clmulepi64_epi128::<0x10>(x, y); - let hl = _mm256_clmulepi64_epi128::<0x01>(x, y); - let hh = _mm256_clmulepi64_epi128::<0x11>(x, y); - let mut ll_words = [0u64; 4]; - let mut lh_words = [0u64; 4]; - let mut hl_words = [0u64; 4]; - let mut hh_words = [0u64; 4]; - _mm256_storeu_si256(ll_words.as_mut_ptr().cast(), ll); - _mm256_storeu_si256(lh_words.as_mut_ptr().cast(), lh); - _mm256_storeu_si256(hl_words.as_mut_ptr().cast(), hl); - _mm256_storeu_si256(hh_words.as_mut_ptr().cast(), hh); - std::array::from_fn(|lane| { - let lo = 2 * lane; - let hi = lo + 1; - F256Unreduced { - r0: ll_words[lo], - r1: ll_words[hi] ^ lh_words[lo] ^ hl_words[lo], - r2: hh_words[lo] ^ lh_words[hi] ^ hl_words[hi], - r3: hh_words[hi], - } - }) - } - } - - /// Four reduced field products, one in each 128-bit AVX-512 lane. - /// - /// # Safety - /// Requires `vpclmulqdq`, `avx512f`, and `avx512bw`. - #[cfg(all( - target_feature = "vpclmulqdq", - target_feature = "avx512f", - target_feature = "avx512bw" - ))] - #[target_feature(enable = "vpclmulqdq", enable = "avx512f", enable = "avx512bw")] - pub unsafe fn ghash_mul_vec4_clmul(a: [F128; 4], b: [F128; 4]) -> [F128; 4] { - // SAFETY: this function carries all required target features. - unsafe { - let x = _mm512_loadu_si512(a.as_ptr().cast()); - let y = _mm512_loadu_si512(b.as_ptr().cast()); - let poly = _mm512_set_epi64(0, 0x87, 0, 0x87, 0, 0x87, 0, 0x87); - let t0 = _mm512_clmulepi64_epi128::<0x00>(x, y); - let t1a = _mm512_clmulepi64_epi128::<0x10>(x, y); - let t1b = _mm512_clmulepi64_epi128::<0x01>(x, y); - let t2 = _mm512_clmulepi64_epi128::<0x11>(x, y); - let mut t1 = _mm512_xor_si512(t1a, t1b); - t1 = _mm512_xor_si512(t1, _mm512_bslli_epi128::<8>(t2)); - t1 = _mm512_xor_si512(t1, _mm512_clmulepi64_epi128::<0x01>(t2, poly)); - let mut t0 = t0; - t0 = _mm512_xor_si512(t0, _mm512_bslli_epi128::<8>(t1)); - t0 = _mm512_xor_si512(t0, _mm512_clmulepi64_epi128::<0x01>(t1, poly)); - let mut out = [F128::ZERO; 4]; - _mm512_storeu_si512(out.as_mut_ptr().cast(), t0); - out - } - } - - /// Four full carry-less products in parallel, without reduction. - /// - /// # Safety - /// Requires `vpclmulqdq` and `avx512f`. - #[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] - #[target_feature(enable = "vpclmulqdq", enable = "avx512f")] - pub unsafe fn ghash_mul_unreduced_vec4_clmul(a: [F128; 4], b: [F128; 4]) -> [F256Unreduced; 4] { - // SAFETY: this function carries the required target features and every - // load/store spans exactly four repr(C) field elements. - unsafe { - let x = _mm512_loadu_si512(a.as_ptr().cast()); - let y = _mm512_loadu_si512(b.as_ptr().cast()); - let ll = _mm512_clmulepi64_epi128::<0x00>(x, y); - let lh = _mm512_clmulepi64_epi128::<0x10>(x, y); - let hl = _mm512_clmulepi64_epi128::<0x01>(x, y); - let hh = _mm512_clmulepi64_epi128::<0x11>(x, y); - let mut ll_words = [0u64; 8]; - let mut lh_words = [0u64; 8]; - let mut hl_words = [0u64; 8]; - let mut hh_words = [0u64; 8]; - _mm512_storeu_si512(ll_words.as_mut_ptr().cast(), ll); - _mm512_storeu_si512(lh_words.as_mut_ptr().cast(), lh); - _mm512_storeu_si512(hl_words.as_mut_ptr().cast(), hl); - _mm512_storeu_si512(hh_words.as_mut_ptr().cast(), hh); - std::array::from_fn(|lane| { - let lo = 2 * lane; - let hi = lo + 1; - F256Unreduced { - r0: ll_words[lo], - r1: ll_words[hi] ^ lh_words[lo] ^ hl_words[lo], - r2: hh_words[lo] ^ lh_words[hi] ^ hl_words[hi], - r3: hh_words[hi], - } - }) - } - } - - /// Full 256-bit carry-less product `a · b`, no mod-p reduction. The standard - /// middle-cross fold is baked in: r1 = ll_hi ^ cross_lo, r2 = hh_lo ^ cross_hi. - /// - /// # Safety - /// Requires the `pclmulqdq` target feature; only call where it is - /// statically enabled or has been runtime-detected. - #[target_feature(enable = "pclmulqdq", enable = "sse2")] - pub unsafe fn ghash_mul_unreduced_clmul(a: F128, b: F128) -> F256Unreduced { - // SAFETY: function carries the pclmulqdq+sse2 target features. - unsafe { - let x = load(a); - let y = load(b); - - let p_ll = _mm_clmulepi64_si128::<0x00>(x, y); - let p_lh = _mm_clmulepi64_si128::<0x10>(x, y); - let p_hl = _mm_clmulepi64_si128::<0x01>(x, y); - let p_hh = _mm_clmulepi64_si128::<0x11>(x, y); - let cross = _mm_xor_si128(p_lh, p_hl); - - F256Unreduced { - r0: lane0(p_ll), - r1: lane1(p_ll) ^ lane0(cross), - r2: lane0(p_hh) ^ lane1(cross), - r3: lane1(p_hh), - } - } - } -} - -// --------------------------------------------------------------------------- -// Software fallback: bit-by-bit clmul64. Slow but portable; also the reference -// the NEON path is checked against in tests. -// --------------------------------------------------------------------------- - -pub mod software { - use super::{F128, F256Unreduced, ghash_reduce}; - - /// 64×64 carry-less product into 128 bits (lo, hi). - pub fn clmul64(a: u64, b: u64) -> (u64, u64) { - let mut lo: u64 = 0; - let mut hi: u64 = 0; - let mut i = 0; - while i < 64 { - if (a >> i) & 1 != 0 { - lo ^= b << i; - if i != 0 { - hi ^= b >> (64 - i); - } - } - i += 1; - } - (lo, hi) - } - - pub fn ghash_mul_unreduced(a: F128, b: F128) -> F256Unreduced { - let (ll_lo, ll_hi) = clmul64(a.lo, b.lo); - let (lh_lo, lh_hi) = clmul64(a.lo, b.hi); - let (hl_lo, hl_hi) = clmul64(a.hi, b.lo); - let (hh_lo, hh_hi) = clmul64(a.hi, b.hi); - let cr_lo = lh_lo ^ hl_lo; - let cr_hi = lh_hi ^ hl_hi; - F256Unreduced { - r0: ll_lo, - r1: ll_hi ^ cr_lo, - r2: hh_lo ^ cr_hi, - r3: hh_hi, - } - } - - pub fn ghash_mul(a: F128, b: F128) -> F128 { - let u = ghash_mul_unreduced(a, b); - ghash_reduce(u.r0, u.r1, u.r2, u.r3) - } -} - -#[inline] -fn ghash_mul_unreduced(a: F128, b: F128) -> F256Unreduced { - #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] - { - // SAFETY: aes target feature is enabled at compile time. - unsafe { aarch64::ghash_mul_unreduced_neon(a, b) } - } - #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] - { - // SAFETY: pclmulqdq target feature is enabled at compile time. - unsafe { x86_64::ghash_mul_unreduced_clmul(a, b) } - } - #[cfg(not(any( - all(target_arch = "aarch64", target_feature = "aes"), - all(target_arch = "x86_64", target_feature = "pclmulqdq") - )))] - { - software::ghash_mul_unreduced(a, b) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - struct Rng(u64); - impl Rng { - fn new(seed: u64) -> Self { - Self(seed) - } - fn next_u64(&mut self) -> u64 { - self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); - let mut z = self.0; - z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); - z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); - z ^ (z >> 31) - } - fn next_f128(&mut self) -> F128 { - F128 { - lo: self.next_u64(), - hi: self.next_u64(), - } - } - } - - #[test] - fn add_identities() { - let mut rng = Rng::new(1); - for _ in 0..64 { - let a = rng.next_f128(); - assert_eq!(a + F128::ZERO, a); - assert_eq!(a + a, F128::ZERO); - } - } - - #[test] - fn mul_identities() { - let mut rng = Rng::new(2); - for _ in 0..64 { - let a = rng.next_f128(); - assert_eq!(a * F128::ZERO, F128::ZERO); - assert_eq!(a * F128::ONE, a); - } - } - - #[test] - fn mul_by_x_matches_mul_by_gen() { - let mut rng = Rng::new(3); - for _ in 0..256 { - let a = rng.next_f128(); - assert_eq!(mul_by_x(a), a * F128::generator()); - } - } - - #[test] - fn deferred_reduction_matches_direct() { - let mut rng = Rng::new(4); - for _ in 0..64 { - let a = rng.next_f128(); - let b = rng.next_f128(); - let direct = a * b; - let deferred = a.mul_unreduced(b).reduce(); - assert_eq!(direct, deferred); - } - } - - #[test] - fn deferred_xor_commutes_with_reduction() { - // Σ aᵢ·bᵢ in F128 must equal reduce(XOR-sum of unreduced products). - let mut rng = Rng::new(5); - let n = 16; - let pairs: Vec<(F128, F128)> = (0..n).map(|_| (rng.next_f128(), rng.next_f128())).collect(); - - let direct: F128 = pairs.iter().fold(F128::ZERO, |acc, (a, b)| acc + *a * *b); - - let mut acc = F256Unreduced::ZERO; - for (a, b) in &pairs { - acc ^= a.mul_unreduced(*b); - } - assert_eq!(direct, acc.reduce()); - } - - #[test] - fn inverse_roundtrip() { - let mut rng = Rng::new(6); - for _ in 0..16 { - let a = rng.next_f128(); - if a.is_zero() { - continue; - } - assert_eq!(a * a.inv(), F128::ONE); - } - } - - #[test] - fn associativity_random() { - let mut rng = Rng::new(7); - for _ in 0..64 { - let a = rng.next_f128(); - let b = rng.next_f128(); - let c = rng.next_f128(); - assert_eq!((a * b) * c, a * (b * c)); - assert_eq!(a * (b + c), a * b + a * c); - } - } - - #[test] - fn mul_commutativity() { - let mut rng = Rng::new(91); - for _ in 0..256 { - let a = rng.next_f128(); - let b = rng.next_f128(); - assert_eq!(a * b, b * a); - } - } - - #[test] - fn ghash_reduction_smoking_gun() { - // The defining identity of the GHASH polynomial: - // x · x^127 = x^128 = x^7 + x^2 + x + 1 = 0x87. - // If the reduction constant 0x87 is wrong (e.g. 0x86, 0x07, byte-swapped), - // this test fails immediately and pinpoints the bug. - let x = F128::generator(); - let x_127 = F128 { - lo: 0, - hi: 1u64 << 63, - }; - assert_eq!(x * x_127, F128 { lo: 0x87, hi: 0 }, "x · x^127"); - - // x · x^63 = x^64 — crosses the lo/hi word boundary with no reduction. - // Catches lo/hi swaps and off-by-one in the 64-bit word split. - let x_63 = F128 { - lo: 1u64 << 63, - hi: 0, - }; - assert_eq!(x * x_63, F128 { lo: 0, hi: 1 }, "x · x^63 = x^64"); - - // x^64 · x^64 = x^128 = 0x87 — reaches the reduction through a different - // multiplication path (high·high product). - let x_64 = F128 { lo: 0, hi: 1 }; - assert_eq!(x_64 * x_64, F128 { lo: 0x87, hi: 0 }, "x^64 · x^64"); - - // x · x = x^2 (no reduction). - assert_eq!(x * x, F128 { lo: 4, hi: 0 }, "x^2"); - } - - #[test] - fn high_bit_inputs_reduce_correctly() { - // Verify mul still satisfies a^{-1} · a = 1 when both inputs have the - // top bit (x^127) set — exercising the most overflow-prone code path - // of `ghash_reduce`. The inverse test naturally lands here for random - // inputs only by luck; this makes it deterministic. - let high = F128 { - lo: 0, - hi: 1u64 << 63, - }; - assert_eq!(high * high.inv(), F128::ONE); - let almost_max = F128 { - lo: u64::MAX, - hi: u64::MAX, - }; - assert_eq!(almost_max * almost_max.inv(), F128::ONE); - let just_top = F128 { - lo: 0, - hi: u64::MAX, - }; - assert_eq!(just_top * just_top.inv(), F128::ONE); - } - - #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] - #[test] - fn neon_mul_vec2_matches_scalar() { - let mut rng = Rng::new(11); - for _ in 0..128 { - let a0 = rng.next_f128(); - let a1 = rng.next_f128(); - let b0 = rng.next_f128(); - let b1 = rng.next_f128(); - let expected = [a0 * b0, a1 * b1]; - let result = unsafe { aarch64::ghash_mul_vec2_neon([a0, a1], [b0, b1]) }; - assert_eq!(result[0], expected[0], "lane 0"); - assert_eq!(result[1], expected[1], "lane 1"); - } - } - - #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] - #[test] - fn neon_unreduced_vec2_matches_scalar() { - let mut rng = Rng::new(16); - for _ in 0..128 { - let a = std::array::from_fn(|_| rng.next_f128()); - let b = std::array::from_fn(|_| rng.next_f128()); - let expected = std::array::from_fn(|i| a[i].mul_unreduced(b[i])); - let result = unsafe { aarch64::ghash_mul_unreduced_vec2_neon(a, b) }; - assert_eq!(result, expected); - } - } - - #[cfg(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx2" - ))] - #[test] - fn clmul_mul_vec2_matches_scalar() { - let mut rng = Rng::new(13); - for _ in 0..128 { - let a0 = rng.next_f128(); - let a1 = rng.next_f128(); - let b0 = rng.next_f128(); - let b1 = rng.next_f128(); - let expected = [a0 * b0, a1 * b1]; - let result = unsafe { x86_64::ghash_mul_vec2_clmul([a0, a1], [b0, b1]) }; - assert_eq!(result[0], expected[0], "lane 0"); - assert_eq!(result[1], expected[1], "lane 1"); - } - } - - #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2"))] - #[test] - fn clmul_unreduced_vec2_matches_scalar() { - let mut rng = Rng::new(17); - for _ in 0..128 { - let a = std::array::from_fn(|_| rng.next_f128()); - let b = std::array::from_fn(|_| rng.next_f128()); - let expected = std::array::from_fn(|i| a[i].mul_unreduced(b[i])); - let result = unsafe { x86_64::ghash_mul_unreduced_vec2_clmul(a, b) }; - assert_eq!(result, expected); - } - } - - #[cfg(all( - target_arch = "x86_64", - target_feature = "vpclmulqdq", - target_feature = "avx512f", - target_feature = "avx512bw" - ))] - #[test] - fn clmul_mul_vec4_matches_scalar() { - let mut rng = Rng::new(14); - for _ in 0..128 { - let a = std::array::from_fn(|_| rng.next_f128()); - let b = std::array::from_fn(|_| rng.next_f128()); - let expected = std::array::from_fn(|i| a[i] * b[i]); - let result = unsafe { x86_64::ghash_mul_vec4_clmul(a, b) }; - assert_eq!(result, expected); - } - } - - #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] - #[test] - fn clmul_unreduced_vec4_matches_scalar() { - let mut rng = Rng::new(15); - for _ in 0..128 { - let a = std::array::from_fn(|_| rng.next_f128()); - let b = std::array::from_fn(|_| rng.next_f128()); - let expected = std::array::from_fn(|i| a[i].mul_unreduced(b[i])); - let result = unsafe { x86_64::ghash_mul_unreduced_vec4_clmul(a, b) }; - assert_eq!(result, expected); - } - } - - #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] - #[test] - fn clmul_variants_match_software() { - let mut rng = Rng::new(12); - for _ in 0..128 { - let a = rng.next_f128(); - let b = rng.next_f128(); - let sw = software::ghash_mul(a, b); - let cl = unsafe { x86_64::ghash_mul_clmul(a, b) }; - assert_eq!(sw, cl, "reduced mul"); - let sw_u = software::ghash_mul_unreduced(a, b); - let cl_u = unsafe { x86_64::ghash_mul_unreduced_clmul(a, b) }; - assert_eq!(sw_u, cl_u, "unreduced mul"); - } - } - - #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] - #[test] - fn all_neon_variants_agree() { - let mut rng = Rng::new(8); - for _ in 0..128 { - let a = rng.next_f128(); - let b = rng.next_f128(); - let sw = software::ghash_mul(a, b); - let sb = unsafe { aarch64::ghash_mul_schoolbook(a, b) }; - let ka = unsafe { aarch64::ghash_mul_karatsuba(a, b) }; - let kb = unsafe { aarch64::ghash_mul_karatsuba_barrett(a, b) }; - let bi = unsafe { aarch64::ghash_mul_binius(a, b) }; - assert_eq!(sw, sb); - assert_eq!(sw, ka); - assert_eq!(sw, kb); - assert_eq!(sw, bi); - } - } -} diff --git a/crates/primitives/src/field/gf2_64.rs b/crates/primitives/src/field/gf2_64.rs new file mode 100644 index 00000000..bbc6fa8e --- /dev/null +++ b/crates/primitives/src/field/gf2_64.rs @@ -0,0 +1,446 @@ +// CREDIT: https://github.com/binius-zk/binius64, Apache-2.0. +//! `K = GF(2)[x]/(x^64 + x^4 + x^3 + x + 1)`, +//! `R64 = 0x1B = x^4 + x^3 + x + 1`, and `ord(x) = 2^64 - 1`. +//! One multiplication = 1 product PMULL + 1 +//! fold PMULL + a ≤4-bit overflow tail; the product and fold never leave +//! the NEON register file. +//! +//! This is the base field for [`super::gf2_64x3`] (F192's tower base); the +//! reduction helper [`super::gf2_64x3::base_reduce_128`] is shared. + +use core::ops::{Add, AddAssign, Mul, MulAssign}; + +use serde::{Deserialize, Serialize}; + +#[cfg(target_arch = "aarch64")] +use super::gf2_64x3::R64; +use super::gf2_64x3::base_reduce_128; + +/// A GF(2^64) element; bit i = coefficient of x^i. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[repr(transparent)] +pub struct F64(pub u64); + +impl F64 { + pub const ZERO: Self = Self(0); + pub const ONE: Self = Self(1); + /// `x`, with `ord(x) = 2^64 - 1`. + pub const G: Self = Self(2); + + #[inline] + pub const fn is_zero(self) -> bool { + self.0 == 0 + } + + /// Squaring (cross terms vanish in char 2): same cost as mul here (the + /// PMULL already squares), kept for API symmetry with the other fields. + #[inline] + pub fn square(self) -> Self { + self * self + } + + /// Multiplicative inverse via Fermat: x^(2^64 − 2). `ZERO.inv() == ZERO`. + pub fn inv(self) -> Self { + let mut cur = self.square(); + let mut r = cur; + for _ in 2..64 { + cur = cur.square(); + r *= cur; + } + r + } +} + +#[allow(clippy::suspicious_arithmetic_impl)] +impl Add for F64 { + type Output = Self; + #[inline] + fn add(self, rhs: Self) -> Self { + Self(self.0 ^ rhs.0) + } +} + +#[allow(clippy::suspicious_op_assign_impl)] +impl AddAssign for F64 { + #[inline] + fn add_assign(&mut self, rhs: Self) { + self.0 ^= rhs.0; + } +} + +impl Mul for F64 { + type Output = Self; + #[inline] + fn mul(self, rhs: Self) -> Self { + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + { + // SAFETY: aes target feature is enabled at compile time. + unsafe { aarch64::mul_neon(self, rhs) } + } + #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] + { + // SAFETY: pclmulqdq target feature is enabled at compile time. + unsafe { x86_64::mul(self, rhs) } + } + #[cfg(not(any( + all(target_arch = "aarch64", target_feature = "aes"), + all(target_arch = "x86_64", target_feature = "pclmulqdq") + )))] + { + software::mul(self, rhs) + } + } +} + +impl MulAssign for F64 { + #[inline] + fn mul_assign(&mut self, rhs: Self) { + *self = *self * rhs; + } +} + +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +pub mod aarch64 { + use super::{F64, R64}; + use core::arch::aarch64::*; + use core::mem::transmute; + + const fn clmul8(a: u64, b: u64) -> u64 { + let mut r = 0u64; + let mut i = 0; + while i < 8 { + if (a >> i) & 1 == 1 { + r ^= b << i; + } + i += 1; + } + r + } + + /// `FOLD_TBL[ov] = clmul(ov, 0x1B)` for the ≤4-bit second-order overflow + /// `ov`: the exact final fold (fits in 8 bits), as a 16-entry TBL table. + /// One TBL folds both lanes of a pair reduction at once. + const FOLD_TBL: [u8; 16] = { + let mut t = [0u8; 16]; + let mut n = 0; + while n < 16 { + t[n] = clmul8(n as u64, R64) as u8; + n += 1; + } + t + }; + + /// 64x64 carry-less product as a 128-bit NEON vector. + /// + /// # Safety + /// Requires the `aes` target feature (compiles to PMULL); only call where + /// `aes` is statically enabled or has been runtime-detected. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn pmull(a: u64, b: u64) -> uint64x2_t { + // SAFETY: u128 and uint64x2_t are both 128-bit values. + unsafe { transmute::(vmull_p64(a, b)) } + } + + /// Carry-less product of the two *high* lanes: PMULL2 on the register + /// pair, no lane extraction (the lane-crossing-free way to fold a + /// product's high half). + /// + /// # Safety + /// Requires the `aes` target feature; see [`pmull`]. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn pmull_hi(a: uint64x2_t, b: uint64x2_t) -> uint64x2_t { + // SAFETY: bit-level reinterprets between 128-bit vector types. + unsafe { + transmute::(vmull_high_p64( + transmute::(a), + transmute::(b), + )) + } + } + + /// Reduce two 128-bit carry-less products into GF(2^64) as a lane pair: + /// returns `{reduce(p0), reduce(p1)}`. One PMULL-by-0x1B per product + /// folds the high half; the two ≤4-bit second-order overflows are folded + /// together by one vectorized shift-XOR (exact: ov·0x1B fits in 8 bits). + /// + /// # Safety + /// Requires the `aes` target feature; see [`pmull`]. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn reduce_pair(p0: uint64x2_t, p1: uint64x2_t) -> uint64x2_t { + // SAFETY: function carries the aes target feature. + unsafe { + let r = vdupq_n_u64(R64); + let t0 = pmull_hi(p0, r); + let t1 = pmull_hi(p1, r); + let lo = vtrn1q_u64(veorq_u64(p0, t0), veorq_u64(p1, t1)); + let ov = vtrn2q_u64(t0, t1); + let f = veorq_u64( + veorq_u64(ov, vshlq_n_u64::<1>(ov)), + veorq_u64(vshlq_n_u64::<3>(ov), vshlq_n_u64::<4>(ov)), + ); + veorq_u64(lo, f) + } + } + + /// Like [`reduce_pair`] but the second-order overflows also fold by + /// PMULL (4 PMULL total, minimal non-PMULL op count). Fastest pair + /// reduction in memory-resident loops (the NTT butterfly shape) on + /// M-series, where PMULL throughput is plentiful. + /// + /// # Safety + /// Requires the `aes` target feature; see [`pmull`]. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn reduce_pair_pmull4(p0: uint64x2_t, p1: uint64x2_t) -> uint64x2_t { + // SAFETY: function carries the aes target feature. + unsafe { + let r = vdupq_n_u64(R64); + let t0 = pmull_hi(p0, r); + let t1 = pmull_hi(p1, r); + // clmul(t.hi, 0x1B) fits in 8 bits (high lane 0): the exact fold + // of the ≤4-bit overflow, ready to XOR into lane 0. + let u0 = pmull_hi(t0, r); + let u1 = pmull_hi(t1, r); + vtrn1q_u64(veorq_u64(veorq_u64(p0, t0), u0), veorq_u64(veorq_u64(p1, t1), u1)) + } + } + + /// Like [`reduce_pair`] but the two ≤4-bit second-order overflows fold + /// through one 16-byte TBL lookup ([`FOLD_TBL`]): the overflow nibbles + /// sit in bytes 0 and 8 of the transposed high words and the table maps + /// each to its exact 8-bit fold in a single instruction. Shortest + /// dependency chain of the three pair reductions. + /// + /// # Safety + /// Requires the `aes` target feature; see [`pmull`]. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn reduce_pair_tbl(p0: uint64x2_t, p1: uint64x2_t) -> uint64x2_t { + // SAFETY: function carries the aes target feature. + unsafe { + let r = vdupq_n_u64(R64); + let t0 = pmull_hi(p0, r); + let t1 = pmull_hi(p1, r); + let lo = vtrn1q_u64(veorq_u64(p0, t0), veorq_u64(p1, t1)); + // ov = {t0.hi, t1.hi}, each ≤ 4 bits: byte 0 and byte 8 index the + // table; all other bytes are zero and map to zero. + let ov = vtrn2q_u64(t0, t1); + let table: uint8x16_t = transmute(FOLD_TBL); + let f = vreinterpretq_u64_u8(vqtbl1q_u8(table, vreinterpretq_u8_u64(ov))); + veorq_u64(lo, f) + } + } + + /// 3-PMULL fully vector-resident multiply: product, PMULL-by-0x1B fold of + /// the high half, second PMULL fold of the ≤4-bit overflow. Benchmark + /// alternate: best serial-chain latency by a hair, but the extra PMULL + /// costs throughput next to [`mul_shift_tail`]. + /// + /// # Safety + /// Requires the `aes` target feature; see [`pmull`]. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn mul_pmull_fold(a: F64, b: F64) -> F64 { + // SAFETY: function carries the aes target feature. + unsafe { + let r = vdupq_n_u64(R64); + let p = pmull(a.0, b.0); + let t = pmull_hi(p, r); // clmul(p.hi, 0x1B), ≤68 bits + let u = pmull_hi(t, r); // clmul(t.hi, 0x1B), ≤8 bits, high lane 0 + F64(vgetq_lane_u64::<0>(veorq_u64(veorq_u64(p, t), u))) + } + } + + /// 2-PMULL multiply: product, PMULL-by-0x1B fold, and a shift-XOR fold of + /// the ≤4-bit overflow (exact: ov·0x1B fits in 8 bits). LLVM lowers the + /// tail onto the scalar ports, which run free next to the PMULL-saturated + /// vector pipes: best throughput of the variants tried. + /// + /// # Safety + /// Requires the `aes` target feature; see [`pmull`]. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn mul_shift_tail(a: F64, b: F64) -> F64 { + // SAFETY: function carries the aes target feature. + unsafe { + let p = pmull(a.0, b.0); + let t = pmull_hi(p, vdupq_n_u64(R64)); + let ov = vdupq_laneq_u64::<1>(t); + let f = veorq_u64( + veorq_u64(ov, vshlq_n_u64::<1>(ov)), + veorq_u64(vshlq_n_u64::<3>(ov), vshlq_n_u64::<4>(ov)), + ); + F64(vgetq_lane_u64::<0>(veorq_u64(veorq_u64(p, t), f))) + } + } + + /// Default multiply kernel: [`mul_shift_tail`] (best throughput in both + /// register-chain and array loops; within 3% of the best latency). + /// + /// # Safety + /// Requires the `aes` target feature; see [`pmull`]. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn mul_neon(a: F64, b: F64) -> F64 { + // SAFETY: function carries the aes target feature. + unsafe { mul_shift_tail(a, b) } + } +} + +/// x86-64 `pclmulqdq` path — the twin of [`aarch64`] for AMD/Intel. GF(2^64) +/// multiply is one CLMUL product plus a two-CLMUL fold by `R64` (= x^64 mod P), +/// the same reduction as [`base_reduce_128`]. +#[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] +pub mod x86_64 { + use super::F64; + use crate::field::gf2_64x3::R64; + use core::arch::x86_64::*; + + /// 64×64 carry-less product as a 128-bit vector `{lo, hi}`. + /// + /// # Safety + /// Requires the `pclmulqdq` target feature; only call where it is + /// statically enabled or has been runtime-detected. + #[inline] + #[target_feature(enable = "pclmulqdq", enable = "sse2")] + pub unsafe fn clmul(a: u64, b: u64) -> __m128i { + _mm_clmulepi64_si128::<0x00>(_mm_set_epi64x(0, a as i64), _mm_set_epi64x(0, b as i64)) + } + + /// Reduce a 128-bit carry-less product `{lo, hi}` into GF(2^64): fold the + /// high word by `R64` (= x^64 mod P), then fold the ≤5-bit second-order + /// overflow once more. Two CLMUL; the exact residue of [`base_reduce_128`]. + /// + /// Credit: binius64 + /// (`crates/arith-bench/src/monbijou/clmul.rs::reduce`), whose Monbijou + /// field is this same GF(2^64) — a `<0x01>` CLMUL fold by `0x1B` applied + /// twice, XOR-ing the low halves. + /// + /// # Safety + /// Requires the `pclmulqdq` target feature; see [`clmul`]. + #[inline] + #[target_feature(enable = "pclmulqdq", enable = "sse2")] + pub unsafe fn reduce(p: __m128i) -> u64 { + let r = _mm_set_epi64x(0, R64 as i64); + let t = _mm_clmulepi64_si128::<0x01>(p, r); // clmul(p.hi, R64), ≤68 bits + let u = _mm_clmulepi64_si128::<0x01>(t, r); // clmul(t.hi, R64), ≤9 bits + _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(p, t), u)) as u64 + } + + /// One GF(2^64) multiply: product + reduction (3 CLMUL). + /// + /// # Safety + /// Requires the `pclmulqdq` target feature; see [`clmul`]. + #[inline] + #[target_feature(enable = "pclmulqdq", enable = "sse2")] + pub unsafe fn mul(a: F64, b: F64) -> F64 { + // SAFETY: function carries the pclmulqdq+sse2 target features. + unsafe { F64(reduce(clmul(a.0, b.0))) } + } +} + +pub mod software { + use super::{F64, base_reduce_128}; + use crate::field::gf2_64x3::clmul64; + + pub fn mul(a: F64, b: F64) -> F64 { + let (lo, hi) = clmul64(a.0, b.0); + F64(base_reduce_128(lo, hi)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// Independent Python reference vectors: (a, b, a·b). + const VECTORS: [(u64, u64, u64); 3] = [ + (0x01090913877ed8ed, 0x66ab35ac2768468f, 0x50c4519dc383744a), + (0xa7715ae18f12a3b5, 0x05743059f43fa4f5, 0xeb64cd9cd9cda6df), + (0xbd3efb4705e79ddd, 0x3aff618604de4ae0, 0xc3d7a95fa9cb59bb), + ]; + + #[test] + fn python_vectors() { + for (a, b, c) in VECTORS { + assert_eq!(F64(a) * F64(b), F64(c)); + assert_eq!(software::mul(F64(a), F64(b)), F64(c)); + } + } + + #[test] + fn neon_matches_software_and_axioms() { + let mut s = 1u64; + for _ in 0..10_000 { + let (a, b, c) = ( + F64(splitmix64(&mut s)), + F64(splitmix64(&mut s)), + F64(splitmix64(&mut s)), + ); + assert_eq!(a * b, software::mul(a, b)); + assert_eq!(a * b, b * a); + assert_eq!((a * b) * c, a * (b * c)); + assert_eq!(a * (b + c), a * b + a * c); + } + } + + /// Every NEON mul variant agrees with the software reference. + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + #[test] + fn neon_variants_match_software() { + let mut s = 5u64; + for _ in 0..10_000 { + let (a, b) = (F64(splitmix64(&mut s)), F64(splitmix64(&mut s))); + let want = software::mul(a, b); + // SAFETY: aes target feature is enabled at compile time. + unsafe { + assert_eq!(aarch64::mul_pmull_fold(a, b), want); + assert_eq!(aarch64::mul_shift_tail(a, b), want); + } + } + } + + #[test] + fn inv_and_identities() { + let mut s = 2u64; + for _ in 0..200 { + let a = F64(splitmix64(&mut s)); + assert_eq!(a * F64::ONE, a); + if !a.is_zero() { + assert_eq!(a * a.inv(), F64::ONE); + } + } + assert_eq!(F64::ZERO.inv(), F64::ZERO); + } + + /// x is primitive: x^((2^64−1)/q) ≠ 1 for every prime q | 2^64 − 1. + #[test] + fn x_is_primitive() { + fn pow(mut base: F64, mut e: u128) -> F64 { + let mut r = F64::ONE; + while e > 0 { + if e & 1 == 1 { + r *= base; + } + base = base.square(); + e >>= 1; + } + r + } + let n: u128 = (1 << 64) - 1; + for q in [3u128, 5, 17, 257, 641, 65537, 6700417] { + assert_ne!(pow(F64::G, n / q), F64::ONE, "x^((2^64-1)/{q}) == 1"); + } + } +} diff --git a/crates/primitives/src/field/gf2_64x3.rs b/crates/primitives/src/field/gf2_64x3.rs new file mode 100644 index 00000000..4e234004 --- /dev/null +++ b/crates/primitives/src/field/gf2_64x3.rs @@ -0,0 +1,1191 @@ +//! `K = GF(2)[x]/(x^64 + x^4 + x^3 + x + 1)`, +//! `R64 = 0x1B = x^4 + x^3 + x + 1`, and +//! `F192 = K[y]/(y^3 + y + 1)`. +//! +//! Layout: coefficients `c0 + c1·y + c2·y²`, each a GF(2^64) element (bit i of +//! `cj` = coeff of x^i). +//! +//! Hardware strategy (aarch64 + AES/PMULL): +//! - one base-field 64×64 product = one `vmull_p64`; +//! - extension mult = 3-term Karatsuba (6 PMULL, optimal for 3-term bilinear +//! over GF(2)) producing 5 unreduced 128-bit coefficients; +//! - y-fold (y³ = y+1, y⁴ = y²+y) on the unreduced coefficients — 4 NEON XORs; +//! - base reduction per coefficient: 1 PMULL by 0x1B; the ≤4-bit overflow is +//! folded with 3 scalar shift-XORs (0x1B·overflow fits in 8 bits, exact). +//! +//! Total: 9 PMULL per multiplication. Squaring uses 3 PMULL + 3 reduction +//! PMULL = 6. +//! +//! Why no packed-lane tricks: PMULL is one 64×64 product per instruction; base +//! coefficients already fill the operand exactly, so — unlike GF(2^32) — no +//! width is wasted. + +use core::ops::{Add, AddAssign, BitXor, BitXorAssign, Mul, MulAssign}; + +use serde::{Deserialize, Serialize}; + +use super::gf2_64::F64; + +/// Reduction constant of the base field: x^64 ≡ x^4 + x^3 + x + 1. +pub const R64: u64 = 0x1B; + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[repr(C)] +pub struct F192 { + pub c0: u64, + pub c1: u64, + pub c2: u64, +} + +impl F192 { + pub const ZERO: Self = Self { c0: 0, c1: 0, c2: 0 }; + pub const ONE: Self = Self { c0: 1, c1: 0, c2: 0 }; + /// The element `y` (root of y^3 + y + 1 over the base field). + pub const Y: Self = Self { c0: 0, c1: 1, c2: 0 }; + + #[inline] + pub const fn new(c0: u64, c1: u64, c2: u64) -> Self { + Self { c0, c1, c2 } + } + + #[inline] + pub const fn is_zero(self) -> bool { + self.c0 == 0 && self.c1 == 0 && self.c2 == 0 + } + + /// Unreduced product: the 5 raw 128-bit polynomial coefficients, before the + /// y-fold and base reductions. XOR-accumulate many of these and `.reduce()` + /// once — both folds are GF(2)-linear, so they commute with XOR. + #[inline] + pub fn mul_unreduced(self, rhs: Self) -> F192Unreduced { + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + { + // SAFETY: aes target feature is enabled at compile time. + unsafe { aarch64::mul_unreduced_neon(self, rhs) } + } + #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] + { + // SAFETY: pclmulqdq is enabled at compile time. + unsafe { x86_64::mul_unreduced(self, rhs) } + } + #[cfg(not(any( + all(target_arch = "aarch64", target_feature = "aes"), + all(target_arch = "x86_64", target_feature = "pclmulqdq") + )))] + { + software::mul_unreduced(self, rhs) + } + } + + /// Mixed product by a base-field scalar. Since the multiplier has no + /// `y` component, the three coefficients multiply independently in K. + #[inline] + pub fn mul_base(self, k: F64) -> Self { + Self { + c0: (F64(self.c0) * k).0, + c1: (F64(self.c1) * k).0, + c2: (F64(self.c2) * k).0, + } + } + + /// Unreduced mixed product for deferred inner-product accumulation. + #[inline] + pub fn mul_base_unreduced(self, k: F64) -> F192BaseUnreduced { + F192BaseUnreduced { + p0: kclmul(self.c0, k.0), + p1: kclmul(self.c1, k.0), + p2: kclmul(self.c2, k.0), + } + } + + /// Squaring. Char-2 cross terms vanish: (c0 + c1·y + c2·y²)² = + /// c0² + c1²·y² + c2²·y⁴, so 3 PMULL + reduction instead of 6. + #[inline] + pub fn square(self) -> Self { + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + { + // SAFETY: aes target feature is enabled at compile time. + unsafe { aarch64::square_neon(self) } + } + #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] + { + // SAFETY: pclmulqdq is enabled at compile time. + unsafe { x86_64::square(self) } + } + #[cfg(not(any( + all(target_arch = "aarch64", target_feature = "aes"), + all(target_arch = "x86_64", target_feature = "pclmulqdq") + )))] + { + software::square(self) + } + } + + /// Multiplicative inverse via Fermat: self^(2^192 − 2) = ∏_{i=1..191} self^(2^i). + /// One-time-setup speed class, not a hot path. `ZERO.inv() == ZERO`. + pub fn inv(self) -> Self { + let mut cur = self.square(); + let mut r = cur; + for _ in 2..192 { + cur = cur.square(); + r *= cur; + } + r + } +} + +impl Add for F192 { + type Output = Self; + #[inline] + fn add(self, rhs: Self) -> Self { + Self { + c0: self.c0 ^ rhs.c0, + c1: self.c1 ^ rhs.c1, + c2: self.c2 ^ rhs.c2, + } + } +} + +impl AddAssign for F192 { + #[inline] + fn add_assign(&mut self, rhs: Self) { + self.c0 ^= rhs.c0; + self.c1 ^= rhs.c1; + self.c2 ^= rhs.c2; + } +} + +impl From for F192 { + #[inline] + fn from(k: F64) -> Self { + Self { c0: k.0, c1: 0, c2: 0 } + } +} + +impl Mul for F192 { + type Output = Self; + #[inline] + fn mul(self, rhs: Self) -> Self { + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + { + // SAFETY: aes target feature is enabled at compile time. + unsafe { aarch64::mul_karatsuba(self, rhs) } + } + #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] + { + self.mul_unreduced(rhs).reduce() + } + #[cfg(not(any( + all(target_arch = "aarch64", target_feature = "aes"), + all(target_arch = "x86_64", target_feature = "pclmulqdq") + )))] + { + software::mul(self, rhs) + } + } +} + +impl MulAssign for F192 { + #[inline] + fn mul_assign(&mut self, rhs: Self) { + *self = *self * rhs; + } +} + +// --------------------------------------------------------------------------- +// Deferred reduction: 5 unreduced 128-bit coefficients, XOR-accumulable. +// --------------------------------------------------------------------------- + +/// Unreduced F192 product: the degree-4 polynomial product over K before any +/// reduction. `w[2k], w[2k+1]` = (lo, hi) of the 128-bit coefficient of y^k. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct F192Unreduced { + pub w: [u64; 10], +} + +impl F192Unreduced { + pub const ZERO: Self = Self { w: [0; 10] }; + + #[inline] + pub fn reduce(self) -> F192 { + let w = &self.w; + // y-fold: y³ = y + 1, y⁴ = y² + y (on 128-bit coefficients). + let d0 = (w[0] ^ w[6], w[1] ^ w[7]); + let d1 = (w[2] ^ w[6] ^ w[8], w[3] ^ w[7] ^ w[9]); + let d2 = (w[4] ^ w[8], w[5] ^ w[9]); + F192 { + c0: base_reduce_128(d0.0, d0.1), + c1: base_reduce_128(d1.0, d1.1), + c2: base_reduce_128(d2.0, d2.1), + } + } +} + +impl BitXor for F192Unreduced { + type Output = Self; + #[inline] + fn bitxor(self, rhs: Self) -> Self { + let mut w = self.w; + for i in 0..10 { + w[i] ^= rhs.w[i]; + } + Self { w } + } +} + +impl BitXorAssign for F192Unreduced { + #[inline] + fn bitxor_assign(&mut self, rhs: Self) { + for i in 0..10 { + self.w[i] ^= rhs.w[i]; + } + } +} + +/// Three raw K products from multiplying an F192 element by an F64 scalar. +/// They can be XOR-accumulated and reduced once per coefficient. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct F192BaseUnreduced { + pub p0: u128, + pub p1: u128, + pub p2: u128, +} + +impl F192BaseUnreduced { + pub const ZERO: Self = Self { p0: 0, p1: 0, p2: 0 }; + + #[inline] + pub fn reduce(self) -> F192 { + F192 { + c0: kreduce_u128(self.p0), + c1: kreduce_u128(self.p1), + c2: kreduce_u128(self.p2), + } + } +} + +impl BitXor for F192BaseUnreduced { + type Output = Self; + + #[inline] + fn bitxor(self, rhs: Self) -> Self { + Self { + p0: self.p0 ^ rhs.p0, + p1: self.p1 ^ rhs.p1, + p2: self.p2 ^ rhs.p2, + } + } +} + +impl BitXorAssign for F192BaseUnreduced { + #[inline] + fn bitxor_assign(&mut self, rhs: Self) { + self.p0 ^= rhs.p0; + self.p1 ^= rhs.p1; + self.p2 ^= rhs.p2; + } +} + +#[inline] +fn kclmul(a: u64, b: u64) -> u128 { + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + { + // SAFETY: aes is enabled at compile time. + unsafe { + core::mem::transmute::(crate::field::gf2_64::aarch64::pmull(a, b)) + } + } + #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] + { + // SAFETY: pclmulqdq is enabled at compile time. + unsafe { core::mem::transmute::(crate::field::gf2_64::x86_64::clmul(a, b)) } + } + #[cfg(not(any( + all(target_arch = "aarch64", target_feature = "aes"), + all(target_arch = "x86_64", target_feature = "pclmulqdq") + )))] + { + let (lo, hi) = clmul64(a, b); + lo as u128 | ((hi as u128) << 64) + } +} + +#[inline] +fn kreduce_u128(v: u128) -> u64 { + #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] + { + // SAFETY: pclmulqdq is enabled at compile time. + unsafe { crate::field::gf2_64::x86_64::reduce(core::mem::transmute::(v)) } + } + #[cfg(not(all(target_arch = "x86_64", target_feature = "pclmulqdq")))] + { + base_reduce_128(v as u64, (v >> 64) as u64) + } +} + +// --------------------------------------------------------------------------- +// Base-field reduction mod x^64 + x^4 + x^3 + x + 1. Works on any target. +// --------------------------------------------------------------------------- + +/// Fold a 128-bit carry-less product (lo, hi) into GF(2^64). +/// x^64 ≡ x^4+x^3+x+1, so U·x^64 ≡ U ^ U<<1 ^ U<<3 ^ U<<4; the ≤4 bits that +/// shift out past position 63 are folded once more (their product with 0x1B +/// fits in 8 bits — exact). +#[inline] +pub const fn base_reduce_128(lo: u64, hi: u64) -> u64 { + let f = hi ^ (hi << 1) ^ (hi << 3) ^ (hi << 4); + let ov = (hi >> 63) ^ (hi >> 61) ^ (hi >> 60); // bits shifted past 63 + lo ^ f ^ ov ^ (ov << 1) ^ (ov << 3) ^ (ov << 4) +} + +/// Portable 64×64 carry-less product, used by setup and fallback paths. +pub(crate) fn clmul64(a: u64, b: u64) -> (u64, u64) { + let (mut lo, mut hi) = (0, 0); + for i in 0..64 { + if (a >> i) & 1 != 0 { + lo ^= b << i; + if i != 0 { + hi ^= b >> (64 - i); + } + } + } + (lo, hi) +} + +/// Portable base-field helpers (reference-grade; tests and setup only). +pub mod base { + use super::{base_reduce_128, clmul64}; + + /// GF(2^64) multiply: carry-less 64×64 then fold. + pub fn mul(a: u64, b: u64) -> u64 { + let (lo, hi) = clmul64(a, b); + base_reduce_128(lo, hi) + } + + pub fn square(a: u64) -> u64 { + mul(a, a) + } + + /// Fermat inverse in GF(2^64): a^(2^64 − 2). + pub fn inv(a: u64) -> u64 { + let mut cur = square(a); + let mut r = cur; + for _ in 2..64 { + cur = square(cur); + r = mul(r, cur); + } + r + } +} + +// --------------------------------------------------------------------------- +// aarch64 + AES: PMULL-based multiplication variants. +// --------------------------------------------------------------------------- + +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +pub mod aarch64 { + use super::{F192, F192Unreduced, R64, base_reduce_128}; + use core::arch::aarch64::*; + use core::mem::transmute; + + /// 64×64 carry-less product as a 128-bit vector. + /// + /// # Safety + /// Requires the `aes` target feature (statically satisfied: every caller + /// is itself `#[target_feature(enable = "aes")]`). + #[inline] + #[target_feature(enable = "aes")] + unsafe fn pmull(a: u64, b: u64) -> uint64x2_t { + let prod = vmull_p64(a, b); + // SAFETY: u128 and uint64x2_t are both 128-bit values; bit-level + // reinterpret with no UB. + unsafe { transmute::(prod) } + } + + /// Fold one 128-bit coefficient into GF(2^64): 1 PMULL by 0x1B, then the + /// ≤4-bit overflow (high lane of the PMULL) is folded with shift-XORs on + /// the scalar side, off the busy PMULL ports. + /// + /// # Safety + /// Requires the `aes` target feature. + #[inline] + #[target_feature(enable = "aes")] + unsafe fn base_reduce(d: uint64x2_t) -> u64 { + // SAFETY: function carries the aes target feature. + unsafe { + let t = pmull(vgetq_lane_u64::<1>(d), R64); + let ov = vgetq_lane_u64::<1>(t); // ≤ 4 bits (deg(hi·0x1B) ≤ 67) + vgetq_lane_u64::<0>(d) ^ vgetq_lane_u64::<0>(t) ^ ov ^ (ov << 1) ^ (ov << 3) ^ (ov << 4) + } + } + + /// Karatsuba-3: 6 PMULL products (optimal bilinear count for 3 terms), + /// NEON XOR combination, y-fold, then 3 PMULL base reductions. 9 PMULL + /// total. Default `Mul` implementation. + /// + /// # Safety + /// Requires the `aes` target feature (compiles to PMULL); only call where + /// `aes` is statically enabled or has been runtime-detected. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn mul_karatsuba(a: F192, b: F192) -> F192 { + // SAFETY: function carries the aes target feature. + unsafe { + let p0 = pmull(a.c0, b.c0); + let p1 = pmull(a.c1, b.c1); + let p2 = pmull(a.c2, b.c2); + let p01 = pmull(a.c0 ^ a.c1, b.c0 ^ b.c1); + let p02 = pmull(a.c0 ^ a.c2, b.c0 ^ b.c2); + let p12 = pmull(a.c1 ^ a.c2, b.c1 ^ b.c2); + + // c0 = p0 c3 = p12 ^ p1 ^ p2 + // c1 = p01 ^ p0 ^ p1 c4 = p2 + // c2 = p02 ^ p0 ^ p1 ^ p2 + let t01 = veorq_u64(p0, p1); + let t12 = veorq_u64(p1, p2); + let c1 = veorq_u64(p01, t01); + let c2 = veorq_u64(veorq_u64(p02, p0), t12); + let c3 = veorq_u64(p12, t12); + + // y-fold: d0 = c0^c3, d1 = c1^c3^c4, d2 = c2^c4 (c4 = p2). + let d0 = veorq_u64(p0, c3); + let d1 = veorq_u64(veorq_u64(c1, c3), p2); + let d2 = veorq_u64(c2, p2); + + F192 { + c0: base_reduce(d0), + c1: base_reduce(d1), + c2: base_reduce(d2), + } + } + } + + /// Schoolbook: 9 fully independent PMULL products + 3 reduction PMULL. + /// More PMULL pressure than Karatsuba but no input-sum dependencies — + /// kept as a benchmark variant. + /// + /// # Safety + /// Requires the `aes` target feature; see [`mul_karatsuba`]. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn mul_schoolbook(a: F192, b: F192) -> F192 { + // SAFETY: function carries the aes target feature. + unsafe { + let q00 = pmull(a.c0, b.c0); + let q01 = pmull(a.c0, b.c1); + let q02 = pmull(a.c0, b.c2); + let q10 = pmull(a.c1, b.c0); + let q11 = pmull(a.c1, b.c1); + let q12 = pmull(a.c1, b.c2); + let q20 = pmull(a.c2, b.c0); + let q21 = pmull(a.c2, b.c1); + let q22 = pmull(a.c2, b.c2); + + let c1 = veorq_u64(q01, q10); + let c2 = veorq_u64(veorq_u64(q02, q11), q20); + let c3 = veorq_u64(q12, q21); + + let d0 = veorq_u64(q00, c3); + let d1 = veorq_u64(veorq_u64(c1, c3), q22); + let d2 = veorq_u64(c2, q22); + + F192 { + c0: base_reduce(d0), + c1: base_reduce(d1), + c2: base_reduce(d2), + } + } + } + + /// Karatsuba products + fully scalar shift-XOR base reduction (no + /// reduction PMULLs — 6 PMULL total). Trades PMULL-port pressure for + /// integer-ALU work; benchmark variant. + /// + /// # Safety + /// Requires the `aes` target feature; see [`mul_karatsuba`]. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn mul_karatsuba_scalar_reduce(a: F192, b: F192) -> F192 { + // SAFETY: function carries the aes target feature. + unsafe { + let p0 = pmull(a.c0, b.c0); + let p1 = pmull(a.c1, b.c1); + let p2 = pmull(a.c2, b.c2); + let p01 = pmull(a.c0 ^ a.c1, b.c0 ^ b.c1); + let p02 = pmull(a.c0 ^ a.c2, b.c0 ^ b.c2); + let p12 = pmull(a.c1 ^ a.c2, b.c1 ^ b.c2); + + let t01 = veorq_u64(p0, p1); + let t12 = veorq_u64(p1, p2); + let c1 = veorq_u64(p01, t01); + let c2 = veorq_u64(veorq_u64(p02, p0), t12); + let c3 = veorq_u64(p12, t12); + + let d0 = veorq_u64(p0, c3); + let d1 = veorq_u64(veorq_u64(c1, c3), p2); + let d2 = veorq_u64(c2, p2); + + F192 { + c0: base_reduce_128(vgetq_lane_u64::<0>(d0), vgetq_lane_u64::<1>(d0)), + c1: base_reduce_128(vgetq_lane_u64::<0>(d1), vgetq_lane_u64::<1>(d1)), + c2: base_reduce_128(vgetq_lane_u64::<0>(d2), vgetq_lane_u64::<1>(d2)), + } + } + } + + /// Karatsuba products only — 6 PMULL, no reduction at all. The caller + /// XOR-accumulates the raw coefficients (inner products, sumcheck-style). + /// + /// # Safety + /// Requires the `aes` target feature; see [`mul_karatsuba`]. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn mul_unreduced_neon(a: F192, b: F192) -> F192Unreduced { + // SAFETY: function carries the aes target feature. + unsafe { + let p0 = pmull(a.c0, b.c0); + let p1 = pmull(a.c1, b.c1); + let p2 = pmull(a.c2, b.c2); + let p01 = pmull(a.c0 ^ a.c1, b.c0 ^ b.c1); + let p02 = pmull(a.c0 ^ a.c2, b.c0 ^ b.c2); + let p12 = pmull(a.c1 ^ a.c2, b.c1 ^ b.c2); + + let t01 = veorq_u64(p0, p1); + let t12 = veorq_u64(p1, p2); + let c1 = veorq_u64(p01, t01); + let c2 = veorq_u64(veorq_u64(p02, p0), t12); + let c3 = veorq_u64(p12, t12); + + F192Unreduced { + w: [ + vgetq_lane_u64::<0>(p0), + vgetq_lane_u64::<1>(p0), + vgetq_lane_u64::<0>(c1), + vgetq_lane_u64::<1>(c1), + vgetq_lane_u64::<0>(c2), + vgetq_lane_u64::<1>(c2), + vgetq_lane_u64::<0>(c3), + vgetq_lane_u64::<1>(c3), + vgetq_lane_u64::<0>(p2), + vgetq_lane_u64::<1>(p2), + ], + } + } + } + + /// Squaring: cross terms vanish, squares land on y^0, y^2, y^4. + /// 3 PMULL squares + y-fold + 3 PMULL reductions. + /// + /// # Safety + /// Requires the `aes` target feature; see [`mul_karatsuba`]. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn square_neon(a: F192) -> F192 { + // SAFETY: function carries the aes target feature. + unsafe { + let s0 = pmull(a.c0, a.c0); + let s1 = pmull(a.c1, a.c1); + let s2 = pmull(a.c2, a.c2); + // (c0 + c1 y + c2 y²)² = s0 + s1 y² + s2 y⁴; y⁴ = y² + y: + // d0 = s0, d1 = s2, d2 = s1 ^ s2. + F192 { + c0: base_reduce(s0), + c1: base_reduce(s2), + c2: base_reduce(veorq_u64(s1, s2)), + } + } + } +} + +// --------------------------------------------------------------------------- +// x86-64 + PCLMULQDQ. +// --------------------------------------------------------------------------- + +#[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] +pub mod x86_64 { + use super::{F192, F192Unreduced}; + use crate::field::gf2_64::x86_64::clmul; + use core::arch::x86_64::__m128i; + + #[inline] + #[target_feature(enable = "pclmulqdq", enable = "sse2")] + unsafe fn product(a: u64, b: u64) -> u128 { + // SAFETY: the function carries pclmulqdq and both representations are 128 bits. + unsafe { core::mem::transmute::<__m128i, u128>(clmul(a, b)) } + } + + /// Six-product Karatsuba multiplication before either field reduction. + /// + /// # Safety + /// + /// The caller must run on a CPU with PCLMULQDQ and SSE2 support. + #[inline] + #[target_feature(enable = "pclmulqdq", enable = "sse2")] + pub unsafe fn mul_unreduced(a: F192, b: F192) -> F192Unreduced { + // SAFETY: the function carries pclmulqdq. + unsafe { + let p0 = product(a.c0, b.c0); + let p1 = product(a.c1, b.c1); + let p2 = product(a.c2, b.c2); + let p01 = product(a.c0 ^ a.c1, b.c0 ^ b.c1); + let p02 = product(a.c0 ^ a.c2, b.c0 ^ b.c2); + let p12 = product(a.c1 ^ a.c2, b.c1 ^ b.c2); + + let c1 = p01 ^ p0 ^ p1; + let c2 = p02 ^ p0 ^ p1 ^ p2; + let c3 = p12 ^ p1 ^ p2; + F192Unreduced { + w: [ + p0 as u64, + (p0 >> 64) as u64, + c1 as u64, + (c1 >> 64) as u64, + c2 as u64, + (c2 >> 64) as u64, + c3 as u64, + (c3 >> 64) as u64, + p2 as u64, + (p2 >> 64) as u64, + ], + } + } + } + + /// Two independent tower-field products in the two 128-bit lanes of a + /// YMM register. Each of the six Karatsuba base products is issued once + /// for both lanes, then the tower and base reductions stay packed. + /// + /// # Safety + /// Requires VPCLMULQDQ and AVX2 support. + #[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx2"))] + #[target_feature(enable = "vpclmulqdq", enable = "avx2")] + pub unsafe fn mul_vec2(a: [F192; 2], b: [F192; 2]) -> [F192; 2] { + use core::arch::x86_64::*; + + #[inline] + #[target_feature(enable = "vpclmulqdq", enable = "avx2")] + unsafe fn products(a: [F192; 2], b: [F192; 2]) -> [__m256i; 5] { + let pack = |x0: u64, x1: u64| _mm256_set_epi64x(0, x1 as i64, 0, x0 as i64); + let (a0, a1, a2) = (pack(a[0].c0, a[1].c0), pack(a[0].c1, a[1].c1), pack(a[0].c2, a[1].c2)); + let (b0, b1, b2) = (pack(b[0].c0, b[1].c0), pack(b[0].c1, b[1].c1), pack(b[0].c2, b[1].c2)); + let p0 = _mm256_clmulepi64_epi128::<0x00>(a0, b0); + let p1 = _mm256_clmulepi64_epi128::<0x00>(a1, b1); + let p2 = _mm256_clmulepi64_epi128::<0x00>(a2, b2); + let p01 = _mm256_clmulepi64_epi128::<0x00>(_mm256_xor_si256(a0, a1), _mm256_xor_si256(b0, b1)); + let p02 = _mm256_clmulepi64_epi128::<0x00>(_mm256_xor_si256(a0, a2), _mm256_xor_si256(b0, b2)); + let p12 = _mm256_clmulepi64_epi128::<0x00>(_mm256_xor_si256(a1, a2), _mm256_xor_si256(b1, b2)); + [ + p0, + _mm256_xor_si256(p01, _mm256_xor_si256(p0, p1)), + _mm256_xor_si256(p02, _mm256_xor_si256(p0, _mm256_xor_si256(p1, p2))), + _mm256_xor_si256(p12, _mm256_xor_si256(p1, p2)), + p2, + ] + } + + #[inline] + #[target_feature(enable = "vpclmulqdq", enable = "avx2")] + unsafe fn reduce_base(value: __m256i) -> __m256i { + let modulus = _mm256_set1_epi64x(super::R64 as i64); + let first = _mm256_clmulepi64_epi128::<0x01>(value, modulus); + let second = _mm256_clmulepi64_epi128::<0x01>(first, modulus); + _mm256_xor_si256(value, _mm256_xor_si256(first, second)) + } + + unsafe { + let [p0, p1, p2, p3, p4] = products(a, b); + let d0 = reduce_base(_mm256_xor_si256(p0, p3)); + let d1 = reduce_base(_mm256_xor_si256(p1, _mm256_xor_si256(p3, p4))); + let d2 = reduce_base(_mm256_xor_si256(p2, p4)); + let mut c0 = [0u64; 4]; + let mut c1 = [0u64; 4]; + let mut c2 = [0u64; 4]; + _mm256_storeu_si256(c0.as_mut_ptr().cast(), d0); + _mm256_storeu_si256(c1.as_mut_ptr().cast(), d1); + _mm256_storeu_si256(c2.as_mut_ptr().cast(), d2); + [F192::new(c0[0], c1[0], c2[0]), F192::new(c0[2], c1[2], c2[2])] + } + } + + /// Two independent unreduced products, packed exactly as [`mul_vec2`]. + /// + /// # Safety + /// Requires VPCLMULQDQ and AVX2 support. + #[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx2"))] + #[target_feature(enable = "vpclmulqdq", enable = "avx2")] + pub unsafe fn mul_unreduced_vec2(a: [F192; 2], b: [F192; 2]) -> [F192Unreduced; 2] { + use core::arch::x86_64::*; + + unsafe { + let pack = |x0: u64, x1: u64| _mm256_set_epi64x(0, x1 as i64, 0, x0 as i64); + let (a0, a1, a2) = (pack(a[0].c0, a[1].c0), pack(a[0].c1, a[1].c1), pack(a[0].c2, a[1].c2)); + let (b0, b1, b2) = (pack(b[0].c0, b[1].c0), pack(b[0].c1, b[1].c1), pack(b[0].c2, b[1].c2)); + let p0 = _mm256_clmulepi64_epi128::<0x00>(a0, b0); + let d1 = _mm256_clmulepi64_epi128::<0x00>(a1, b1); + let p4 = _mm256_clmulepi64_epi128::<0x00>(a2, b2); + let p1 = _mm256_xor_si256( + _mm256_clmulepi64_epi128::<0x00>(_mm256_xor_si256(a0, a1), _mm256_xor_si256(b0, b1)), + _mm256_xor_si256(p0, d1), + ); + let p2 = _mm256_xor_si256( + _mm256_clmulepi64_epi128::<0x00>(_mm256_xor_si256(a0, a2), _mm256_xor_si256(b0, b2)), + _mm256_xor_si256(p0, _mm256_xor_si256(d1, p4)), + ); + let p3 = _mm256_xor_si256( + _mm256_clmulepi64_epi128::<0x00>(_mm256_xor_si256(a1, a2), _mm256_xor_si256(b1, b2)), + _mm256_xor_si256(d1, p4), + ); + let mut words = [[0u64; 4]; 5]; + for (out, value) in words.iter_mut().zip([p0, p1, p2, p3, p4]) { + _mm256_storeu_si256(out.as_mut_ptr().cast(), value); + } + std::array::from_fn(|lane| { + let lo = 2 * lane; + F192Unreduced { + w: std::array::from_fn(|word| words[word / 2][lo + word % 2]), + } + }) + } + } + + /// Four independent tower-field products in four AVX-512 128-bit lanes. + /// + /// # Safety + /// Requires VPCLMULQDQ and AVX-512F support. + #[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] + #[target_feature(enable = "vpclmulqdq", enable = "avx512f")] + pub unsafe fn mul_vec4(a: [F192; 4], b: [F192; 4]) -> [F192; 4] { + use core::arch::x86_64::*; + + #[inline] + #[target_feature(enable = "vpclmulqdq", enable = "avx512f")] + unsafe fn pack(values: [F192; 4], coefficient: usize) -> __m512i { + let get = |value: F192| [value.c0, value.c1, value.c2][coefficient] as i64; + _mm512_set_epi64( + 0, + get(values[3]), + 0, + get(values[2]), + 0, + get(values[1]), + 0, + get(values[0]), + ) + } + + #[inline] + #[target_feature(enable = "vpclmulqdq", enable = "avx512f")] + unsafe fn reduce_base(value: __m512i) -> __m512i { + let modulus = _mm512_set1_epi64(super::R64 as i64); + let first = _mm512_clmulepi64_epi128::<0x01>(value, modulus); + let second = _mm512_clmulepi64_epi128::<0x01>(first, modulus); + _mm512_xor_si512(value, _mm512_xor_si512(first, second)) + } + + unsafe { + let (a0, a1, a2) = (pack(a, 0), pack(a, 1), pack(a, 2)); + let (b0, b1, b2) = (pack(b, 0), pack(b, 1), pack(b, 2)); + let p0 = _mm512_clmulepi64_epi128::<0x00>(a0, b0); + let p1 = _mm512_clmulepi64_epi128::<0x00>(a1, b1); + let p2 = _mm512_clmulepi64_epi128::<0x00>(a2, b2); + let p01 = _mm512_clmulepi64_epi128::<0x00>(_mm512_xor_si512(a0, a1), _mm512_xor_si512(b0, b1)); + let p02 = _mm512_clmulepi64_epi128::<0x00>(_mm512_xor_si512(a0, a2), _mm512_xor_si512(b0, b2)); + let p12 = _mm512_clmulepi64_epi128::<0x00>(_mm512_xor_si512(a1, a2), _mm512_xor_si512(b1, b2)); + let p3 = _mm512_xor_si512(p12, _mm512_xor_si512(p1, p2)); + let p4 = p2; + let p2 = _mm512_xor_si512(p02, _mm512_xor_si512(p0, _mm512_xor_si512(p1, p2))); + let p1 = _mm512_xor_si512(p01, _mm512_xor_si512(p0, p1)); + let d0 = reduce_base(_mm512_xor_si512(p0, p3)); + let d1 = reduce_base(_mm512_xor_si512(p1, _mm512_xor_si512(p3, p4))); + let d2 = reduce_base(_mm512_xor_si512(p2, p4)); + let mut c0 = [0u64; 8]; + let mut c1 = [0u64; 8]; + let mut c2 = [0u64; 8]; + _mm512_storeu_si512(c0.as_mut_ptr().cast(), d0); + _mm512_storeu_si512(c1.as_mut_ptr().cast(), d1); + _mm512_storeu_si512(c2.as_mut_ptr().cast(), d2); + std::array::from_fn(|lane| F192::new(c0[2 * lane], c1[2 * lane], c2[2 * lane])) + } + } + + /// Four independent unreduced products in four AVX-512 lanes. + /// + /// # Safety + /// Requires VPCLMULQDQ and AVX-512F support. + #[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] + #[target_feature(enable = "vpclmulqdq", enable = "avx512f")] + pub unsafe fn mul_unreduced_vec4(a: [F192; 4], b: [F192; 4]) -> [F192Unreduced; 4] { + use core::arch::x86_64::*; + + unsafe { + let pack = |values: [F192; 4], coefficient: usize| { + let get = |value: F192| [value.c0, value.c1, value.c2][coefficient] as i64; + _mm512_set_epi64( + 0, + get(values[3]), + 0, + get(values[2]), + 0, + get(values[1]), + 0, + get(values[0]), + ) + }; + let (a0, a1, a2) = (pack(a, 0), pack(a, 1), pack(a, 2)); + let (b0, b1, b2) = (pack(b, 0), pack(b, 1), pack(b, 2)); + let p0 = _mm512_clmulepi64_epi128::<0x00>(a0, b0); + let d1 = _mm512_clmulepi64_epi128::<0x00>(a1, b1); + let p4 = _mm512_clmulepi64_epi128::<0x00>(a2, b2); + let p1 = _mm512_xor_si512( + _mm512_clmulepi64_epi128::<0x00>(_mm512_xor_si512(a0, a1), _mm512_xor_si512(b0, b1)), + _mm512_xor_si512(p0, d1), + ); + let p2 = _mm512_xor_si512( + _mm512_clmulepi64_epi128::<0x00>(_mm512_xor_si512(a0, a2), _mm512_xor_si512(b0, b2)), + _mm512_xor_si512(p0, _mm512_xor_si512(d1, p4)), + ); + let p3 = _mm512_xor_si512( + _mm512_clmulepi64_epi128::<0x00>(_mm512_xor_si512(a1, a2), _mm512_xor_si512(b1, b2)), + _mm512_xor_si512(d1, p4), + ); + let mut words = [[0u64; 8]; 5]; + for (out, value) in words.iter_mut().zip([p0, p1, p2, p3, p4]) { + _mm512_storeu_si512(out.as_mut_ptr().cast(), value); + } + std::array::from_fn(|lane| { + let lo = 2 * lane; + F192Unreduced { + w: std::array::from_fn(|word| words[word / 2][lo + word % 2]), + } + }) + } + } + + /// Three carry-less squares followed by the common F192 reduction. + /// + /// # Safety + /// + /// The caller must run on a CPU with PCLMULQDQ and SSE2 support. + #[inline] + #[target_feature(enable = "pclmulqdq", enable = "sse2")] + pub unsafe fn square(a: F192) -> F192 { + // SAFETY: the function carries pclmulqdq. + unsafe { + let s0 = product(a.c0, a.c0); + let s1 = product(a.c1, a.c1); + let s2 = product(a.c2, a.c2); + F192Unreduced { + w: [ + s0 as u64, + (s0 >> 64) as u64, + 0, + 0, + s1 as u64, + (s1 >> 64) as u64, + 0, + 0, + s2 as u64, + (s2 >> 64) as u64, + ], + } + .reduce() + } + } +} + +// --------------------------------------------------------------------------- +// Software fallback: portable, also the reference the NEON path is tested +// against. +// --------------------------------------------------------------------------- + +pub mod software { + use super::{F192, F192Unreduced, base_reduce_128, clmul64}; + + /// Schoolbook 9-product unreduced coefficients. + pub fn mul_unreduced(a: F192, b: F192) -> F192Unreduced { + let a_ = [a.c0, a.c1, a.c2]; + let b_ = [b.c0, b.c1, b.c2]; + let mut c = [(0u64, 0u64); 5]; + for i in 0..3 { + for j in 0..3 { + let (lo, hi) = clmul64(a_[i], b_[j]); + c[i + j].0 ^= lo; + c[i + j].1 ^= hi; + } + } + F192Unreduced { + w: [ + c[0].0, c[0].1, c[1].0, c[1].1, c[2].0, c[2].1, c[3].0, c[3].1, c[4].0, c[4].1, + ], + } + } + + pub fn mul(a: F192, b: F192) -> F192 { + mul_unreduced(a, b).reduce() + } + + pub fn square(a: F192) -> F192 { + let (l0, h0) = clmul64(a.c0, a.c0); + let (l1, h1) = clmul64(a.c1, a.c1); + let (l2, h2) = clmul64(a.c2, a.c2); + // Squares land on y^0, y^2, y^4; y^4 = y^2 + y. + F192 { + c0: base_reduce_128(l0, h0), + c1: base_reduce_128(l2, h2), + c2: base_reduce_128(l1 ^ l2, h1 ^ h2), + } + } +} + +// --------------------------------------------------------------------------- +// Tests: NEON vs software, independent Python vectors, field axioms, and +// computational irreducibility proofs for both moduli. +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn splitmix64(state: &mut u64) -> u64 { + *state = state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = *state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + fn rand_elem(s: &mut u64) -> F192 { + F192::new(splitmix64(s), splitmix64(s), splitmix64(s)) + } + + /// Vectors generated by an independent Python implementation + /// (scratchpad/fieldref.py): (a, b, a·b, a·a). + type ReferenceVector = ([u64; 3], [u64; 3], [u64; 3], [u64; 3]); + const VECTORS: [ReferenceVector; 4] = [ + ( + [0x950e87d7f5606615, 0x2c61275c9e6b6cf8, 0x1f00bca0042db923], + [0x6dbca290a9eab706, 0x4c10a4fe30cffdda, 0xf26fff4cc4fd394d], + [0x888a0fc35abaf5f6, 0x68a84cbc132b0649, 0x9fdeaf613003cabe], + [0x8fba131ad5d46b8c, 0x1c170457f537a805, 0x3632cc098ca15135], + ), + ( + [0x6814a2bc786a6d2d, 0xa26b351e6c8042c5, 0x54760e7fbc051c6c], + [0xd4c08880a5a4666d, 0x29610ae0eed8f1e7, 0xc34bd8e2fe5213e5], + [0x2ad322ebf2f9043b, 0x8ac800aa67154c80, 0x6d0f76651d3c4d0c], + [0xcf800ef2b83bb43a, 0xefe1c6cd064dd44c, 0x57dc5c7a60e2981b], + ), + ( + [0x6c50afb6e9fb123d, 0x6f28d015a2aa0b9d, 0x4e385994ebac94af], + [0x194f9545adba52ce, 0xc675ce05588f882f, 0x57de8c051d4b7ef2], + [0xea6b9f9d23d4a1ff, 0xd82aa6058c431457, 0x5fd4d8fda2f1e74a], + [0x8f30fe43aa05b396, 0xe3593591eccd9efe, 0x7c5a1b128788c51f], + ), + ( + [0xd998efd82733e933, 0x6df216c33f8f3201, 0x11dc6f3fcb57d5d8], + [0x8860a84722025e05, 0x33176469aa6ef630, 0x607507ebc5b864d7], + [0xfa3a0d66cdfbc1b3, 0xbd47bd3343aad307, 0xdaf50186477f6a77], + [0x69c8d8c24f416884, 0x4b597d648a162147, 0x95603a5d95c9512a], + ), + ]; + + #[test] + fn python_vectors() { + for (a, b, c, s) in VECTORS { + let (a, b) = (F192::new(a[0], a[1], a[2]), F192::new(b[0], b[1], b[2])); + assert_eq!(a * b, F192::new(c[0], c[1], c[2])); + assert_eq!(a.square(), F192::new(s[0], s[1], s[2])); + assert_eq!(software::mul(a, b), F192::new(c[0], c[1], c[2])); + } + } + + #[test] + fn identities() { + let mut s = 1u64; + for _ in 0..100 { + let a = rand_elem(&mut s); + assert_eq!(a * F192::ONE, a); + assert_eq!(a * F192::ZERO, F192::ZERO); + assert_eq!(a + F192::ZERO, a); + assert_eq!(a + a, F192::ZERO); + } + // y^3 = y + 1 + assert_eq!(F192::Y * F192::Y * F192::Y, F192::Y + F192::ONE); + } + + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + #[test] + fn neon_variants_match_software() { + let mut s = 2u64; + for _ in 0..10_000 { + let a = rand_elem(&mut s); + let b = rand_elem(&mut s); + let want = software::mul(a, b); + // SAFETY: cfg-gated on the aes target feature. + unsafe { + assert_eq!(aarch64::mul_karatsuba(a, b), want); + assert_eq!(aarch64::mul_schoolbook(a, b), want); + assert_eq!(aarch64::mul_karatsuba_scalar_reduce(a, b), want); + assert_eq!(aarch64::mul_unreduced_neon(a, b).reduce(), want); + assert_eq!(aarch64::square_neon(a), software::square(a)); + } + } + } + + #[test] + fn axioms() { + let mut s = 3u64; + for _ in 0..1_000 { + let a = rand_elem(&mut s); + let b = rand_elem(&mut s); + let c = rand_elem(&mut s); + assert_eq!(a * b, b * a); + assert_eq!((a * b) * c, a * (b * c)); + assert_eq!(a * (b + c), a * b + a * c); + } + } + + #[test] + fn square_and_inv() { + let mut s = 4u64; + for _ in 0..50 { + let a = rand_elem(&mut s); + assert_eq!(a.square(), a * a); + if !a.is_zero() { + assert_eq!(a * a.inv(), F192::ONE); + } + } + assert_eq!(F192::ZERO.inv(), F192::ZERO); + } + + #[test] + fn unreduced_accumulation_matches_reduced_sum() { + let mut s = 5u64; + for _ in 0..100 { + let pairs: Vec<(F192, F192)> = (0..16).map(|_| (rand_elem(&mut s), rand_elem(&mut s))).collect(); + let mut acc = F192Unreduced::ZERO; + let mut want = F192::ZERO; + for &(a, b) in &pairs { + acc ^= a.mul_unreduced(b); + want += a * b; + } + assert_eq!(acc.reduce(), want); + } + } + + #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2"))] + #[test] + fn x86_batch_products_match_scalar() { + let mut state = 6u64; + for _ in 0..1_000 { + let a: [F192; 4] = std::array::from_fn(|_| rand_elem(&mut state)); + let b: [F192; 4] = std::array::from_fn(|_| rand_elem(&mut state)); + let expected: [F192; 4] = std::array::from_fn(|lane| a[lane] * b[lane]); + // SAFETY: this test is compiled only when VPCLMULQDQ and AVX2 are enabled. + unsafe { + assert_eq!(x86_64::mul_vec2([a[0], a[1]], [b[0], b[1]]), [expected[0], expected[1]]); + let unreduced = x86_64::mul_unreduced_vec2([a[2], a[3]], [b[2], b[3]]); + assert_eq!( + [unreduced[0].reduce(), unreduced[1].reduce()], + [expected[2], expected[3]] + ); + } + #[cfg(target_feature = "avx512f")] + // SAFETY: the nested cfg additionally guarantees AVX-512F. + unsafe { + assert_eq!(x86_64::mul_vec4(a, b), expected); + assert_eq!(x86_64::mul_unreduced_vec4(a, b).map(F192Unreduced::reduce), expected); + } + } + } + + // -- GF(2)[x] helpers on u128 for the base-polynomial irreducibility test. + + fn gf2_mod(mut a: u128, m: u128) -> u128 { + let dm = 127 - m.leading_zeros(); + while a != 0 { + let da = 127 - a.leading_zeros(); + if da < dm { + break; + } + a ^= m << (da - dm); + } + a + } + + fn gf2_gcd(mut a: u128, mut b: u128) -> u128 { + while b != 0 { + let r = gf2_mod(a, b); + a = b; + b = r; + } + a + } + + /// Rabin: p64 (degree 64 = 2^6) is irreducible iff x^(2^64) ≡ x mod p64 + /// and gcd(x^(2^32) − x, p64) = 1. + #[test] + fn base_poly_irreducible() { + const P64: u128 = (1u128 << 64) | (R64 as u128); + let mut t = 2u64; // the element x + for _ in 0..32 { + t = base::square(t); + } + assert_eq!(gf2_gcd((t as u128) ^ 2, P64), 1, "factor of degree | 32"); + for _ in 0..32 { + t = base::square(t); + } + assert_eq!(t, 2, "x^(2^64) != x mod p64"); + } + + // -- K[y] gcd for the extension-polynomial irreducibility test. + + fn pdeg(p: &[u64]) -> Option { + p.iter().rposition(|&c| c != 0) + } + + fn poly_mod(mut a: Vec, b: &[u64]) -> Vec { + let db = pdeg(b).expect("mod by zero poly"); + let lead_inv = base::inv(b[db]); + while let Some(da) = pdeg(&a) { + if da < db { + break; + } + let q = base::mul(a[da], lead_inv); + for i in 0..=db { + a[da - db + i] ^= base::mul(q, b[i]); + } + } + a + } + + fn poly_gcd(mut a: Vec, mut b: Vec) -> Vec { + while pdeg(&b).is_some() { + let r = poly_mod(a, &b); + a = b; + b = r; + } + a + } + + /// Checks `gcd(y^|K| - y, y^3+y+1) = 1`; computes `y^(2^64)` by 64 + /// squarings. + #[test] + fn extension_poly_irreducible_over_base() { + let mut t = F192::Y; + for _ in 0..64 { + t = t.square(); + } + let d = t + F192::Y; // y^(2^64) − y as a deg ≤ 2 poly over K + assert!(!d.is_zero()); + let f = vec![1u64, 1, 0, 1]; // y^3 + y + 1 + let g = poly_gcd(f, vec![d.c0, d.c1, d.c2]); + assert_eq!(pdeg(&g), Some(0), "y^3+y+1 has a root in GF(2^64)"); + } + + #[test] + fn serde_roundtrip() { + let a = F192::new(0x0123456789abcdef, 0xfedcba9876543210, 0x1122334455667788); + let ser = bincode::serialize(&a).unwrap(); + assert_eq!(bincode::deserialize::(&ser).unwrap(), a); + } +} diff --git a/crates/primitives/src/field/gf2_8.rs b/crates/primitives/src/field/gf2_8.rs index eedebbd2..e3c929e9 100644 --- a/crates/primitives/src/field/gf2_8.rs +++ b/crates/primitives/src/field/gf2_8.rs @@ -1,4 +1,5 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// CREDIT: https://github.com/binius-zk/binius64 (`packed_aes_16x8b_multiply`), Apache-2.0. // Copyright 2025 The Binius Developers // Copyright 2025 Irreducible, Inc. // Modifications copyright 2026 Succinct Labs, Benedikt Bunz, William Wang @@ -9,7 +10,7 @@ // (https://github.com/binius-zk/binius64, // `crates/field/src/arch/aarch64/simd_arithmetic.rs`). -//! GF(2^8) with the AES irreducible polynomial x^8 + x^4 + x^3 + x + 1. +//! `GF(2)[x]/(x^8 + x^4 + x^3 + x + 1)`. //! //! Reduction: x^8 ≡ x^4 + x^3 + x + 1, so the upper byte h folds back as //! h ^ (h<<1) ^ (h<<3) ^ (h<<4). @@ -155,7 +156,7 @@ pub mod neon { /// passed as `(c0, c1)`) modulo `x^8 + x^4 + x^3 + x + 1`, returning 16 reduced /// GF(2^8) values. /// - /// Two-stage Binius-style reduction: + /// Two-stage reduction: /// Stage 1: ch · QPLUS_RSH1 then ·2 (corrects for /x in QPLUS_RSH1) /// Stage 2: high bytes of stage-1 · QSTAR; take low bytes only. /// @@ -316,16 +317,8 @@ mod tests { // xtime: a · 0x02 (used by MixColumns), exhaustively cross-check // against the spec'd formula: xtime(a) = (a << 1) ^ (0x1B if a high bit). for a in 0u8..=255 { - let expected = if a & 0x80 != 0 { - (a << 1) ^ 0x1b - } else { - a << 1 - }; - assert_eq!( - (F8(a) * F8(0x02)).0, - expected, - "xtime mismatch at a=0x{a:02x}" - ); + let expected = if a & 0x80 != 0 { (a << 1) ^ 0x1b } else { a << 1 }; + assert_eq!((F8(a) * F8(0x02)).0, expected, "xtime mismatch at a=0x{a:02x}"); } } @@ -373,4 +366,4 @@ mod tests { assert_eq!(p, F8::ONE, "a^255 != 1 for a=0x{v:02x}"); } } -} \ No newline at end of file +} diff --git a/crates/primitives/src/field/mod.rs b/crates/primitives/src/field/mod.rs new file mode 100644 index 00000000..5beaca80 --- /dev/null +++ b/crates/primitives/src/field/mod.rs @@ -0,0 +1,108 @@ +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +//! `K = F64 = GF(2)[x]/(x^64 + x^4 + x^3 + x + 1)` and +//! `E = F192 = K[y]/(y^3 + y + 1)`. Addresses, pc/fp, counters, and physical +//! committed columns are K-valued. A machine word is `c0 + c1*y + c2*y² ∈ E` +//! with `c0,c1,c2 ∈ K`; challenges, sumcheck/GKR values, and transcript scalars +//! are E-valued. +//! +//! - [`F64`] — GF(2^64), polynomial x^64 + x^4 + x^3 + x + 1 +//! - [`F8`] — GF(2^8) with AES polynomial x^8 + x^4 + x^3 + x + 1 +//! - [`F192`] — `K[y]/(y^3 + y + 1)` +//! - [`F192Unreduced`] — its deferred-reduction accumulator + +#[cfg(target_arch = "aarch64")] +pub mod neon; + +pub mod gf2_64; +pub mod gf2_64x3; +pub mod gf2_8; +pub mod phi8_tower; + +pub use gf2_8::F8; +pub use gf2_64::F64; +pub use gf2_64x3::{F192, F192BaseUnreduced, F192Unreduced}; +pub use phi8_tower::{PHI_8_TABLE_192, phi8_192}; + +// --------------------------------------------------------------------------- +// leanVM g-power helpers: domain separators / opcodes as x^k, and the g-power +// index encoding (§1, §8). +// --------------------------------------------------------------------------- + +use rayon::prelude::*; + +/// Multiply by `x = g` in `K`, where `x^64 = x^4 + x^3 + x + 1` and +/// `0x1B = x^4 + x^3 + x + 1`. +/// `const` so table constants (`g^k` separators, opcodes) evaluate at compile time. +#[inline] +pub const fn mul_by_g(a: F64) -> F64 { + let carry = a.0 >> 63; + F64((a.0 << 1) ^ (0x1B * carry)) +} + +/// Multiply an `E`-element by the base generator `g = x ∈ K`: lane-wise +/// [`mul_by_g`] on all three `K`-coefficients — three shift+folds, no PMULL. +#[inline] +pub const fn mul_by_g_e(a: F192) -> F192 { + F192 { + c0: mul_by_g(F64(a.c0)).0, + c1: mul_by_g(F64(a.c1)).0, + c2: mul_by_g(F64(a.c2)).0, + } +} + +/// `[g^0, g^1, …, g^{n-1}]`, built in parallel: each chunk seeds with one g-power +/// (`x_pow`, `O(log)`) and fills by `mul_by_g`, breaking the serial prefix chain +/// across cores. +pub fn g_powers(n: usize) -> Vec { + const CHUNK: usize = 1 << 12; + let mut v = vec![F64::ZERO; n]; + v.par_chunks_mut(CHUNK).enumerate().for_each(|(ci, chunk)| { + let mut acc = x_pow(ci * CHUNK); + for slot in chunk.iter_mut() { + *slot = acc; + acc = mul_by_g(acc); + } + }); + v +} + +/// `x^k` in the monomial basis of `K` by square-and-multiply (`O(log k)`). Used +/// for domain separators, opcodes, and the g-power index encoding. +pub fn x_pow(k: usize) -> F64 { + let mut result = F64::ONE; + let mut base = G; // x = g + let mut e = k; + while e > 0 { + if e & 1 == 1 { + result *= base; + } + base = base * base; + e >>= 1; + } + result +} + +/// The fixed generator `g = x ∈ K`, with `ord(g) = 2^64 - 1` (pinned by a +/// field test), larger than every index any admissible +/// instance uses (the verifier's instance caps, §cpu). For `k < 64`, `g^k` is +/// the monomial `x^k` (bit `k`), which the XMSS encoding check relies on. +pub const G: F64 = F64::G; + +/// `g^i`, the g-power encoding of index `i` (§1). +#[inline] +pub fn g_pow(i: usize) -> F64 { + x_pow(i) +} + +/// MLE of the index column `[g^0, …, g^{2^n−1}]` over the `n`-variable cube, +/// evaluated at an `E`-point: `∏_k (1 + ζ_k·(1 + g^{2^k}))` in `O(n)` (§5.3). +/// The `g^{2^k}` factors are `K`-constants, so each term is one mixed product. +pub fn index_mle(zeta: &[F192]) -> F192 { + let mut acc = F192::ONE; + let mut g2k = G; // g^{2^0} = g + for &z in zeta { + acc *= F192::ONE + z.mul_base(F64::ONE + g2k); + g2k = g2k * g2k; + } + acc +} diff --git a/crates/primitives/src/field/neon.rs b/crates/primitives/src/field/neon.rs index 5acd3319..8dda8313 100644 --- a/crates/primitives/src/field/neon.rs +++ b/crates/primitives/src/field/neon.rs @@ -1,9 +1,8 @@ -//! Small AArch64 SIMD helpers shared by every NEON kernel in the workspace. +//! Small AArch64 SIMD helpers shared by the NEON kernels. //! //! The ARMv8.2 SHA3 extension — present on every Apple M core and enabled by //! the workspace's `-C target-cpu=native` — provides `EOR3`, a three-way XOR -//! in one instruction. The binary-field kernels are XOR-dominated (GHASH -//! reduction, table-gather accumulation, unreduced product folds), so folding +//! in one instruction. The binary-field kernels are XOR-dominated, so folding //! pairs of dependent `EOR`s into single `EOR3`s cuts both instruction count //! and dependency-chain length in the hot loops. Builds without the extension //! transparently keep the two-`EOR` form. @@ -13,16 +12,16 @@ use core::arch::aarch64::*; -/// Three-way XOR of byte-lane vectors. +/// Three-way XOR of 64-bit-lane vectors. /// /// # Safety /// Requires the `sha3` target feature on the EOR3 arm (statically satisfied by /// the `cfg` gate); the fallback arm has no requirement. #[cfg(target_feature = "sha3")] #[inline(always)] -pub unsafe fn xor3_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t { +pub unsafe fn xor3_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t { // SAFETY: `sha3` is statically enabled by the cfg gate. - unsafe { veor3q_u8(a, b, c) } + unsafe { veor3q_u64(a, b, c) } } /// Two-`EOR` fallback for targets without the SHA3 extension. @@ -31,6 +30,6 @@ pub unsafe fn xor3_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t /// No requirements; `unsafe` only to match the EOR3 arm's signature. #[cfg(not(target_feature = "sha3"))] #[inline(always)] -pub unsafe fn xor3_u8(a: uint8x16_t, b: uint8x16_t, c: uint8x16_t) -> uint8x16_t { - veorq_u8(a, veorq_u8(b, c)) +pub unsafe fn xor3_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> uint64x2_t { + veorq_u64(a, veorq_u64(b, c)) } diff --git a/crates/primitives/src/field/phi8.rs b/crates/primitives/src/field/phi8.rs deleted file mode 100644 index 1af2cb92..00000000 --- a/crates/primitives/src/field/phi8.rs +++ /dev/null @@ -1,86 +0,0 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. -// Copyright 2025 The Binius Developers -// Copyright 2025 Irreducible, Inc. -// Modifications copyright 2026 Succinct Labs, Benedikt Bunz, William Wang -// SPDX-License-Identifier: Apache-2.0 OR MIT -// -// The eight `PHI_8_BASIS` values come from binius64's `PHI_8` table -// (https://github.com/binius-zk/binius64, `crates/field/src/ghash.rs`). - -//! φ₈: GF(2⁸) → GF(2¹²⁸)-GHASH subfield embedding. -//! -//! The embedding is GF(2)-linear, so the 256-entry [`PHI_8_TABLE`] is generated -//! at compile time from the images of the eight polynomial-basis vectors. The -//! basis images were extracted from binius64's `crates/field/src/ghash.rs` and -//! are checked here against the homomorphism property -//! `phi8(a*b) = phi8(a)*phi8(b)`. - -use super::{F8, F128}; - -const PHI_8_BASIS: [F128; 8] = [ - F128::new(0x0000_0000_0000_0001, 0x0000_0000_0000_0000), - F128::new(0x6b83_3048_3c2e_9849, 0x0dcb_3646_40a2_22fe), - F128::new(0x7573_da4a_5f77_10ed, 0x3d5b_d35c_9464_6a24), - F128::new(0x41a1_2db1_f974_f3ac, 0x6d58_c4e1_81f9_199f), - F128::new(0x5e2f_716f_4ede_412f, 0xa72e_c177_64d7_ced5), - F128::new(0x5cb1_0fba_bcf0_0118, 0x4d52_354a_3a3d_8c86), - F128::new(0x95ed_1f57_f363_2d4d, 0x553e_92e8_bc0a_e9a7), - F128::new(0x5126_25b1_f09f_a87e, 0x9325_2331_bf04_2b11), -]; - -const fn build_phi8_table() -> [F128; 256] { - let mut table = [F128::ZERO; 256]; - let mut value = 1; - while value < table.len() { - let mut image = F128::ZERO; - let mut bit = 0; - while bit < PHI_8_BASIS.len() { - if value & (1 << bit) != 0 { - image.lo ^= PHI_8_BASIS[bit].lo; - image.hi ^= PHI_8_BASIS[bit].hi; - } - bit += 1; - } - table[value] = image; - value += 1; - } - table -} - -pub static PHI_8_TABLE: [F128; 256] = build_phi8_table(); - -#[inline] -pub fn phi8(a: F8) -> F128 { - PHI_8_TABLE[a.0 as usize] -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn known_images_map_correctly() { - assert_eq!(phi8(F8::ZERO), F128::ZERO); - assert_eq!(phi8(F8::ONE), F128::ONE); - assert_eq!(phi8(F8(2)), F128::new(0x6b83_3048_3c2e_9849, 0x0dcb_3646_40a2_22fe)); - } - - #[test] - fn homomorphism_full() { - // Exhaustive check: φ(a·b) = φ(a)·φ(b) and φ(a+b) = φ(a)+φ(b) - // for all 65536 ordered pairs in F_8. - for a in 0u8..=255 { - for b in 0u8..=255 { - let fa = F8(a); - let fb = F8(b); - let lhs_mul = phi8(fa * fb); - let rhs_mul = phi8(fa) * phi8(fb); - assert_eq!(lhs_mul, rhs_mul, "mul mismatch at a={a}, b={b}"); - - let lhs_add = phi8(fa + fb); - let rhs_add = phi8(fa) + phi8(fb); - assert_eq!(lhs_add, rhs_add, "add mismatch at a={a}, b={b}"); - } - } - } -} diff --git a/crates/primitives/src/field/phi8_tower.rs b/crates/primitives/src/field/phi8_tower.rs new file mode 100644 index 00000000..90a8d01e --- /dev/null +++ b/crates/primitives/src/field/phi8_tower.rs @@ -0,0 +1,86 @@ +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +// Copyright 2025 The Binius Developers / Irreducible, Inc. +// Modifications copyright 2026 Succinct Labs, Benedikt Bunz, William Wang. +// SPDX-License-Identifier: Apache-2.0 OR MIT + +//! `φ₈: GF(2⁸) → F64`, embedded in F192. +//! +//! `im(φ₈) ⊂ F64`, hence all extension limbs above `c0` are zero. The +//! homomorphism is checked exhaustively below. + +use super::{F8, F192}; + +/// φ₈(2ᵏ) for k ∈ [0,8): the images of the GF(2⁸) polynomial basis. All in +/// `F64` (`c1 == c2 == 0`). +const PHI_8_BASIS: [u64; 8] = [ + 0x0000000000000001, + 0x033ce8beddc8a656, + 0x512620375ed2a108, + 0x0c9e636090aafc01, + 0xba4f3cd82801769c, + 0xba26e7904adb4a47, + 0x467698598926dc01, + 0x4418ae808b28bdd0, +]; + +const fn build_phi8_table_192() -> [F192; 256] { + let mut table = [F192::ZERO; 256]; + let mut value = 1; + while value < table.len() { + let mut c0 = 0u64; + let mut bit = 0; + while bit < PHI_8_BASIS.len() { + if value & (1 << bit) != 0 { + c0 ^= PHI_8_BASIS[bit]; + } + bit += 1; + } + table[value] = F192::new(c0, 0, 0); + value += 1; + } + table +} + +/// The unique GF(2^8) subfield embedded in F192. It lies in the F64 base, so +/// both higher extension coordinates are zero. +pub static PHI_8_TABLE_192: [F192; 256] = build_phi8_table_192(); + +#[inline] +pub fn phi8_192(a: F8) -> F192 { + PHI_8_TABLE_192[a.0 as usize] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn known_images() { + assert_eq!(phi8_192(F8::ZERO), F192::ZERO); + assert_eq!(phi8_192(F8::ONE), F192::ONE); + assert_eq!(phi8_192(F8(2)), F192::new(PHI_8_BASIS[1], 0, 0)); + } + + #[test] + fn images_live_in_f64() { + // The GF(2⁸) subfield of F192 lies in F64, so c1 == c2 == 0 everywhere. + for v in 0u16..256 { + let image = phi8_192(F8(v as u8)); + assert_eq!(image.c1, 0, "phi8_192({v}) escaped F64"); + assert_eq!(image.c2, 0, "phi8_192({v}) escaped F64"); + } + } + + #[test] + fn homomorphism_full() { + // Exhaustive: φ(a·b)=φ(a)·φ(b) and φ(a+b)=φ(a)+φ(b) over all 65536 pairs. + for a in 0u16..256 { + for b in 0u16..256 { + let fa = F8(a as u8); + let fb = F8(b as u8); + assert_eq!(phi8_192(fa * fb), phi8_192(fa) * phi8_192(fb), "mul at a={a}, b={b}"); + assert_eq!(phi8_192(fa + fb), phi8_192(fa) + phi8_192(fb), "add at a={a}, b={b}"); + } + } + } +} diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 884b2a3f..30ddeb92 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -1,6 +1,5 @@ -//! Shared primitives: the GF(2^128)/GF(2^8) field kernels, bit transposes, -//! multilinear helpers, the scratch buffer pool, the efficiency-core helper -//! pool, and small integer utilities. +//! Shared primitives: field kernels, bit transposes, +//! multilinear helpers, the scratch buffer pool, and small integer utilities. pub mod bits; pub mod epool; @@ -8,7 +7,7 @@ pub mod field; pub mod multilinear; pub mod scratch; -pub use field::{F128, F256Unreduced, G, g_pow, g_powers, mul_by_x, x_pow}; +pub use field::{F64, F192, G, g_pow, g_powers, x_pow}; fn format_trace_tree(tree: &tracing_forest::tree::Tree) -> Result { use tracing_forest::Formatter; @@ -31,9 +30,7 @@ fn collect_parent_percentages( let percentage = match parent_duration { None => 100.0, Some(duration) if duration.is_zero() => 0.0, - Some(duration) => { - 100.0 * span.total_duration().as_nanos() as f64 / duration.as_nanos() as f64 - } + Some(duration) => 100.0 * span.total_duration().as_nanos() as f64 / duration.as_nanos() as f64, }; percentages.push(percentage); @@ -175,9 +172,9 @@ pub fn pretty_f64(value: f64) -> String { return "0".to_string(); } - let (integer, fraction) = raw.split_once('.').map_or((raw.as_str(), None), |(integer, fraction)| { - (integer, Some(fraction)) - }); + let (integer, fraction) = raw + .split_once('.') + .map_or((raw.as_str(), None), |(integer, fraction)| (integer, Some(fraction))); let mut out = pretty_integer(integer); if let Some(fraction) = fraction { out.push('.'); @@ -262,38 +259,6 @@ pub fn log2_ceil_usize(n: usize) -> usize { usize::BITS as usize - (n - 1).leading_zeros() as usize } -/// Allocate a `Vec` of length `n` whose contents are NOT zero-initialized. -/// Caller MUST write every slot before reading it. -/// -/// Used to skip the eager zero-init of large ping-pong buffers in hot prover -/// paths (Ligerito codeword + Merkle tree, zerocheck Round-2 fold, NTT -/// scratch, lincheck packing). At m=29 the -/// zero-fill of a fresh 128 MB `vec![T::default(); n]` runs sequentially on -/// the main thread (~22 ms), which caps the parallel speedup of those phases. -/// -/// `T: Copy` ensures `T` has no Drop impl, so the leaked uninitialized -/// elements are a no-op on drop. -/// -/// # Safety contract -/// -/// Reading uninitialized memory is UB per Rust's memory model regardless of -/// whether all bit patterns are valid for `T`. Caller must ensure every slot -/// is written before any read. -// `uninit_vec` flags exactly this pattern; here it is the deliberate purpose of -// the function (the safety contract above is what makes it sound). -#[allow(clippy::uninit_vec)] -pub fn alloc_uninit_vec(n: usize) -> Vec { - let mut v: Vec = Vec::with_capacity(n); - // SAFETY: - // - capacity == n was just allocated, so set_len(n) is in bounds. - // - T: Copy implies !Drop, so leaking uninit elements is a no-op. - // - Caller upholds write-before-read. - unsafe { - v.set_len(n); - } - v -} - /// Allocate a zero-filled `Vec` through the global allocator's zeroed path. /// Large allocations can therefore start as demand-zero pages instead of /// paying an eager single-threaded fill before parallel work begins. @@ -317,7 +282,26 @@ pub unsafe fn alloc_zeroed_vec(n: usize) -> Vec { unsafe { Vec::from_raw_parts(ptr, n, n) } } -/// Cached [`perf_core_count`]. The uncached version may spawn `sysctl`; this +/// Allocate `n` slots without initializing `T` values. +pub fn alloc_uninit(n: usize) -> Vec> { + let mut values = Vec::with_capacity(n); + values.resize_with(n, std::mem::MaybeUninit::uninit); + values +} + +/// Convert a vector after every slot has been initialized. +/// +/// # Safety +/// +/// Every element of `values` must contain a valid `T`. +pub unsafe fn assume_init(values: Vec>) -> Vec { + let mut values = std::mem::ManuallyDrop::new(values); + // SAFETY: `MaybeUninit` has the same layout as `T`; the caller guarantees + // that all elements are initialized, and ManuallyDrop transfers ownership. + unsafe { Vec::from_raw_parts(values.as_mut_ptr().cast(), values.len(), values.capacity()) } +} + +/// Cached `perf_core_count`. The uncached version may spawn `sysctl`; this /// memoizes it so hot paths can cheaply ask "is the current rayon pool the /// homogeneous P-core pool?" (i.e. `current_num_threads() <= this`). #[cfg_attr(not(target_arch = "aarch64"), allow(dead_code))] // caller is aarch64-only @@ -344,7 +328,5 @@ fn perf_core_count() -> usize { return n; } } - std::thread::available_parallelism() - .map(|n| n.get()) - .unwrap_or(1) + std::thread::available_parallelism().map(|n| n.get()).unwrap_or(1) } diff --git a/crates/primitives/src/multilinear.rs b/crates/primitives/src/multilinear.rs index 852de029..43cb8251 100644 --- a/crates/primitives/src/multilinear.rs +++ b/crates/primitives/src/multilinear.rs @@ -1,39 +1,98 @@ -// `build_eq` and `lagrange_weights_naive` come from https://github.com/succinctlabs/flock (MIT OR Apache-2.0). +// CREDIT: https://github.com/succinctlabs/flock (`build_eq` and `lagrange_weights_naive`), MIT OR Apache-2.0. //! Multilinear-extension utilities: the equality polynomial, single-variable //! folding, and MLE evaluation. Truth tables are indexed little-endian (variable //! `k` is bit `k`). Sumchecks here consume variables from either end, so folding -//! and `eq`-marginalization come in a low and a high variant. +//! and `eq`-marginalization come in low and high variants. Committed data is +//! `K`-valued (`F64`) while randomness is `E`-valued (`F192`), so the first +//! fold of a committed table also lifts it into `E`. -use crate::field::F128; +use crate::field::{F64, F192, F192Unreduced}; -/// Multilinear interpolation in one variable: `lo + t·(lo+hi)`, the char-2 form of -/// `(1−t)·lo + t·hi`. +/// Multilinear interpolation in one variable over `E`: `lo + t·(lo+hi)`, the +/// char-2 form of `(1−t)·lo + t·hi`. #[inline] -pub fn interp(lo: F128, hi: F128, t: F128) -> F128 { +pub fn interp(lo: F192, hi: F192, t: F192) -> F192 { lo + t * (lo + hi) } -/// `eq(r, x) = ∏_i (1 + r_i + x_i)` — 1 at `x = r`, 0 at every other Boolean point. -pub fn eq_eval(r: &[F128], x: &[F128]) -> F128 { +/// Mixed interpolation: two `K` endpoints against an `E` parameter, one +/// `mul_base` (`lo + t·(lo+hi)` with `lo, hi ∈ K`). +#[inline] +pub fn interp_k(lo: F64, hi: F64, t: F192) -> F192 { + F192::from(lo) + t.mul_base(lo + hi) +} + +/// `eq(r, x) = ∏_i (1 + r_i + x_i)`. For Boolean `r`, this is the indicator +/// of `x = r`; for arbitrary `r`, it is the multilinear interpolation weight. +pub fn eq_eval(r: &[F192], x: &[F192]) -> F192 { debug_assert_eq!(r.len(), x.len()); - let mut acc = F128::ONE; + let mut acc = F192::ONE; for i in 0..r.len() { - acc *= F128::ONE + r[i] + x[i]; + acc *= F192::ONE + r[i] + x[i]; } acc } -/// Standard inner product of two equal-length field vectors. -#[inline] -pub fn inner_product(a: &[F128], b: &[F128]) -> F128 { - assert_eq!(a.len(), b.len()); - a.iter().zip(b).fold(F128::ZERO, |acc, (&x, &y)| acc + x * y) +/// The `eq(r, ·)` table over `n = r.len()` variables, expanded a level at a +/// time: each level writes the new high half from the low half and rewrites +/// the low half in place. +/// +/// One multiply per pair, not two. In characteristic 2, +/// `e · (1 + r) = e + e · r`, so the low child is the high child XOR the +/// parent — the second product is redundant. (This is orthogonal to batching: +/// `rk` is loop-invariant, and the scalar product still beats `F192::mul2` +/// here, 3.37 vs 3.73 ns/entry.) +/// +/// A level's pairs are independent, so levels wide enough to cover rayon's +/// dispatch are split across threads. +pub fn eq_table(r: &[F192]) -> Vec { + use rayon::prelude::*; + + let mut eq = vec![F192::ZERO; 1usize << r.len()]; + eq[0] = F192::ONE; + const PAR_THRESHOLD: usize = 1 << 12; + for (i, &rk) in r.iter().enumerate() { + let half = 1usize << i; + let (lo, hi_rest) = eq.split_at_mut(half); + let hi = &mut hi_rest[..half]; + let build_pair = |lo_x: &mut F192, hi_x: &mut F192| { + let e = *lo_x; + let high = e * rk; + *hi_x = high; + *lo_x = e + high; + }; + if half < PAR_THRESHOLD { + lo.iter_mut().zip(hi.iter_mut()).for_each(|(l, h)| build_pair(l, h)); + } else { + lo.par_iter_mut() + .zip(hi.par_iter_mut()) + .for_each(|(l, h)| build_pair(l, h)); + } + } + eq +} + +/// The mixed fold: bind the lowest variable of a `K`-table to an +/// `E`-challenge, producing the `E`-table the remaining rounds fold. One +/// `mul_base` per output entry. +pub fn fold_low_k(table: &[F64], rho: F192) -> Vec { + debug_assert_eq!(table.len() % 2, 0); + (0..table.len() / 2) + .map(|i| interp_k(table[2 * i], table[2 * i + 1], rho)) + .collect() +} + +/// Bind the highest variable of a `K`-table and lift the result into `E`. +pub fn fold_high_k(table: &[F64], rho: F192) -> Vec { + debug_assert_eq!(table.len() % 2, 0); + let half = table.len() / 2; + (0..half).map(|i| interp_k(table[i], table[i + half], rho)).collect() } /// Bind the highest free variable of `table` to `rho` in place: `table[i] = /// interp(table[i], table[i + half], rho)`. Binding from the top down leaves the /// low variables, the ones every table of a batch shares, for last. -pub fn fold_high_inplace(table: &mut Vec, rho: F128) { +pub fn fold_high_inplace(table: &mut Vec, rho: F192) { debug_assert_eq!(table.len() % 2, 0); let half = table.len() / 2; for i in 0..half { @@ -45,7 +104,7 @@ pub fn fold_high_inplace(table: &mut Vec, rho: F128) { /// Marginalize the lowest variable out of an `eq` table (in place). `eq(r_0, 0) + /// eq(r_0, 1) = 1`, so summing adjacent entries drops `r_0` with no multiplies, /// versus `2^{n-1}` to rebuild the table. -pub fn shrink_eq_low(table: &mut Vec) { +pub fn shrink_eq_low(table: &mut Vec) { let half = table.len() / 2; for i in 0..half { table[i] = table[2 * i] + table[2 * i + 1]; @@ -55,7 +114,7 @@ pub fn shrink_eq_low(table: &mut Vec) { /// Marginalize the highest variable out of an `eq` table (in place), the /// [`shrink_eq_low`] counterpart for a top-down sumcheck. -pub fn shrink_eq_high(table: &mut Vec) { +pub fn shrink_eq_high(table: &mut Vec) { let half = table.len() / 2; for i in 0..half { let hi = table[i + half]; @@ -67,13 +126,13 @@ pub fn shrink_eq_high(table: &mut Vec) { /// Lagrange evaluation: given distinct `nodes` and a polynomial's `values` there, /// evaluate the interpolant at `p`. Reads a sumcheck round's univariate (sent as /// evaluations) at the verifier's challenge. -pub fn lagrange_eval(nodes: &[F128], values: &[F128], p: F128) -> F128 { +pub fn lagrange_eval(nodes: &[F192], values: &[F192], p: F192) -> F192 { debug_assert_eq!(nodes.len(), values.len()); let n = nodes.len(); - let mut acc = F128::ZERO; + let mut acc = F192::ZERO; for i in 0..n { - let mut num = F128::ONE; - let mut den = F128::ONE; + let mut num = F192::ONE; + let mut den = F192::ONE; for k in 0..n { if k == i { continue; @@ -87,19 +146,20 @@ pub fn lagrange_eval(nodes: &[F128], values: &[F128], p: F128) -> F128 { } /// The 3 nodes {0, 1, g} at which a degree-2 sumcheck round univariate is sent -/// (the eq weight is factored out). Shared by `lean_vm::constraints` and `lean_vm::gkr`. +/// (the eq weight is factored out); `g` embedded into `E`. Shared by +/// `lean_vm::constraints` and `lean_vm::gkr`. #[inline] -pub fn tri_nodes() -> [F128; 3] { - [F128::ZERO, F128::ONE, F128::generator()] +pub fn tri_nodes() -> [F192; 3] { + [F192::ZERO, F192::ONE, F192::from(crate::field::G)] } /// The 4 nodes {0, 1, g, g²} at which a degree-3 sumcheck round univariate is sent /// WHOLE, eq weight included. Costs one field element more than [`tri_nodes`] and /// buys a verifier that reapplies nothing: `h(0) + h(1) = claim`, then interpolate. #[inline] -pub fn quad_nodes() -> [F128; 4] { - let g = F128::generator(); - [F128::ZERO, F128::ONE, g, g * g] +pub fn quad_nodes() -> [F192; 4] { + let g = F192::from(crate::field::G); + [F192::ZERO, F192::ONE, g, g * g] } /// Evaluate a degree-four eq-trick round from its four independent transcript @@ -107,14 +167,14 @@ pub fn quad_nodes() -> [F128; 4] { /// constant coefficient, and characteristic two fixes the linear coefficient. #[inline] pub fn quartic_eval_from_eq( - claim: F128, - eq_point: F128, - difference: F128, - c2: F128, - c3: F128, - c4: F128, - point: F128, -) -> F128 { + claim: F192, + eq_point: F192, + difference: F192, + c2: F192, + c3: F192, + c4: F192, + point: F192, +) -> F192 { let c0 = claim + eq_point * difference; let c1 = difference + c2 + c3 + c4; c0 + point * (c1 + point * (c2 + point * (c3 + point * c4))) @@ -122,21 +182,38 @@ pub fn quartic_eval_from_eq( /// Add two 3-coefficient sumcheck accumulators componentwise. #[inline] -pub fn add3(mut x: [F128; 3], y: [F128; 3]) -> [F128; 3] { +pub fn add3(mut x: [F192; 3], y: [F192; 3]) -> [F192; 3] { for i in 0..3 { x[i] += y[i]; } x } -/// Evaluate the MLE with truth table `table` at `point` (length `log2(len)`), -/// binding variables LSB-first. One copy, then folded in place. -pub fn mle_eval(table: &[F128], point: &[F128]) -> F128 { +/// XOR two 3-slot deferred-reduction accumulators componentwise (the unreduced +/// counterpart of [`add3`]; XOR IS addition on the unreduced parts). +#[inline] +pub fn xor3(mut x: [F192Unreduced; 3], y: [F192Unreduced; 3]) -> [F192Unreduced; 3] { + for i in 0..3 { + x[i] ^= y[i]; + } + x +} + +/// Evaluate the MLE of a `K`-valued truth table at an `E`-point (length +/// `log2(len)`), binding variables LSB-first: the first fold is mixed +/// ([`fold_low_k`]), the rest pure `E` in place. +pub fn mle_eval(table: &[F64], point: &[F192]) -> F192 { debug_assert_eq!(table.len(), 1 << point.len()); - let mut cur = table.to_vec(); + if point.is_empty() { + return F192::from(table[0]); + } + let mut cur = fold_low_k(table, point[0]); let mut len = cur.len(); - for &p in point { + for &p in &point[1..] { len /= 2; + // Deliberately scalar: the fold's mul has the loop-invariant `p` on + // one side, and pairing outputs through `F192::mul2` measures slower + // (1.75 vs 2.14 ns/output, same shape as the GKR `par_fold`). for i in 0..len { cur[i] = interp(cur[2 * i], cur[2 * i + 1], p); } @@ -144,61 +221,21 @@ pub fn mle_eval(table: &[F128], point: &[F128]) -> F128 { cur[0] } -/// Build the multilinear-eq evaluation table over `r`: -/// `table[x] = ∏_i ((1 + r_i) · (1 ⊕ bit_i(x)) + r_i · bit_i(x))` for `x ∈ {0,1}^n`, -/// where `n = r.len()`. Standard in-place power-of-two doubling. -/// -/// Each doubling level needs only ONE field multiply per pair: in -/// characteristic 2, `v · (1 + r) = v + v · r`, so the low child is the high -/// child XOR the parent. Levels are independent within themselves and -/// parallelize once they are large enough to cover rayon's dispatch. -pub fn build_eq(r: &[F128]) -> Vec { - use rayon::prelude::*; - - let n = r.len(); - // Uninit alloc — at level `i` the loop reads `t[..2^i]` (written by an - // earlier level or the `t[0] = ONE` seed) and writes `t[2^i..2^(i+1)]` - // (purely written), so every slot is written before any read. - let mut t = crate::alloc_uninit_vec::(1usize << n); - t[0] = F128::ONE; - const PAR_THRESHOLD: usize = 1 << 12; - for (i, &r_i) in r.iter().enumerate() { - let half = 1usize << i; - let (lo, hi_rest) = t.split_at_mut(half); - let hi = &mut hi_rest[..half]; - let build_pair = |lo_x: &mut F128, hi_x: &mut F128| { - let v = *lo_x; - let high = v * r_i; - *hi_x = high; - *lo_x = v + high; - }; - if half < PAR_THRESHOLD { - lo.iter_mut() - .zip(hi.iter_mut()) - .for_each(|(l, h)| build_pair(l, h)); - } else { - lo.par_iter_mut() - .zip(hi.par_iter_mut()) - .for_each(|(l, h)| build_pair(l, h)); - } - } - t -} - /// O(2^{2·k_skip}) field multiplies — one-time cost. -pub fn lagrange_weights_naive(k_skip: usize, z: F128) -> Vec { +pub fn lagrange_weights_naive(k_skip: usize, z: F192) -> Vec { + use crate::field::PHI_8_TABLE_192 as PHI_8_TABLE; let ell = 1usize << k_skip; assert!(ell <= 256, "k_skip > 8 would exceed PHI_8_TABLE"); - let mut weights = vec![F128::ZERO; ell]; + let mut weights = vec![F192::ZERO; ell]; for i in 0..ell { - let si = crate::field::phi8::PHI_8_TABLE[i]; - let mut num = F128::ONE; - let mut den = F128::ONE; + let si = PHI_8_TABLE[i]; + let mut num = F192::ONE; + let mut den = F192::ONE; for j in 0..ell { if j == i { continue; } - let sj = crate::field::phi8::PHI_8_TABLE[j]; + let sj = PHI_8_TABLE[j]; num *= z + sj; den *= si + sj; } diff --git a/crates/primitives/src/scratch.rs b/crates/primitives/src/scratch.rs index 1454a80f..5c990fa6 100644 --- a/crates/primitives/src/scratch.rs +++ b/crates/primitives/src/scratch.rs @@ -1,7 +1,7 @@ -// Credit: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. -//! Process-global pool for the prover's large transient `F128` buffers. +// CREDIT: https://github.com/succinctlabs/flock (flock-core), MIT OR Apache-2.0. +//! Process-global pool for the prover's large transient `F192` buffers. //! -//! Each prove allocates, faults in, and frees several 64–128 MB vectors +//! Each prove allocates, faults in, and frees several large F192 vectors //! (the RS codeword, the round-2 fold outputs, the multilinear tail's //! ping-pong scratch). The allocator returns such allocations to the OS on //! free (`munmap`), so every prove re-pays soft page faults on first touch @@ -12,16 +12,16 @@ //! hands out a previously-used buffer when one with enough capacity exists, //! `give` returns a buffer for later reuse. Contents are NOT cleared — //! `take` has the same write-before-read contract as -//! [`crate::alloc_uninit_vec`]. +//! a fresh zeroed allocation. //! -//! Steady-state retention is bounded by [`MAX_POOLED`] buffers (~640 MB for -//! the m = 29 prove set). Call [`clear`] to release everything to the OS, -//! e.g. after the last prove of a batch. +//! Steady-state retention is bounded by a fixed buffer count; the byte total +//! depends on the largest proof shape used by the process. Call [`clear`] to +//! release everything to the OS, e.g. after the last prove of a batch. -use crate::field::F128; +use crate::field::F192; use std::sync::Mutex; -static POOL: Mutex>> = Mutex::new(Vec::new()); +static POOL: Mutex>> = Mutex::new(Vec::new()); /// Max buffers retained. The m=29 prove cycle's distinct buffers: witness /// z/a/b, the L0 codeword, zerocheck's 2 fold outputs + 2 ping-pong @@ -34,24 +34,24 @@ static POOL: Mutex>> = Mutex::new(Vec::new()); /// buffers) — measured as a +24% open_batch regression on M4 before this. const MAX_POOLED: usize = 24; -/// Take a length-`n` `F128` vector, preferring a pooled buffer (smallest -/// capacity ≥ `n`); falls back to a fresh uninitialized allocation. +/// Take a length-`n` `F192` vector, preferring a pooled buffer (smallest +/// capacity ≥ `n`); falls back to a fresh demand-zero allocation. /// -/// Contents are UNINITIALIZED in both cases — recycled buffers hold stale -/// data from a previous use. Caller MUST write every slot before reading it -/// (same contract as [`crate::alloc_uninit_vec`]). -pub fn take_f128(n: usize) -> Vec { - if let Some(v) = try_take_f128(n) { +/// Recycled buffers hold valid but unspecified stale values. Callers overwrite +/// them before use. +pub fn take_f192(n: usize) -> Vec { + if let Some(v) = try_take_f192(n) { return v; } - crate::alloc_uninit_vec(n) + // SAFETY: zero is a valid F192 value. + unsafe { crate::alloc_zeroed_vec(n) } } -/// Pool-only variant of [`take_f128`]: returns `None` instead of falling +/// Pool-only variant of [`take_f192`]: returns `None` instead of falling /// back to a fresh allocation. Lets callers branch on warm-vs-cold (e.g. /// the commit prefault skips its page-touch thread when the pool can /// supply an already-resident buffer). -pub(crate) fn try_take_f128(n: usize) -> Option> { +pub(crate) fn try_take_f192(n: usize) -> Option> { let mut pool = POOL.lock().unwrap(); let mut best: Option = None; for (i, v) in pool.iter().enumerate() { @@ -63,9 +63,8 @@ pub(crate) fn try_take_f128(n: usize) -> Option> { let mut v = pool.swap_remove(i); drop(pool); v.clear(); - // SAFETY: capacity ≥ n was checked above; F128: Copy (no Drop), so - // exposing uninit/stale elements is sound to *hold* — the caller - // upholds write-before-read per this function's contract. + // SAFETY: capacity ≥ n was checked above. The backing storage retains + // valid F192 values from its previous use. unsafe { v.set_len(n) }; return Some(v); } @@ -76,7 +75,7 @@ pub(crate) fn try_take_f128(n: usize) -> Option> { /// smallest-capacity buffer is evicted (large buffers are the expensive ones /// to re-fault; a run that ramps problem sizes upward must not get its big /// buffers crowded out by stale small ones). -pub fn give_f128(v: Vec) { +pub fn give_f192(v: Vec) { if v.capacity() == 0 { return; } @@ -102,7 +101,7 @@ pub fn give_f128(v: Vec) { /// work: a race between fault cost and the hiding window flips sign across /// machines; eliminated work doesn't.) /// -/// The set (sizes in F128s): 5 buffers of the 2^(m-6) class (L0 codeword, +/// The set (sizes in F192s): 5 buffers of the 2^(m-6) class (L0 codeword, /// zerocheck round-2 a/b, headroom for the Ligerito recursion's largest /// level codewords) and 11 of the 2^(m-7) class (witness z/a/b, zerocheck /// tail ping-pong ×2, rs_eq_ind ×2, b_combined, the smaller Ligerito level @@ -114,23 +113,23 @@ pub fn prewarm_prover(m: usize) { } let small = 1usize << (m - 7); let large = 1usize << (m - 6); - let mut bufs: Vec> = Vec::new(); + let mut bufs: Vec> = Vec::new(); for _ in 0..5 { - bufs.push(take_f128(large)); + bufs.push(take_f192(large)); } for _ in 0..11 { - bufs.push(take_f128(small)); + bufs.push(take_f192(small)); } // First-touch every page of every buffer, all cores. Already-resident // (re-warmed) buffers cost a fast memset; fresh ones fault here, once. bufs.par_iter_mut().for_each(|b| { b.par_chunks_mut(1 << 16).for_each(|chunk| { - // SAFETY: F128 is plain bytes (no Drop); zero is a valid pattern. + // SAFETY: F192 is plain bytes (no Drop); zero is a valid pattern. unsafe { std::ptr::write_bytes(chunk.as_mut_ptr(), 0u8, chunk.len()) } }); }); for b in bufs { - give_f128(b); + give_f192(b); } } @@ -147,14 +146,14 @@ mod tests { #[test] fn take_reuses_given_buffer() { clear(); - let mut v = take_f128(1024); + let mut v = take_f192(1024); for slot in v.iter_mut() { - *slot = F128 { lo: 7, hi: 9 }; + *slot = F192::new(7, 9, 0); } let ptr = v.as_ptr(); - give_f128(v); + give_f192(v); // Same capacity request gets the same allocation back. - let v2 = take_f128(512); + let v2 = take_f192(512); assert_eq!(v2.as_ptr(), ptr); assert_eq!(v2.len(), 512); clear(); @@ -164,9 +163,9 @@ mod tests { fn pool_is_bounded() { clear(); for _ in 0..(MAX_POOLED + 4) { - give_f128(take_f128(16)); + give_f192(take_f192(16)); } assert!(POOL.lock().unwrap().len() <= MAX_POOLED); clear(); } -} \ No newline at end of file +} diff --git a/crates/primitives/tests/eq_table_identity.rs b/crates/primitives/tests/eq_table_identity.rs new file mode 100644 index 00000000..901db8c8 --- /dev/null +++ b/crates/primitives/tests/eq_table_identity.rs @@ -0,0 +1,79 @@ +//! `eq_table` feeds the verifier (via the lean_vm constraint and GKR paths and +//! flock's univariate skip), so its rewrite must be bit-identical to the +//! two-multiply form, not merely algebraically equal. +//! +//! Old: `hi = e · r`, `lo = e · (1 + r)` — two products. +//! New: `hi = e · r`, `lo = e + e · r` — one product. +//! +//! These agree by distributivity in characteristic 2, and F192 stores a +//! canonical reduced `(c0, c1, c2)`, so the bit patterns must match exactly. +use primitives::field::F192; + +/// Verbatim copy of the pre-change implementation. +fn eq_table_old(r: &[F192]) -> Vec { + let mut eq = vec![F192::ZERO; 1usize << r.len()]; + eq[0] = F192::ONE; + let mut half = 1usize; + for &rk in r { + let one_plus = F192::ONE + rk; + for i in (0..half).rev() { + let e = eq[i]; + eq[i + half] = e * rk; + eq[i] = e * one_plus; + } + half <<= 1; + } + eq +} + +struct Rng(u64); +impl Rng { + fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + fn f192(&mut self) -> F192 { + F192 { + c0: self.next_u64(), + c1: self.next_u64(), + c2: self.next_u64(), + } + } +} + +#[test] +fn eq_table_is_bit_identical_to_two_multiply_form() { + let mut rng = Rng(0xE0_1D_5E_ED); + for n in [0usize, 1, 2, 3, 6, 7, 11, 12, 13, 14] { + for _ in 0..4 { + let r: Vec = (0..n).map(|_| rng.f192()).collect(); + let want = eq_table_old(&r); + let got = primitives::multilinear::eq_table(&r); + assert_eq!(got.len(), want.len(), "length differs at n={n}"); + for (x, (g, w)) in got.iter().zip(want.iter()).enumerate() { + assert_eq!(g, w, "bit mismatch at n={n}, x={x}"); + } + } + } +} + +/// Boolean points are where a representation difference would most likely +/// surface, since `1 + r` collapses to 0 or 1 there. +#[test] +fn eq_table_is_bit_identical_at_boolean_points() { + for n in 1..=10usize { + for mask in 0..(1usize << n) { + let r: Vec = (0..n) + .map(|i| if (mask >> i) & 1 == 1 { F192::ONE } else { F192::ZERO }) + .collect(); + assert_eq!( + primitives::multilinear::eq_table(&r), + eq_table_old(&r), + "bit mismatch at n={n}, mask={mask}" + ); + } + } +} diff --git a/crates/rec_aggregation/Cargo.toml b/crates/rec_aggregation/Cargo.toml index 1df5e346..2c19d2ed 100644 --- a/crates/rec_aggregation/Cargo.toml +++ b/crates/rec_aggregation/Cargo.toml @@ -18,3 +18,5 @@ bincode.workspace = true serde.workspace = true tracing.workspace = true blake3.workspace = true + +[dev-dependencies] diff --git a/crates/rec_aggregation/guests/recursion.py b/crates/rec_aggregation/guests/recursion.py index 232819e8..c2b70843 100644 --- a/crates/rec_aggregation/guests/recursion.py +++ b/crates/rec_aggregation/guests/recursion.py @@ -4,7 +4,7 @@ # prefix the shape dictates); binding always comes from the per-word absorbs. STREAM_CAP = STREAM_CAP_PLACEHOLDER # Per-table tau floor: BLAKE3 is sized to flock's instance count (>= 2^3). -FLOORS = [0, 0, 0, 0, 0, 3] +FLOORS = [0, 0, 0, 0, 0, 3, 0] MIN_LOG_MEM = MIN_LOG_MEM_PLACEHOLDER INV_GEN = INV_GEN_PLACEHOLDER LAGRANGE_INV_0 = LAGRANGE_INV_0_PLACEHOLDER @@ -28,9 +28,6 @@ GKR_ROUNDS_CAP = GKR_ROUNDS_CAP_PLACEHOLDER MU_CAP = MU_CAP_PLACEHOLDER GKR_POINTS_CAP = GKR_POINTS_CAP_PLACEHOLDER -# The bus PoW window is g^(push.mu - BUS_GRIND_SHIFT), BUS_GRIND_SHIFT = -# 127 - SECURITY_BITS (see leaf::grand_product_grinding_bits). -BUS_GRIND_SHIFT = BUS_GRIND_SHIFT_PLACEHOLDER # Bus blocks, flattened across the 3 sides (side s covers blocks # [SIDE_BLOCK_START[s], SIDE_BLOCK_START[s+1])). The block STRUCTURE is @@ -51,7 +48,7 @@ # REAL_IS_FULL_CUBE for the framework blocks (real = 2^kappa, no padding). It is # also what marks a block as owned: an owned block's fingerprint is settled by the # batched zerocheck, off its table's column evaluations. -REAL_IS_FULL_CUBE = 6 +REAL_IS_FULL_CUBE = REAL_IS_FULL_CUBE_PLACEHOLDER SIDE_BLOCK_START = SIDE_BLOCK_START_PLACEHOLDER N_BLOCKS = N_BLOCKS_PLACEHOLDER BLOCK_KAPPA_SRC = BLOCK_KAPPA_SRC_PLACEHOLDER @@ -102,6 +99,7 @@ TABLE_DEREF = 3 TABLE_JUMP = 4 TABLE_BLAKE3 = 5 +TABLE_PACK64X2 = 6 N_TABLES = N_TABLES_PLACEHOLDER # Phase D (flock reduction): the seven fixed inner challenges (+ inverses of 1+c), # the phi8 node table + baked Lagrange inverse denominators (Lambda domain, @@ -116,20 +114,32 @@ FIXED_CHALLENGES = FIXED_CHALLENGES_PLACEHOLDER ONE_PLUS_CHALLENGE_INV = ONE_PLUS_CHALLENGE_INV_PLACEHOLDER PHI8_NODES = PHI8_NODES_PLACEHOLDER +# Tower F192 = F64[Y]/(Y^3+Y+1). Y_TOWER embeds Y for reassembling +# e192(lo,hi,top)=lo+hi*Y+top*Y². Y_INV is also used at the opening boundary +# to deduce the top-limb evaluation after the low and high limbs are transmitted. +Y_TOWER = Y_TOWER_PLACEHOLDER +Y_INV = Y_INV_PLACEHOLDER +# Coordinate basis e_i of F192 (spans the whole field). hint_decompose_bits +# emits a word's coordinate bits, so a value reconstructs as Σ b_i·COORD_BASIS[i] +# = v. (NOT the g-power basis GEN**i, which spans only F64 in the tower.) +COORD_BASIS = COORD_BASIS_PLACEHOLDER LAGRANGE_INV_LAMBDA = LAGRANGE_INV_LAMBDA_PLACEHOLDER LAGRANGE_INV_COMBINED = LAGRANGE_INV_COMBINED_PLACEHOLDER LAGRANGE_INV_S = LAGRANGE_INV_S_PLACEHOLDER LINCHECK_ROUNDS = LINCHECK_ROUNDS_PLACEHOLDER PIN_COLUMN = PIN_COLUMN_PLACEHOLDER K_LOG = K_LOG_PLACEHOLDER -# Phase E: the stacked mixed opening. The two ring-switch fronts (claim check, -# tensor transpose, and eval_rs_eq all in-circuit), followed by the +SLOT_STRIDE_LOG = SLOT_STRIDE_LOG_PLACEHOLDER # = K_LOG - LOG_PACKING (=8); the q_pkd slot stride +# Phase E: the stacked mixed opening. The two ring-switch fronts +# (claim check in-circuit; the tensor transpose + eval_rs_eq DEFERRED); the # gamma-combination of the two ring-switch claims and the N_CLAIMS pool claims. # Phase E2: the Ligerito opening over the stacked commitment, dispatched by # the certified committed log-size m through match_range: the LIG_* tables -# below carry one row per candidate m in [LIG_MIN_LOG_SIZE, +LIG_N_CANDIDATES), +# below carry one row per (rate, m), with rate in 1..=4 and m in the +# supported committed-size interval, # emitted from the SAME derive_profile/level_shapes the prover uses. -# Scalars index as TBL[m_idx]; per-level values as TBL[m_idx * LIG_MAX_LEVELS + lvl]; +# Scalars index as TBL[m_idx]; per-level values as TBL[m_idx * LIG_MAX_LEVELS + lvl], +# where m_idx is the flattened (rate, size) configuration index; # per-fold grind schedules with the LIG_MAX_TOTAL_FOLDS stride; the subspace # vanishing constants with the LIG_MAX_VANISH_LEN stride. The eval_b terminal # claim descriptors keep the fixed point-buffer and committed-column identities @@ -137,6 +147,8 @@ # the native largest-first committed-column placement reconstructed in-circuit. # Opening dispatch: baked committed log-size, candidate range, g^-LIG_MIN_LOG_SIZE. LIG_MIN_LOG_SIZE = LIG_MIN_LOG_SIZE_PLACEHOLDER +LIG_N_LOG_SIZES = LIG_N_LOG_SIZES_PLACEHOLDER +LIG_N_RATES = LIG_N_RATES_PLACEHOLDER # Committed-column kappa sources (0 = const COL_KAPPA_ADJ, 1 = log_mem, 2+t = tau_t) # and the PCS floor for the stacked size. N_COMMITTED_COLS = N_COMMITTED_COLS_PLACEHOLDER @@ -147,6 +159,7 @@ LIG_MAX_LEVELS = LIG_MAX_LEVELS_PLACEHOLDER LIG_MAX_TOTAL_FOLDS = LIG_MAX_TOTAL_FOLDS_PLACEHOLDER LIG_MAX_VANISH_LEN = LIG_MAX_VANISH_LEN_PLACEHOLDER +LIG_MAX_OOD_SAMPLES = LIG_MAX_OOD_SAMPLES_PLACEHOLDER # Global maxima (StackBuf frame sizes are parse-time). LIG_LOG_MSG_COLS_CAP = LIG_LOG_MSG_COLS_CAP_PLACEHOLDER YR_LOG_CAP = YR_LOG_CAP_PLACEHOLDER @@ -163,13 +176,14 @@ LIG_SUMCHECK_LEN = LIG_SUMCHECK_LEN_PLACEHOLDER LIG_ROWS_LEN = LIG_ROWS_LEN_PLACEHOLDER LIG_PATHS_LEN = LIG_PATHS_LEN_PLACEHOLDER -LIG_FOLD_GRIND_LEN = LIG_FOLD_GRIND_LEN_PLACEHOLDER LIG_QUERY_GRIND_BITS = LIG_QUERY_GRIND_BITS_PLACEHOLDER +LIG_OOD_SAMPLES = LIG_OOD_SAMPLES_PLACEHOLDER LIG_QUERIES = LIG_QUERIES_PLACEHOLDER LIG_FOLDS = LIG_FOLDS_PLACEHOLDER LIG_INTERLEAVE = LIG_INTERLEAVE_PLACEHOLDER LIG_LEAF_PAIRS = LIG_LEAF_PAIRS_PLACEHOLDER LIG_LEAF_BLOCKS = LIG_LEAF_BLOCKS_PLACEHOLDER +LIG_PACKED_ROW_CAP = LIG_PACKED_ROW_CAP_PLACEHOLDER LIG_TREE_DEPTH = LIG_TREE_DEPTH_PLACEHOLDER LIG_SQUEEZES = LIG_SQUEEZES_PLACEHOLDER LIG_POSITIONS_OFF = LIG_POSITIONS_OFF_PLACEHOLDER @@ -189,7 +203,7 @@ # eval_b claim descriptors (fixed parts) + the qpkd capacity stride. # CLAIM_COMMITTED_COL maps each pooled logical claim to the compact index of the # committed column it must open. Virtual BLAKE3 value claims map to QPKD. -# CLAIM_QPKD_SLOT_BITS contains the fixed seven VM-slot bits for every logical +# CLAIM_QPKD_SLOT_BITS contains the fixed packed-slot bits for every logical # claim (zero for non-virtual claims), and QPKD_COMMITTED_COL identifies the # ring-switch target. # Which point buffer a pooled claim's x-part lives in (CLAIM_POINT_BUF codes): @@ -197,20 +211,16 @@ POINT_BUF_RHO = 1 POINT_BUF_PI = 2 POINT_BUF_QPKD = 3 +POINT_BUF_QPKD_RHO = 4 CLAIM_POINT_BUF = CLAIM_POINT_BUF_PLACEHOLDER CLAIM_COMMITTED_COL = CLAIM_COMMITTED_COL_PLACEHOLDER CLAIM_QPKD_SLOT_BITS = CLAIM_QPKD_SLOT_BITS_PLACEHOLDER QPKD_COMMITTED_COL = QPKD_COMMITTED_COL_PLACEHOLDER QPKD_VARS_CAP = QPKD_VARS_CAP_PLACEHOLDER -# Ring-switch coefficient factorization for the GHASH power basis. Each -# 15-value row contains d^(-2^k)*x^(127*2^k), then -# 1+x^(-2^(k+t)) for t=0..6, then d^(-2^k)*epsilon_i^(2^k) for i=0..6. -RS_COEFF_ORBIT_WIDTH = 15 -RS_COEFF_ORBITS = RS_COEFF_ORBITS_PLACEHOLDER # Phase F: log rows of the bytecode blocks (the deferred bytecode points). BYTECODE_LOG = BYTECODE_LOG_PLACEHOLDER -# One sub-proof's deferred-claim region: 2*BYTECODE_LOG + LOG2_BYTECODE_COLS -# + 2*LINCHECK_ROUNDS + 69 words (see verify_sub's defer_out layout). +# One sub-proof's deferred-claim region: one bytecode point and the Flock +# lincheck data (see verify_sub's defer_out layout). DEFER_SIZE = DEFER_SIZE_PLACEHOLDER # Aggregation: NSUB sub-proofs of the same program; per-sub proof data arrives # as hints. The seed sponge state after the two byte-string absorbs is baked @@ -234,35 +244,103 @@ DS_SQ = 4 DS_POW = 5 -# Field structure: GF(2^128). Its 128 bits pack into LOG2_FIELD_BITS = 7 -# ring-switch coordinates (the q_pkd slot length, r'' length). -FIELD_BITS = 128 -LOG2_FIELD_BITS = 7 -# Exponent bit-widths: an announced count uses COUNT_BITS bits internally, with -# its top bit constrained to zero so the native strict 32-bit bound holds; any -# structural size (sums of 2^kappa, packing offsets) fits SIZE_BITS bits. +# Field structure: GF(2^192), represented as three GF(2^64) tower limbs. +# Six challenges define the F2-linear map that batches the 192 transposed +# ring-switch coordinates. +FIELD_BITS = 192 +BASE_FIELD_BITS = 64 +RING_MAP_SHIFTS = [32, 16, 8, 4, 2, 1] +# Exponent bit-widths: an announced 32-bit count decomposes into COUNT_BITS +# bits, with its top bit constrained to zero so the native strict 32-bit bound +# holds; any structural size (sums of 2^kappa, packing offsets) fits SIZE_BITS +# bits. COUNT_BITS = 33 SIZE_BITS = 34 +@inline +def f192_from_limbs(c0, c1, c2): + # Horner form saves one multiplication over c0 + c1*Y + c2*Y^2. + return c0 + Y_TOWER * (c1 + Y_TOWER * c2) + + +@inline +def challenge_from_state(state): + # Exact lowering for state = [(d0,d1,0), (d2,d3,0)]: + # hints lo=[d0,d1], hi=[d2,d3] + # PACK64X2(lo[0],lo[1]) -> state[0] (in-place equality check) + # PACK64X2(hi[0],hi[1]) -> state[1] (in-place equality check) + # return d0 + d1*Y + d2*Y^2 + # Write-once memory makes both PACK rows assertions, so all four digest + # lanes are proven to be in K; d3 is constrained but not used. + lo = StackBuf(2) + hi = StackBuf(2) + hint_f192_limbs(lo, state[0]) + hint_f192_limbs(hi, state[1]) + pack64x2_into(lo[0], lo[1], state[0]) + pack64x2_into(hi[0], hi[1], state[1]) + return f192_from_limbs(lo[0], lo[1], hi[0]) + + +@inline +def sponge_compress(state, scalar, tail, out): + # Serialize scalar.c0, scalar.c1, scalar.c2, tail as two canonical cells. + # The hints only provide the decomposition; PACK64X2 proves all four lanes + # are in K, and the equality binds the first three back to scalar. + limbs = StackBuf(3) + hint_f192_limbs(limbs, scalar) + block = StackBuf(2) + pack64x2_into(limbs[0], limbs[1], block[0]) + pack64x2_into(limbs[2], tail, block[1]) + assert scalar == f192_from_limbs(limbs[0], limbs[1], limbs[2]) + blake3(state, block, out) + return + + +@inline +def hash_state_to_words(cell_0, cell_1): + # Convert canonical BLAKE3 cells (d0,d1,0), (d2,d3,0) to the two scalar + # words used when Ligerito observes a Merkle root: (d0,d1,d2), (d3,0,0). + lo = StackBuf(2) + hi = StackBuf(2) + hint_f192_limbs(lo, cell_0) + hint_f192_limbs(hi, cell_1) + pack64x2_into(lo[0], lo[1], cell_0) + pack64x2_into(hi[0], hi[1], cell_1) + return f192_from_limbs(lo[0], lo[1], hi[0]), hi[1] + + +@inline +def hash_words_to_state(word_0, word_1): + # Inverse of hash_state_to_words. PACK64X2 also proves word_1 is in K. + limbs = StackBuf(3) + hint_f192_limbs(limbs, word_0) + state = StackBuf(2) + pack64x2_into(limbs[0], limbs[1], state[0]) + pack64x2_into(limbs[2], word_1, state[1]) + assert word_0 == f192_from_limbs(limbs[0], limbs[1], limbs[2]) + return state + + def squeeze_step(state_0, state_1): # Non-inlined sponge ratchet exposing BOTH output words (challenge and the # next state), so a query-squeeze loop can chain the state through a heap # buffer. Returns (challenge, next_state_0, next_state_1). a = [state_0, state_1] - b = [0, DS_SQ] o = StackBuf(2) - blake3(a, b, o) - return o[0], o[0], o[1] + sponge_compress(a, f192_from_limbs(0, 0, DS_SQ), 0, o) + challenge = challenge_from_state(o) + return challenge, o[0], o[1] -def check_128_bits_decomposition(bits_ptr, v): - # Boolean-constrain FIELD_BITS hinted bits and assert they reconstruct v. +def check_field_bits_decomposition(bits_ptr, v): + # Boolean-constrain FIELD_BITS hinted bits and assert they reconstruct v in + # the F192 COORDINATE basis (hint_decompose_bits emits coordinate bits). acc = 0 for i in unroll(0, FIELD_BITS): b = bits_ptr[GEN ** i] assert b * b == b - acc += b * GEN ** i # accumulate the g-power encoding: bit i contributes g^i + acc += b * COORD_BASIS[i] # bit i contributes the i-th coordinate basis vector assert acc == v return @@ -271,35 +349,56 @@ def decode_query_bits(v, positions_out, bit_ptrs_out, depth: Const): # The squeezed word's bits are advice-decomposed HERE, boolean-constrained, # and tied back by reconstruction; each depth-bit group also becomes a query # position (little-endian), with a pointer to its bit run (the Merkle - # direction bits). Each 128-bit word packs FIELD_BITS // depth positions. + # direction bits). Each field word packs FIELD_BITS // depth positions. per_word = FIELD_BITS // depth bits_ptr = HeapBuf(GEN ** FIELD_BITS) hint_decompose_bits(bits_ptr, v, FIELD_BITS) acc = 0 for j in unroll(0, per_word): - position = 0 + base_bit = j * depth # this group's first coordinate of v + # A group that stays inside one 64-bit limb shifts as a WHOLE: the + # coordinate basis is the polynomial basis there, so + # COORD_BASIS[base_bit + b] == COORD_BASIS[base_bit] * COORD_BASIS[b] + # (exponents below 64, no reduction). The group's contribution to the + # reconstruction is then one multiply by the position value it already + # forms, instead of a constant multiply per bit. A group straddling the + # boundary splits into the two runs that do stay inside a limb. + cut = 64 - base_bit % 64 # bits of this group below the next limb + p_lo = 0 + p_hi = 0 for b in unroll(0, depth): - t = bits_ptr[GEN ** (j * depth + b)] - sq = t * t - assert sq == t - position += t * GEN ** b - positions_out[GEN ** j] = position - bit_ptrs_out[GEN ** j] = bits_ptr * GEN ** (j * depth) - acc += position * GEN ** (j * depth) + t = bits_ptr[GEN ** (base_bit + b)] + # Booleanity as a write-once pin: the cell already holds t, so + # storing t*t back IS the assert t*t == t, one instruction shorter + # (a Cell deref unifies the two sides). + bits_ptr[GEN ** (base_bit + b)] = t * t + # `b // cut == 0` IS `b < cut`, in compile-time integer arithmetic + # (the DSL's `if` compares for equality only). + if b // cut == 0: + p_lo += t * COORD_BASIS[b] + else: + p_hi += t * COORD_BASIS[b - cut] + # position = p_lo + 2^cut * p_hi: multiplying by X^cut concatenates the + # two runs, since both degrees stay below 64. + if cut // depth == 0: # `cut < depth`: this group straddles the boundary + positions_out[GEN ** j] = p_lo + COORD_BASIS[cut] * p_hi + acc += COORD_BASIS[base_bit] * p_lo + COORD_BASIS[base_bit + cut] * p_hi + else: + positions_out[GEN ** j] = p_lo + acc += COORD_BASIS[base_bit] * p_lo + bit_ptrs_out[GEN ** j] = bits_ptr * GEN ** base_bit for i in unroll(per_word * depth, FIELD_BITS): t = bits_ptr[GEN ** i] - sq = t * t - assert sq == t - acc += t * GEN ** i + bits_ptr[GEN ** i] = t * t + acc += t * COORD_BASIS[i] assert acc == v return def grind_check(state_0, state_1, nonce, nbits_g): - # The one grinding check, shared by the bus grind and the Ligerito fold / - # query grinds: digest = H(H(state, (0, POW)), (nonce, POW)); the digest's + # Ligerito fold/query grinding: digest = H(H(state, (0, POW)), (nonce, POW)); the digest's # bits are advice-decomposed HERE and verified (booleanity + reconstruction, - # check_128_bits_decomposition), and the low nbits (nbits_g = g^nbits) must + # check_field_bits_decomposition), and the low nbits (nbits_g = g^nbits) must # be zero — the CONTIGUOUS PoW window of transcript::pow_bits_ok. The # caller absorbs the full field nonce afterwards. The honest prover searches # the deterministic u64 subset, while verification permits the full field: @@ -307,15 +406,14 @@ def grind_check(state_0, state_1, nonce, nbits_g): if nbits_g == GEN ** 0: assert nonce == 0 # native canonical zero-work nonce st = [state_0, state_1] - tag = [0, DS_POW] base = StackBuf(2) - blake3(st, tag, base) - nz = [nonce, DS_POW] + sponge_compress(st, f192_from_limbs(0, 0, DS_POW), 0, base) out = StackBuf(2) - blake3(base, nz, out) + # nonce's three F64 limbs followed by DS_POW, exactly as the native sponge. + sponge_compress(base, nonce, DS_POW, out) digest_bits = HeapBuf(GEN ** FIELD_BITS) hint_decompose_bits(digest_bits, out[0], FIELD_BITS) - check_128_bits_decomposition(digest_bits, out[0]) + check_field_bits_decomposition(digest_bits, out[0]) for xb in mul_range(1, nbits_g): assert digest_bits[xb] == 0 return @@ -396,6 +494,7 @@ def log2_ceil_in_the_exponent(g_N, g_logs_pow2, g_squares, floor: Const, nbits: return g_log +@inline def verify_merkle_path(leaf_0, leaf_1, path_ptr, direction_bits, depth: Const): node_0 = leaf_0 node_1 = leaf_1 @@ -407,7 +506,7 @@ def verify_merkle_path(leaf_0, leaf_1, path_ptr, direction_bits, depth: Const): diff_1 = node_1 + sibling_1 left = [node_0 + dir_bit * diff_0, node_1 + dir_bit * diff_1] right = [diff_0 + left[0], diff_1 + left[1]] - parent = StackBuf(2) # parent = blake3(left, right), the running node one level up + parent = StackBuf(2) blake3(left, right, parent) node_0 = parent[0] node_1 = parent[1] @@ -426,12 +525,11 @@ def sumcheck_round3(state_0, state_1, msg_cursor, claim, eq_acc, prev_challenge) fs, m2, msg_cursor = fs_next(fs, msg_cursor) lhs = eq_acc * ((1 + prev_challenge) * m0 + prev_challenge * m1) assert lhs == claim - fs = squeeze(fs) - round_challenge = fs[0] + fs, round_challenge = squeeze(fs) new_eq = eq_acc * (1 + prev_challenge + round_challenge) - l0 = (round_challenge + 1) * (round_challenge + GEN) * LAGRANGE_INV_0 - l1 = round_challenge * (round_challenge + GEN) * LAGRANGE_INV_1 - l2 = round_challenge * (round_challenge + 1) * LAGRANGE_INV_2 + l0 = ((round_challenge + 1) * (round_challenge + GEN)) * LAGRANGE_INV_0 + l1 = (round_challenge * (round_challenge + GEN)) * LAGRANGE_INV_1 + l2 = (round_challenge * (round_challenge + 1)) * LAGRANGE_INV_2 new_claim = new_eq * (m0 * l0 + m1 * l1 + m2 * l2) return fs[0], fs[1], msg_cursor, new_claim, new_eq, round_challenge @@ -449,8 +547,7 @@ def sumcheck_round5(state_0, state_1, msg_cursor, claim, prev_challenge): fs, c2, msg_cursor = fs_next(fs, msg_cursor) fs, c3, msg_cursor = fs_next(fs, msg_cursor) fs, c4, msg_cursor = fs_next(fs, msg_cursor) - fs = squeeze(fs) - round_challenge = fs[0] + fs, round_challenge = squeeze(fs) new_claim = quartic_eval_from_eq(claim, prev_challenge, difference, c2, c3, c4, round_challenge) return fs[0], fs[1], msg_cursor, new_claim, round_challenge @@ -467,8 +564,7 @@ def sumcheck_round4(state_0, state_1, msg_cursor, claim): fs, h2, msg_cursor = fs_next(fs, msg_cursor) fs, h3, msg_cursor = fs_next(fs, msg_cursor) assert h0 + h1 == claim - fs = squeeze(fs) - y = fs[0] + fs, y = squeeze(fs) l0 = (y + 1) * (y + GEN) * (y + GEN * GEN) * LAG4_INV_0 l1 = y * (y + GEN) * (y + GEN * GEN) * LAG4_INV_1 l2 = y * (y + 1) * (y + GEN * GEN) * LAG4_INV_2 @@ -498,9 +594,8 @@ def fold_final_msg(msg, weights, wbase: Const, log_len: Const): def obs(state, x): # Bind one scalar into the sponge chain: state <- compress(state, (x, SCALAR)). # Returns the successor StackBuf; the call site aliases it (zero copies). - tg = [x, DS_SCALAR] nb = StackBuf(2) - blake3(state, tg, nb) + sponge_compress(state, x, DS_SCALAR, nb) return nb @@ -514,28 +609,27 @@ def fs_next(state, cursor): # cost (state a StackBuf run, cursor a folded g-address), so the usual walk is # just `fs, x, cursor = fs_next(fs, cursor)` with no manual cursor arithmetic. x = cursor[GEN ** 0] - tg = [x, DS_SCALAR] nb = StackBuf(2) - blake3(state, tg, nb) + sponge_compress(state, x, DS_SCALAR, nb) return nb, x, cursor * GEN @inline -def absorb(state, x, tag): - # Tagged absorb (length frames, byte words, grinding nonces). - tg = [x, tag] +def absorb_nonce(state, x): + # Full-field grinding nonce absorb: [x.c0, x.c1, x.c2, DS_POW]. nb = StackBuf(2) - blake3(state, tg, nb) + sponge_compress(state, x, DS_POW, nb) return nb @inline def squeeze(state): - # Ratchet: the compress output is the new state; word 0 is the challenge. - zt = [0, DS_SQ] + # Ratchet: the canonical 128+128 digest is the new state; its first three + # K lanes are reassembled as the F192 challenge. nb = StackBuf(2) - blake3(state, zt, nb) - return nb + sponge_compress(state, f192_from_limbs(0, 0, DS_SQ), 0, nb) + challenge = challenge_from_state(nb) + return nb, challenge @inline @@ -596,9 +690,9 @@ def eqtree(point_ptr, out, n_coords: Const): def open_stacked(m_idx: Const, fs0, fs1, target, commit_root_0, commit_root_1, cursor): - # The stacked Ligerito opening. m_idx is the COMMITTED-LOG-SIZE CANDIDATE - # INDEX: the certified size is m = LIG_MIN_LOG_SIZE + m_idx, and every - # LIG_* table below reads row m_idx (the match_range dispatch bakes one + # The stacked Ligerito opening. m_idx is the flattened (rate, committed + # log-size) configuration index, and every LIG_* table below reads row + # m_idx (the match_range dispatch bakes one # specialization of this function per candidate). All opening proof data is hinted HERE, so # hint lengths specialize per arm; only the executed arm pops its streams. # @@ -612,22 +706,27 @@ def open_stacked(m_idx: Const, fs0, fs1, target, commit_root_0, commit_root_1, c # alpha-batched row dot against the fold eq weights, and verify the # Merkle authentication path against the bound root # (verify_merkle_path); - # 5. sample beta, fold the query sums into the running target. - # Then the per-level residuals (novel-basis prefix x final-message fold) - # are combined; the caller's eval_b terminal asserts the grand total. + # 5. read the level's intro message, sample beta, and fold the query sum + # into the running target. + # Then finish the tail sumcheck and evaluate every transparent basis once + # at its terminal point; the final-message MLE enters as one multiplier. # # Returns (sumcheck_target, fold_challenges, final_msg, residual_total, # yr_log_n_g = g^yr_log_n, yr_pad_g = g^(YR_LOG_CAP - yr_log_n), - # fold_cap_g = g^lenris). yr_log_n_g/yr_pad_g let the terminal zero-pin + # fold_cap_g = g^lenris), tail_challenges, and yr_at_tail. + # yr_log_n_g/yr_pad_g let the terminal zero-pin # residual-slot coordinates beyond final_msg's 2^yr_log_n cells (positions # yr_log_n .. YR_LOG_CAP-1); fold_cap_g is the certified total fold count # the terminal pins its hinted claim lengths against. fs = [fs0, fs1] + # The K opener binds the initial Merkle root as its two F192 scalars (like + # the extension-field opener's add_scalars(hash_to_scalars(root))), not as a byte + # string. Level roots are likewise scalar-observed (via fs_next below). + commit_root_word_0, commit_root_word_1 = hash_state_to_words(commit_root_0, commit_root_1) fs = obs(fs, target) - fs = absorb(fs, 32, DS_LEN) - fs = absorb(fs, commit_root_0, DS_BYTE) - fs = absorb(fs, commit_root_1, DS_BYTE) + fs = obs(fs, commit_root_word_0) + fs = obs(fs, commit_root_word_1) # The opening's scalars (sumcheck messages, level roots, nonces, final # message) ride the SHARED stream: msg_cursor is just the main stream @@ -658,6 +757,10 @@ def open_stacked(m_idx: Const, fs0, fs1, target, commit_root_0, commit_root_1, c alpha_weights = HeapBuf(GEN ** (LIG_N_LEVELS[m_idx] * LIG_MAX_QUERIES[m_idx])) query_positions = HeapBuf(GEN ** (LIG_POSITIONS_LEN[m_idx])) query_bit_ptrs = HeapBuf(GEN ** (LIG_POSITIONS_LEN[m_idx])) + # Explicit OOD claims bind every recursive Johnson-list commitment. L0 + # needs none: the opening claim itself is its post-commit binding value. + ood_z = HeapBuf(GEN ** (LIG_N_LEVELS[m_idx] * LIG_MAX_OOD_SAMPLES * LIG_LOG_MSG_COLS_CAP)) + ood_betas = HeapBuf(GEN ** (LIG_N_LEVELS[m_idx] * LIG_MAX_OOD_SAMPLES)) for lvl in unroll(0, LIG_N_LEVELS[m_idx]): for j in unroll(0, LIG_FOLDS[m_idx * LIG_MAX_LEVELS + lvl]): @@ -666,11 +769,10 @@ def open_stacked(m_idx: Const, fs0, fs1, target, commit_root_0, commit_root_1, c nonce_v = msg_cursor[GEN ** 0] # raw transport word: bound by the DS_POW absorb below msg_cursor = msg_cursor * GEN grind_check(fs[0], fs[1], nonce_v, GEN ** LIG_FOLD_GRIND_BITS[m_idx * LIG_MAX_TOTAL_FOLDS + fold_idx]) - fs = absorb(fs, nonce_v, DS_POW) - fs = squeeze(fs) - fold_challenge = fs[0] + fs = absorb_nonce(fs, nonce_v) + fs, fold_challenge = squeeze(fs) fold_challenges[GEN ** fold_idx] = fold_challenge - sumcheck_target = round_quad_c + fold_challenge * round_quad_b + fold_challenge * fold_challenge * round_quad_a # evaluate this level's folded quadratic at the fold challenge + sumcheck_target = (round_quad_a * fold_challenge + round_quad_b) * fold_challenge + round_quad_c # evaluate this level's folded quadratic at the fold challenge fs, msg_a, msg_cursor = fs_next(fs, msg_cursor) fs, msg_b, msg_cursor = fs_next(fs, msg_cursor) round_quad_c = msg_a @@ -684,15 +786,33 @@ def open_stacked(m_idx: Const, fs0, fs1, target, commit_root_0, commit_root_1, c else: fs, next_root_a, msg_cursor = fs_next(fs, msg_cursor) fs, next_root_b, msg_cursor = fs_next(fs, msg_cursor) - level_roots_0[GEN ** (lvl + 1)] = next_root_a - level_roots_1[GEN ** (lvl + 1)] = next_root_b + next_root = hash_words_to_state(next_root_a, next_root_b) + level_roots_0[GEN ** (lvl + 1)] = next_root[0] + level_roots_1[GEN ** (lvl + 1)] = next_root[1] + # OOD binding for the newly observed level-(lvl+1) commitment. + # The random point has the just-folded witness dimension, namely + # this level's message-column dimension. + for os in unroll(0, LIG_OOD_SAMPLES[m_idx * LIG_MAX_LEVELS + lvl + 1]): + oz = ood_z * GEN ** (((lvl + 1) * LIG_MAX_OOD_SAMPLES + os) * LIG_LOG_MSG_COLS_CAP) + for t in unroll(0, LIG_LOG_MSG_COLS[m_idx * LIG_MAX_LEVELS + lvl]): + fs, oz_challenge = squeeze(fs) + oz[GEN ** t] = oz_challenge + fs, ood_y, msg_cursor = fs_next(fs, msg_cursor) + fs, ood_u0, msg_cursor = fs_next(fs, msg_cursor) + fs, ood_u2, msg_cursor = fs_next(fs, msg_cursor) + fs, ood_beta = squeeze(fs) + ood_betas[GEN ** ((lvl + 1) * LIG_MAX_OOD_SAMPLES + os)] = ood_beta + round_quad_c += ood_beta * ood_u0 + round_quad_b += ood_beta * (ood_y + ood_u2) + round_quad_a += ood_beta * ood_u2 + sumcheck_target += ood_beta * ood_y q_nonce = msg_cursor[GEN ** 0] # raw transport word: bound by the DS_POW absorb below msg_cursor = msg_cursor * GEN if LIG_QUERY_GRIND_BITS[m_idx * LIG_MAX_LEVELS + lvl] != 0: grind_check(fs[0], fs[1], q_nonce, GEN ** LIG_QUERY_GRIND_BITS[m_idx * LIG_MAX_LEVELS + lvl]) else: assert q_nonce == 0 - fs = absorb(fs, q_nonce, DS_POW) + fs = absorb_nonce(fs, q_nonce) sqz_chain_0 = HeapBuf(GEN ** (LIG_MAX_SQUEEZES[m_idx] + 1)) sqz_chain_1 = HeapBuf(GEN ** (LIG_MAX_SQUEEZES[m_idx] + 1)) @@ -708,8 +828,7 @@ def open_stacked(m_idx: Const, fs0, fs1, target, commit_root_0, commit_root_1, c query_alphas = HeapBuf(GEN ** (LIG_MAX_INTERLEAVE[m_idx])) for t in unroll(0, LIG_LOG_QUERIES[m_idx * LIG_MAX_LEVELS + lvl]): - fs = squeeze(fs) - alpha_v = fs[0] + fs, alpha_v = squeeze(fs) query_alphas[GEN ** t] = alpha_v row_eq_weights = HeapBuf(GEN ** (LIG_MAX_INTERLEAVE[m_idx])) for i in unroll(0, LIG_INTERLEAVE[m_idx * LIG_MAX_LEVELS + lvl]): @@ -720,19 +839,60 @@ def open_stacked(m_idx: Const, fs0, fs1, target, commit_root_0, commit_root_1, c query_sum_chain = HeapBuf(GEN ** (LIG_MAX_QUERIES[m_idx] + 1)) query_sum_chain[GEN ** 0] = 0 for xe in mul_range(1, GEN ** LIG_QUERIES[m_idx * LIG_MAX_LEVELS + lvl]): - row_base = xe ** LIG_INTERLEAVE[m_idx * LIG_MAX_LEVELS + lvl] + if lvl == 0: + row_base = xe ** LIG_INTERLEAVE[m_idx * LIG_MAX_LEVELS + lvl] + else: + row_base = xe ** (3 * LIG_INTERLEAVE[m_idx * LIG_MAX_LEVELS + lvl]) row_ptr = merkle_leaf_rows * GEN ** LIG_ROWS_OFF[m_idx * LIG_MAX_LEVELS + lvl] * row_base row_dot = 0 - for jb in unroll(0, LIG_LEAF_PAIRS[m_idx * LIG_MAX_LEVELS + lvl]): - row_pair = [row_ptr[GEN ** (2 * jb)], row_ptr[GEN ** (2 * jb + 1)]] - row_dot += row_pair[0] * row_eq_weights[GEN ** (2 * jb)] + row_pair[1] * row_eq_weights[GEN ** (2 * jb + 1)] - # Standard BLAKE3 of the complete row. Ligerito row widths are - # powers of two no larger than one 1024-byte BLAKE3 chunk. + packed_row = StackBuf(LIG_PACKED_ROW_CAP) + if lvl == 0: + # Level-0 rows are base-field F64, embedded one-per word. Pack + # the lanes into a contiguous run of canonical 128-bit cells for + # the standard leaf hash; the dot consumes the individual lanes. + # PACK64X2 reads both source cells through the memory bus as + # `(lo, 0, 0)`, so the packs also prove every hinted lane is + # genuinely F64 before it enters the hash or row_dot. + for jb in unroll(0, LIG_INTERLEAVE[m_idx * LIG_MAX_LEVELS + lvl] // 4): + e0 = row_ptr[GEN ** (4 * jb)] + e1 = row_ptr[GEN ** (4 * jb + 1)] + e2 = row_ptr[GEN ** (4 * jb + 2)] + e3 = row_ptr[GEN ** (4 * jb + 3)] + pack64x2_into(e0, e1, packed_row[2 * jb]) + pack64x2_into(e2, e3, packed_row[2 * jb + 1]) + row_dot += e0 * row_eq_weights[GEN ** (4 * jb)] + e1 * row_eq_weights[GEN ** (4 * jb + 1)] + e2 * row_eq_weights[GEN ** (4 * jb + 2)] + e3 * row_eq_weights[GEN ** (4 * jb + 3)] + else: + # Higher-level F192 rows arrive as flat F64 tower limbs (three + # per word); constrain every serialized limb before reassembly + # and pack them into the contiguous 24-byte-per-word byte image + # the committed leaf hashes. + # Load each serialized limb ONCE into frame cells, then read the + # words back off the PACKED cells: a pack holds + # `lane(2k) + Y*lane(2k+1)` exactly, so word w (limbs 3w..3w+2) + # is one multiply-add away from the pack that covers its even + # limb pair. The old form re-read all three limbs per word and + # rebuilt it with two multiplies. + lanes = StackBuf(LIG_PACKED_ROW_CAP) # >= 3 limbs per word for every candidate + for jl in unroll(0, 3 * LIG_INTERLEAVE[m_idx * LIG_MAX_LEVELS + lvl]): + lanes[jl] = row_ptr[GEN ** jl] + for jb in unroll(0, LIG_LEAF_PAIRS[m_idx * LIG_MAX_LEVELS + lvl]): + pack64x2_into(lanes[4 * jb], lanes[4 * jb + 1], packed_row[2 * jb]) + pack64x2_into(lanes[4 * jb + 2], lanes[4 * jb + 3], packed_row[2 * jb + 1]) + for jw in unroll(0, LIG_INTERLEAVE[m_idx * LIG_MAX_LEVELS + lvl]): + if 3 * jw % 2 == 0: + # limbs (3w, 3w+1) are a pack; add Y^2 * limb(3w+2). + row_word = packed_row[3 * jw // 2] + Y_TOWER * Y_TOWER * lanes[3 * jw + 2] + else: + # limbs (3w+1, 3w+2) are a pack; shift it by Y and add limb(3w). + row_word = lanes[3 * jw] + Y_TOWER * packed_row[(3 * jw + 1) // 2] + row_dot += row_word * row_eq_weights[GEN ** jw] + # Standard BLAKE3 of the packed row (a power of two of full 64-byte + # blocks, within one 1024-byte chunk). leaf_hash_state = StackBuf(2) - blake3(row_ptr[0:2], row_ptr[2:4], leaf_hash_state, step=0, end=1 // LIG_LEAF_BLOCKS[m_idx * LIG_MAX_LEVELS + lvl], root=1 // LIG_LEAF_BLOCKS[m_idx * LIG_MAX_LEVELS + lvl]) + blake3(packed_row[0:2], packed_row[2:4], leaf_hash_state, step=0, end=1 // LIG_LEAF_BLOCKS[m_idx * LIG_MAX_LEVELS + lvl], root=1 // LIG_LEAF_BLOCKS[m_idx * LIG_MAX_LEVELS + lvl]) for jb in unroll(1, LIG_LEAF_BLOCKS[m_idx * LIG_MAX_LEVELS + lvl]): leaf_digest = StackBuf(2) - blake3(row_ptr[4 * jb:4 * jb + 2], row_ptr[4 * jb + 2:4 * jb + 4], leaf_digest, cv=leaf_hash_state, step=jb, end=(jb + 1) // LIG_LEAF_BLOCKS[m_idx * LIG_MAX_LEVELS + lvl], root=(jb + 1) // LIG_LEAF_BLOCKS[m_idx * LIG_MAX_LEVELS + lvl]) + blake3(packed_row[4 * jb:4 * jb + 2], packed_row[4 * jb + 2:4 * jb + 4], leaf_digest, cv=leaf_hash_state, step=jb, end=(jb + 1) // LIG_LEAF_BLOCKS[m_idx * LIG_MAX_LEVELS + lvl], root=(jb + 1) // LIG_LEAF_BLOCKS[m_idx * LIG_MAX_LEVELS + lvl]) leaf_hash_state = leaf_digest node_0 = leaf_hash_state[0] node_1 = leaf_hash_state[1] @@ -745,18 +905,18 @@ def open_stacked(m_idx: Const, fs0, fs1, target, commit_root_0, commit_root_1, c assert root_0 == commit_root_0 assert root_1 == commit_root_1 else: - assert root_0 == level_roots_0[GEN ** lvl] - assert root_1 == level_roots_1[GEN ** lvl] + # A heap store IS the equality assert here (`DerefMode::Cell` + # unifies the two cells, and the slot was written when the root + # was read off the stream), at one instruction instead of three. + level_roots_0[GEN ** lvl] = root_0 + level_roots_1[GEN ** lvl] = root_1 level_query_sum = query_sum_chain[GEN ** LIG_QUERIES[m_idx * LIG_MAX_LEVELS + lvl]] - # Every level, the last one included, ties its commitment in through an - # intro message folded into the running round polynomial. The last used - # to skip the message because the sumcheck stopped there; it no longer - # does, so the shapes coincide. + # Every level, including the last, ties its commitment in through an + # intro message before drawing its separation challenge. fs, intro_u0, msg_cursor = fs_next(fs, msg_cursor) fs, intro_u2, msg_cursor = fs_next(fs, msg_cursor) - fs = squeeze(fs) - beta_lvl = fs[0] + fs, beta_lvl = squeeze(fs) level_betas[GEN ** lvl] = beta_lvl round_quad_c += beta_lvl * intro_u0 round_quad_b += beta_lvl * (level_query_sum + intro_u2) @@ -764,14 +924,9 @@ def open_stacked(m_idx: Const, fs0, fs1, target, commit_root_0, commit_root_1, c sumcheck_target += beta_lvl * level_query_sum # ---- finish the sumcheck over the tail coordinates ---- - # The opening no longer hands the verifier a residual cube to sum over, so - # every basis below is evaluated at ONE point: the fold challenges followed - # by these. Coordinates past this candidate's yr_log_n are pinned to zero so - # the terminal can unroll to the global cap and have them contribute 1. tail_challenges = HeapBuf(GEN ** YR_LOG_CAP) for j in unroll(0, LIG_YR_LOG_LEN[m_idx] - 1): - fs = squeeze(fs) - tail_c = fs[0] + fs, tail_c = squeeze(fs) tail_challenges[GEN ** j] = tail_c sumcheck_target = round_quad_c + tail_c * round_quad_b + tail_c * tail_c * round_quad_a fs, msg_a, msg_cursor = fs_next(fs, msg_cursor) @@ -779,24 +934,20 @@ def open_stacked(m_idx: Const, fs0, fs1, target, commit_root_0, commit_root_1, c round_quad_c = msg_a round_quad_b = sumcheck_target + msg_b round_quad_a = msg_b - # The last round closes on the point evaluation, so it sends no message. - fs = squeeze(fs) - tail_last = fs[0] + # The closing round sends no following message. + fs, tail_last = squeeze(fs) tail_challenges[GEN ** (LIG_YR_LOG_LEN[m_idx] - 1)] = tail_last sumcheck_target = round_quad_c + tail_last * round_quad_b + tail_last * tail_last * round_quad_a for j in unroll(LIG_YR_LOG_LEN[m_idx], YR_LOG_CAP): tail_challenges[GEN ** j] = 0 - # The transmitted message is public, so its value at the tail point is a - # multilinear evaluation the verifier does itself. Every weight below is - # scaled by it once, at the terminal. tail_w = StackBuf(2 * YR_LOG_CAP) for j in unroll(0, LIG_YR_LOG_LEN[m_idx]): tail_w[2 * j] = 1 + tail_challenges[GEN ** j] tail_w[2 * j + 1] = tail_challenges[GEN ** j] yr_at_tail = fold_final_msg(final_msg, tail_w, 0, LIG_YR_LOG_LEN[m_idx]) - # ---- per-level residuals: novel-basis at the single fold point ---- + # ---- per-level induced bases at the single terminal point ---- inner_chain = HeapBuf(GEN ** (LIG_N_LEVELS[m_idx] + 1)) inner_chain[GEN ** 0] = 0 for lvl in unroll(0, LIG_N_LEVELS[m_idx]): @@ -818,7 +969,23 @@ def open_stacked(m_idx: Const, fs0, fs1, target, commit_root_0, commit_root_1, c prefix_eq *= (1 + tail_c * (1 + basis_w[LIG_RESIDUAL_PREFIX_LEN[m_idx * LIG_MAX_LEVELS + lvl] + j])) residual_chain[xr * GEN] = residual_chain[xr] + alpha_weights[GEN ** (lvl * LIG_MAX_QUERIES[m_idx]) * xr] * prefix_eq inner_chain[GEN ** (lvl + 1)] = inner_chain[GEN ** lvl] + level_betas[GEN ** lvl] * residual_chain[GEN ** LIG_QUERIES[m_idx * LIG_MAX_LEVELS + lvl]] # accumulate beta_lvl * (per-level residual sum) into the grand residual - return sumcheck_target, fold_challenges, final_msg, inner_chain[GEN ** LIG_N_LEVELS[m_idx]], GEN ** LIG_YR_LOG_LEN[m_idx], GEN ** (YR_LOG_CAP - LIG_YR_LOG_LEN[m_idx]), GEN ** LIG_TOTAL_FOLDS[m_idx], tail_challenges, yr_at_tail + + # Explicit OOD eq bases at the same terminal point. + ood_inner = 0 + for ood_lvl in unroll(1, LIG_N_LEVELS[m_idx]): + z_len = LIG_LOG_MSG_COLS[m_idx * LIG_MAX_LEVELS + ood_lvl - 1] + z_folded = z_len - LIG_YR_LOG_LEN[m_idx] + ris_start = LIG_FOLDS_OFF[m_idx * LIG_MAX_LEVELS + ood_lvl] + for os in unroll(0, LIG_OOD_SAMPLES[m_idx * LIG_MAX_LEVELS + ood_lvl]): + oz = ood_z * GEN ** ((ood_lvl * LIG_MAX_OOD_SAMPLES + os) * LIG_LOG_MSG_COLS_CAP) + scalar = ood_betas[GEN ** (ood_lvl * LIG_MAX_OOD_SAMPLES + os)] + for t in unroll(0, z_folded): + scalar *= (1 + oz[GEN ** t] + fold_challenges[GEN ** (ris_start + t)]) + for t in unroll(0, LIG_YR_LOG_LEN[m_idx]): + zt = oz[GEN ** (z_folded + t)] + scalar *= (1 + zt + tail_challenges[GEN ** t]) + ood_inner += scalar + return sumcheck_target, fold_challenges, final_msg, inner_chain[GEN ** LIG_N_LEVELS[m_idx]] + ood_inner, GEN ** LIG_YR_LOG_LEN[m_idx], GEN ** (YR_LOG_CAP - LIG_YR_LOG_LEN[m_idx]), GEN ** LIG_TOTAL_FOLDS[m_idx], tail_challenges, yr_at_tail def exponent_tables(): @@ -826,8 +993,12 @@ def exponent_tables(): # g-powers (so they must be heap, not stack): g_logs_pow2[g^j] = 2^j is 2 # raised to a g-power's log, and g_squares[g^j] = g^(2^j) turns integer # sums of powers of two into field products. Returns the 2 pointers. - g_logs_pow2 = HeapBuf(COUNT_BITS) - for j in unroll(0, COUNT_BITS): + # Both tables span SIZE_BITS: verify_log2_ceil bounds its result by + # SIZE_BITS (assert log(g_log) < SIZE_BITS), so g_log reaches g^(SIZE_BITS-1) + # and indexes g_logs_pow2 there; sizing to COUNT_BITS would leave that lookup + # reading an unwritten (prover-chosen) cell. + g_logs_pow2 = HeapBuf(SIZE_BITS) + for j in unroll(0, SIZE_BITS): g_logs_pow2[GEN ** j] = 2 ** j g_squares = HeapBuf(SIZE_BITS) sq_run = GEN @@ -841,33 +1012,39 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): # In-circuit verification of ONE inner proof for the statement # (pi_0, pi_1). All proof data is hinted HERE: each call pops the next # sub-proof's entry of every witness stream, so the body lowers once and - # main just calls it per statement. The g_logs_pow2/g_squares lookup - # tables are shared read-only tables built once in main; the - # deferred-claim data is written to `defer_out`. + # main just calls it per statement. The exponent lookup tables are shared + # read-only across calls; the deferred-claim data is written to + # `defer_out`. # # Flow (mirrors cpu::verify): # 1. seed the Fiat-Shamir sponge from the statement + program digest; # 2. announced sizes, then certify every structural log against them # (count gadget log2_ceil: tau per table, log_mem); - # 3. bind the commitment root; bus grinding (grind_check, runtime - # bit count); ONE RLC-batched GKR for all three trees (count padded + # 3. bind the commitment root; ONE RLC-batched GKR for all three trees (count padded # to the pair's depth) at runtime depth, ONE shared point zeta; # 4. derive the block kappas, certify the GKR side depths; balance check # with advice-decomposed padding ladders; 3x leaf decomposition, DERIVING # each side's table share from its GKR claim (pooling the # committed-coordinate claims); the stacked-bytecode reduction (deferred); - # 5. ONE batched zerocheck for all six tables, n = max_t tau_t rounds at the + # 5. ONE batched zerocheck for all seven tables, n = max_t tau_t rounds at the # shared point zeta, target derived from the leaf claims (sumcheck_round4); # 6. public-input claim + BLAKE3 pin claims (telescoped prefix MLE); # 7. flock reduction: univariate-skip zerocheck + lincheck (matrix # evaluation deferred); - # 8. ring-switch fronts (shared r'', linearized transpose in-circuit); + # 8. ring-switch fronts (shared linear map, transpose in-circuit); # 9. gamma-combine everything, certify the committed size m, dispatch # the stacked Ligerito opening (open_stacked), and assert its # eval_b terminal; # 10. export the deferred-claim region for the aggregation. # Claim pool: values of every committed-coordinate claim, in decompose order # (their points are the GKR ζ's, resolvable from the baked block structure). + # `1 + g^(2^k)` per bit position, in FRAME cells: the bit-ladder rebuilds + # below (padding surplus, placement offsets) each need this factor once per + # bit, and a StackBuf entry is an instruction operand, where the g_squares + # HeapBuf costs a load and an add every time. + gsq_plus = StackBuf(SIZE_BITS) + for k in unroll(0, SIZE_BITS): + gsq_plus[k] = 1 + g_squares[GEN ** k] claim_pool = HeapBuf(N_CLAIMS) # certified low dimension (cplen) per pooled claim, filled as the pool is # built (from the in-scope certified kappa/tau); the terminal pins each @@ -876,7 +1053,7 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): # The ONE shared GKR leaf point (all three trees reduce to it). # ---- seed (statement pre-bound: hinted sub pi + baked program digest) ---- - fs = [TRANSCRIPT_SEED_0, TRANSCRIPT_SEED_1] # the sponge state after the b"leanvm-b" domain label + fs = [TRANSCRIPT_SEED_0, TRANSCRIPT_SEED_1] fs = obs(fs, seed_0) # the FS seed: H(flock circuit family, inner program fs = obs(fs, seed_1) # bytecode, ...) — from the recursion's public input fs = obs(fs, pi_0) # bind the sub-proof's statement (its public input) @@ -885,18 +1062,22 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): hint_witness(stream[0:STREAM_CAP], "stream") cursor = stream # the proof stream is replayed word by word; cursor walks it (advance = * g) - # ---- announced sizes: log_mem + 6 row counts (observed, then certified) ---- + # ---- announced layout and PCS rate (observed, then certified) ---- sizes = StackBuf(N_TABLES + 1) for i in unroll(0, N_TABLES + 1): fs, x, cursor = fs_next(fs, cursor) sizes[i] = x + fs, log_inv_rate, cursor = fs_next(fs, cursor) + g_log_inv_rate = g_power_of_word(log_inv_rate, g_squares, COUNT_BITS) + rate_sel = g_log_inv_rate / GEN # g^(log_inv_rate - 1) + assert log(rate_sel) < LIG_N_RATES # ---- structural logs: certify g^log_mem, compute the taus ---- # The stream announced the sizes as integer WORDS; the shape-generic phases # need them as G-POWERS (loop bounds, match_range scrutinees). dims_g[0] = # g^log_mem arrives as a hint pinned to the word; dims_g[1 + t] = g^tau_t # is computed by the count gadget. - dims_g = HeapBuf(N_TABLES + 1) # [g^log_mem, g^tau_0 .. g^tau_5], all computed + dims_g = HeapBuf(N_TABLES + 1) # [g^log_mem, g^tau_0 .. g^tau_{N_TABLES-1}] # log_mem is announced AS a log (an integer word L): g^L is assembled from # L's advice-decomposed bits — no hint, no g^j -> j lookup table. g_log_mem = g_power_of_word(sizes[0], g_squares, COUNT_BITS) @@ -936,20 +1117,9 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): fs, commit_root_0, cursor = fs_next(fs, cursor) fs, commit_root_1, cursor = fs_next(fs, cursor) - # ---- bus: grinding FIRST, then α and γ (the PoW covers both) ---- - # grinding nonce: raw stream word (NOT observed), PoW-checked, then bound. - nonce = cursor[GEN ** 0] - cursor *= GEN - # Bus grind bits = push.mu - 7 (= SECURITY + push.mu + 1 - 128; see - # leaf::grand_product_grinding_bits), with g_bus_mu computed above from the - # derived block kappas. - bus_grind_window = g_bus_mu * INV_GEN ** BUS_GRIND_SHIFT # g^(push.mu - shift): the bus PoW bit count - grind_check(fs[0], fs[1], nonce, bus_grind_window) - fs = absorb(fs, nonce, DS_POW) - fs = squeeze(fs) - alpha = fs[0] - fs = squeeze(fs) - gamma = fs[0] + # ---- bus challenges (F192 provides the soundness margin without grinding) ---- + fs, alpha = squeeze(fs) + fs, gamma = squeeze(fs) # ---- ONE GKR grand product: push, pull, and count RLC-batched ---- # Push and pull have equal depth (matched blocks) and the count tree is @@ -980,8 +1150,8 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): fs, root_push, cursor = fs_next(fs, cursor) fs, root_pull, cursor = fs_next(fs, cursor) fs, root_count, cursor = fs_next(fs, cursor) - fs = squeeze(fs) - gkr_layer_lambda[GEN ** 0] = fs[0] # λ over the three roots + fs, initial_layer_lambda = squeeze(fs) + gkr_layer_lambda[GEN ** 0] = initial_layer_lambda # λ over the three roots gkr_layer_fs0[GEN ** 0] = fs[0] gkr_layer_fs1[GEN ** 0] = fs[1] gkr_layer_cursor[GEN ** 0] = cursor @@ -1028,15 +1198,14 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): tail_fs, e0_count, tcur = fs_next(tail_fs, tcur) tail_fs, e1_count, tcur = fs_next(tail_fs, tcur) assert tclaim == e0_push * e1_push + lam * (e0_pull * e1_pull + lam * (e0_count * e1_count)) - tail_fs = squeeze(tail_fs) - layer_challenge = tail_fs[0] + tail_fs, layer_challenge = squeeze(tail_fs) nextrow[GEN ** 0] = layer_challenge xln = x_layer * GEN gkr_layer_claim[xln] = e0_push + layer_challenge * (e0_push + e1_push) gkr_layer_claim_b[xln] = e0_pull + layer_challenge * (e0_pull + e1_pull) gkr_layer_claim_c[xln] = e0_count + layer_challenge * (e0_count + e1_count) - tail_fs = squeeze(tail_fs) # fresh λ pins the tail individuals - gkr_layer_lambda[xln] = tail_fs[0] + tail_fs, tail_lambda = squeeze(tail_fs) # fresh λ pins the tail individuals + gkr_layer_lambda[xln] = tail_lambda gkr_layer_fs0[xln] = tail_fs[0] gkr_layer_fs1[xln] = tail_fs[1] gkr_layer_cursor[xln] = tcur @@ -1085,10 +1254,8 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): pull_product = e0_pull * e1_pull * e2_pull * e3_pull count_product = e0_count * e1_count * e2_count * e3_count assert tclaim == push_product + lam * (pull_product + lam * count_product) - tail_fs = squeeze(tail_fs) - c0 = tail_fs[0] - tail_fs = squeeze(tail_fs) - c1 = tail_fs[0] + tail_fs, c0 = squeeze(tail_fs) + tail_fs, c1 = squeeze(tail_fs) nextrow[GEN ** 0] = c0 nextrow[GEN ** 1] = c1 push_lo = e0_push + c0 * (e0_push + e1_push) @@ -1101,8 +1268,8 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): gkr_layer_claim[xln] = push_lo + c1 * (push_lo + push_hi) gkr_layer_claim_b[xln] = pull_lo + c1 * (pull_lo + pull_hi) gkr_layer_claim_c[xln] = count_lo + c1 * (count_lo + count_hi) - tail_fs = squeeze(tail_fs) - gkr_layer_lambda[xln] = tail_fs[0] + tail_fs, tail_lambda = squeeze(tail_fs) + gkr_layer_lambda[xln] = tail_lambda gkr_layer_fs0[xln] = tail_fs[0] gkr_layer_fs1[xln] = tail_fs[1] gkr_layer_cursor[xln] = tcur @@ -1169,33 +1336,43 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): # (γ+fp)^DELTA ladder and are pinned by g^real · g^DELTA == g^(2^κ); real is # count_t for table blocks, 2^κ for shared blocks (DELTA = 0). An unpinned # DELTA would forge the balance (dlog is cheap in this field). + # ONE ladder per (side, table), not per block: every flush block of table t + # takes its kappa from the same certified source (tau_t) and its real row + # count from the same count_t, so the whole group shares one DELTA, and + # prod_b (gamma + fp_b)^DELTA == (prod_b (gamma + fp_b))^DELTA. + # The framework blocks are REAL_IS_FULL_CUBE (= N_TABLES, so the table loop + # skips them): real = 2^kappa makes DELTA = 0 by construction, and the ladder + # they run today is forced to return 1 anyway -- g^DELTA == g^(2^kappa)/g^real + # == 1 with COUNT_BITS bits far below the group order pins every bit to zero. + # Dropping it removes a hint, not a constraint. pad_products = HeapBuf(2) for s in unroll(0, 2): side_pad_product = GEN ** 0 - for b in unroll(SIDE_BLOCK_START[s], SIDE_BLOCK_START[s + 1]): - pad_fp = 0 - alpha_pow = GEN ** 0 - for i in unroll(0, BLOCK_COORD_COUNT[b]): - pad_fp += alpha_pow * COORD_PAD_VAL[BLOCK_COORD_OFF[b] + i] - alpha_pow *= alpha - g_two_kappa = g_squares[block_kappa[GEN ** b]] # g^(2^κ_b) - if BLOCK_REAL_TABLE[b] == REAL_IS_FULL_CUBE: - g_real = g_two_kappa # shared block: real = 2^κ, so DELTA = 0 - else: - g_real = count_gpows[GEN ** BLOCK_REAL_TABLE[b]] # g^count_t + for t in unroll(0, N_TABLES): + group_base = GEN ** 0 + for b in unroll(SIDE_BLOCK_START[s], SIDE_BLOCK_START[s + 1]): + if BLOCK_REAL_TABLE[b] == t: + pad_fp = 0 + alpha_pow = GEN ** 0 + for i in unroll(0, BLOCK_COORD_COUNT[b]): + pad_fp += alpha_pow * COORD_PAD_VAL[BLOCK_COORD_OFF[b] + i] + alpha_pow *= alpha + group_base *= (gamma + pad_fp) + g_two_kappa = g_squares[dims_g[GEN ** (t + 1)]] # g^(2^tau_t), the group's kappa + g_real = count_gpows[GEN ** t] # g^count_t g_delta_want = g_two_kappa / g_real # g^DELTA (feeds the advice below) pad_bits = HeapBuf(GEN ** COUNT_BITS) hint_decompose_bits_exponent(pad_bits, g_delta_want, COUNT_BITS) ladder = GEN ** 0 - ladder_square = gamma + pad_fp + ladder_square = group_base g_delta = GEN ** 0 for j in unroll(0, COUNT_BITS): pad_bit = pad_bits[GEN ** j] assert pad_bit * pad_bit == pad_bit ladder *= (1 + pad_bit * (ladder_square + 1)) - g_delta *= (1 + pad_bit * (g_squares[GEN ** j] + 1)) # g^DELTA + g_delta *= (1 + pad_bit * gsq_plus[j]) # g^DELTA ladder_square *= ladder_square - assert g_real * g_delta == g_two_kappa # real_b + DELTA_b == 2^κ_b + assert g_real * g_delta == g_two_kappa # count_t + DELTA_t == 2^tau_t side_pad_product *= ladder pad_products[GEN ** s] = side_pad_product lhsb = gkr_roots[PUSH_SIDE] * pad_products[GEN ** PULL_SIDE] # balance: push_root * d_pull == pull_root * d_push (padding cancels) @@ -1282,7 +1459,7 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): claim_cplen_g[GEN ** COORD_CLAIM_SLOT[BLOCK_COORD_OFF[b] + i]] = kappa_g # cplen = block kappa else: rawv = claim_pool[GEN ** COORD_CLAIM_SLOT[BLOCK_COORD_OFF[b] + i]] - coord_val = GEN * rawv + coord_val = COORD_CONST[BLOCK_COORD_OFF[b] + i] * rawv if COORD_TYPE[BLOCK_COORD_OFF[b] + i] == COORD_KIND_INDEX: if s == PULL_SIDE: coord_val = block_index_mle[GEN ** (b - SIDE_BLOCK_START[PULL_SIDE])] @@ -1325,14 +1502,13 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): fs = obs(fs, bytecode_vals[GEN ** k]) bytecode_sel = HeapBuf(LOG2_BYTECODE_COLS) for t in unroll(0, LOG2_BYTECODE_COLS): - fs = squeeze(fs) - sv = fs[0] + fs, sv = squeeze(fs) bytecode_sel[GEN ** t] = sv bytecode_reduced = 0 for c in unroll(0, BYTECODE_COLS): bytecode_reduced += eq_weight(bytecode_sel, LOG2_BYTECODE_COLS, c, 0) * bytecode_vals[GEN ** c] - # ---- ONE batched zerocheck for all six tables (XOR, MUL, SET, DEREF, JUMP, BLAKE3) ---- + # ---- ONE batched zerocheck for all seven tables ---- # Mirrors lean_vm::constraints::verify. eta ONCE, each table folding its own # identities with a DISJOINT range of its powers (ETA_OFFSET[t]); one shared # point zeta (the bus GKR's); n = max_t tau_t rounds. Rounds bind the HIGHEST @@ -1355,8 +1531,7 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): for t in unroll(0, N_TABLES): zc_dominates = g_zc_n / dims_g[GEN ** (t + 1)] assert log(zc_dominates) < COUNT_BITS - fs = squeeze(fs) - eta = fs[0] + fs, eta = squeeze(fs) eta_pows = StackBuf(N_ETA_POWS) eta_pows[0] = 1 for k in unroll(1, N_ETA_POWS): @@ -1420,24 +1595,38 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): # the table's AIR constraint at the final point (col_evals is indexed by # local column index; the formulas mirror tables.rs eval_constraint). if t == TABLE_XOR: - constraint_eval = eta_pows[ETA_OFFSET[t] + 0] * (col_evals[5] + col_evals[1] * col_evals[2]) + eta_pows[ETA_OFFSET[t] + 1] * (col_evals[6] + col_evals[1] * col_evals[3]) + eta_pows[ETA_OFFSET[t] + 2] * (col_evals[7] + col_evals[1] * col_evals[4]) + eta_pows[ETA_OFFSET[t] + 3] * (col_evals[10] + col_evals[8] + col_evals[9]) + va = f192_from_limbs(col_evals[8], col_evals[9], col_evals[10]) + vb = f192_from_limbs(col_evals[11], col_evals[12], col_evals[13]) + vc = f192_from_limbs(col_evals[14], col_evals[15], col_evals[16]) + constraint_eval = eta_pows[ETA_OFFSET[t] + 0] * (col_evals[5] + col_evals[1] * col_evals[2]) + eta_pows[ETA_OFFSET[t] + 1] * (col_evals[6] + col_evals[1] * col_evals[3]) + eta_pows[ETA_OFFSET[t] + 2] * (col_evals[7] + col_evals[1] * col_evals[4]) + eta_pows[ETA_OFFSET[t] + 3] * (vc + va + vb) if t == TABLE_MUL: - constraint_eval = eta_pows[ETA_OFFSET[t] + 0] * (col_evals[5] + col_evals[1] * col_evals[2]) + eta_pows[ETA_OFFSET[t] + 1] * (col_evals[6] + col_evals[1] * col_evals[3]) + eta_pows[ETA_OFFSET[t] + 2] * (col_evals[7] + col_evals[1] * col_evals[4]) + eta_pows[ETA_OFFSET[t] + 3] * (col_evals[10] + col_evals[8] * col_evals[9]) + va = f192_from_limbs(col_evals[8], col_evals[9], col_evals[10]) + vb = f192_from_limbs(col_evals[11], col_evals[12], col_evals[13]) + vc = f192_from_limbs(col_evals[14], col_evals[15], col_evals[16]) + constraint_eval = eta_pows[ETA_OFFSET[t] + 0] * (col_evals[5] + col_evals[1] * col_evals[2]) + eta_pows[ETA_OFFSET[t] + 1] * (col_evals[6] + col_evals[1] * col_evals[3]) + eta_pows[ETA_OFFSET[t] + 2] * (col_evals[7] + col_evals[1] * col_evals[4]) + eta_pows[ETA_OFFSET[t] + 3] * (vc + va * vb) if t == TABLE_SET: - constraint_eval = eta_pows[ETA_OFFSET[t] + 0] * (col_evals[4] + col_evals[1] * col_evals[2]) + constraint_eval = eta_pows[ETA_OFFSET[t] + 0] * (col_evals[6] + col_evals[1] * col_evals[2]) if t == TABLE_DEREF: - src = (1 + col_evals[5] + col_evals[6]) * col_evals[12] + col_evals[5] * (GEN * GEN * col_evals[0]) + col_evals[6] * col_evals[1] - constraint_eval = eta_pows[ETA_OFFSET[t] + 0] * (col_evals[7] + col_evals[1] * col_evals[2]) + eta_pows[ETA_OFFSET[t] + 1] * (col_evals[8] + col_evals[10] * col_evals[3]) + eta_pows[ETA_OFFSET[t] + 2] * (col_evals[9] + col_evals[1] * col_evals[4]) + eta_pows[ETA_OFFSET[t] + 3] * (col_evals[11] + src) + v2 = f192_from_limbs(col_evals[11], col_evals[12], col_evals[13]) + v3 = f192_from_limbs(col_evals[14], col_evals[15], col_evals[16]) + src = (1 + col_evals[5] + col_evals[6]) * v3 + col_evals[5] * (GEN * GEN * col_evals[0]) + col_evals[6] * col_evals[1] + constraint_eval = eta_pows[ETA_OFFSET[t] + 0] * (col_evals[7] + col_evals[1] * col_evals[2]) + eta_pows[ETA_OFFSET[t] + 1] * (col_evals[8] + col_evals[10] * col_evals[3]) + eta_pows[ETA_OFFSET[t] + 2] * (col_evals[9] + col_evals[1] * col_evals[4]) + eta_pows[ETA_OFFSET[t] + 3] * (v2 + src) if t == TABLE_JUMP: ft = GEN * col_evals[0] + c = f192_from_limbs(col_evals[10], col_evals[11], col_evals[12]) + d = f192_from_limbs(col_evals[13], col_evals[14], col_evals[15]) + ff = f192_from_limbs(col_evals[16], col_evals[17], col_evals[18]) + w = f192_from_limbs(col_evals[23], col_evals[24], col_evals[25]) addrs = eta_pows[ETA_OFFSET[t] + 0] * (col_evals[7] + col_evals[1] * col_evals[4]) + eta_pows[ETA_OFFSET[t] + 1] * (col_evals[8] + col_evals[1] * col_evals[5]) + eta_pows[ETA_OFFSET[t] + 2] * (col_evals[9] + col_evals[1] * col_evals[6]) - ind_def = eta_pows[ETA_OFFSET[t] + 3] * (col_evals[18] + col_evals[10] * col_evals[17]) - ind_nz = eta_pows[ETA_OFFSET[t] + 4] * (col_evals[10] * (col_evals[18] + 1)) - sel_pc = eta_pows[ETA_OFFSET[t] + 5] * (col_evals[2] + col_evals[18] * col_evals[11] + (col_evals[18] + 1) * ft) - sel_fp = eta_pows[ETA_OFFSET[t] + 6] * (col_evals[3] + col_evals[18] * col_evals[12] + (col_evals[18] + 1) * col_evals[1]) + ind_def = eta_pows[ETA_OFFSET[t] + 3] * (col_evals[26] + c * w) + ind_nz = eta_pows[ETA_OFFSET[t] + 4] * (c * (col_evals[26] + 1)) + sel_pc = eta_pows[ETA_OFFSET[t] + 5] * (col_evals[2] + col_evals[26] * d + (col_evals[26] + 1) * ft) + sel_fp = eta_pows[ETA_OFFSET[t] + 6] * (col_evals[3] + col_evals[26] * ff + (col_evals[26] + 1) * col_evals[1]) constraint_eval = addrs + ind_def + ind_nz + sel_pc + sel_fp if t == TABLE_BLAKE3: constraint_eval = eta_pows[ETA_OFFSET[t] + 0] * (col_evals[8] + col_evals[1] * col_evals[2]) + eta_pows[ETA_OFFSET[t] + 1] * (col_evals[9] + col_evals[1] * col_evals[3]) + eta_pows[ETA_OFFSET[t] + 2] * (col_evals[10] + col_evals[1] * col_evals[4]) + eta_pows[ETA_OFFSET[t] + 3] * (col_evals[11] + col_evals[1] * col_evals[5]) + eta_pows[ETA_OFFSET[t] + 4] * (col_evals[12] + col_evals[1] * col_evals[6]) + eta_pows[ETA_OFFSET[t] + 5] * (col_evals[13] + col_evals[1] * col_evals[7]) + if t == TABLE_PACK64X2: + constraint_eval = eta_pows[ETA_OFFSET[t] + 0] * (col_evals[5] + col_evals[1] * col_evals[2]) + eta_pows[ETA_OFFSET[t] + 1] * (col_evals[6] + col_evals[1] * col_evals[3]) + eta_pows[ETA_OFFSET[t] + 2] * (col_evals[7] + col_evals[1] * col_evals[4]) # The table's three bus forms, evaluated at the SAME column evaluations: # Σ_b eq_hi(b) · (γ + Σ_i α^i · coord_i), the coords read off col_evals at # their local index. This is what replaces opening those columns at ζ. @@ -1454,7 +1643,7 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): if COORD_TYPE[BLOCK_COORD_OFF[b] + i] == COORD_KIND_COL: cv = col_evals[COORD_COL_LOCAL[BLOCK_COORD_OFF[b] + i]] if COORD_TYPE[BLOCK_COORD_OFF[b] + i] == COORD_KIND_GCOL: - cv = GEN * col_evals[COORD_COL_LOCAL[BLOCK_COORD_OFF[b] + i]] + cv = COORD_CONST[BLOCK_COORD_OFF[b] + i] * col_evals[COORD_COL_LOCAL[BLOCK_COORD_OFF[b] + i]] if sd == COUNT_SIDE: inner += cv else: @@ -1468,18 +1657,29 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): air_acc += zc_round_cprod[g_zc_n / tau_g] * zc_peq[tau_g] * constraint_eval # cprod[n - tau] * peq[tau] assert air_acc == claim - # ---- public-input binding claim: MEM(r_m, 0..) = interp(pi0, pi1, r_m) ---- - fs = squeeze(fs) - rm = fs[0] - pi_interp = pi_0 + rm * (pi_0 + pi_1) # MLE of the 2-cell public memory at the sampled point rm - claim_pool[GEN ** claim_idx] = pi_interp + # ---- public-input binding claim: MEM as ONE logical E-column ---- + # The VM's bind_pi_claim makes a SINGLE E-claim at [rm, 0..]: + # MEM(rm) = interp(pi_0, pi_1, rm) = pi_0 + rm*(pi_0 + pi_1) + # over the E-valued public input (no lane splitting, no Frobenius). The + # opening-boundary decompose transmits v_lo and v_hi; the top lane is + # deduced from MEM = v_lo + Y*v_hi + Y²*v_top. + fs, rm = squeeze(fs) + mem = pi_0 + rm * (pi_0 + pi_1) + fs, mem_lo, cursor = fs_next(fs, cursor) + fs, mem_hi, cursor = fs_next(fs, cursor) + mem_top = (mem + mem_lo + mem_hi * Y_TOWER) * Y_INV * Y_INV + claim_pool[GEN ** claim_idx] = mem_lo + claim_idx += 1 + claim_pool[GEN ** claim_idx] = mem_hi + claim_idx += 1 + claim_pool[GEN ** claim_idx] = mem_top claim_idx += 1 # ---- flock zerocheck (univariate skip, k_skip = 6) ---- - tau_blake3_g = dims_g[GEN ** N_TABLES] # the BLAKE3 table's certified tau + tau_blake3_g = dims_g[GEN ** (TABLE_BLAKE3 + 1)] # the BLAKE3 table's certified tau # tau's reach is bounded: the count gadget gives tau < 34 (all flock # buffers are sized for that), and q_pkd's committed kappa = - # LOG2_FIELD_BITS + tau feeds the certified size m, whose opening + # K_LOG + tau feeds the certified size m, whose opening # dispatch bound caps tau well below any baked structure. # flock's sub-proof scalars are ordinary stream words (add_scalar on the # native side); the cursor walks them, fetching and observing each in one @@ -1491,8 +1691,7 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): mr1cs_g = tau_blake3_g * GEN ** K_LOG # runtime m = K_LOG + tau_5 (certified) in the exponent zerocheck_r = HeapBuf(mr1cs_g) for i in unroll(0, K_SKIP): - fs = squeeze(fs) - rv = fs[0] + fs, rv = squeeze(fs) zerocheck_r[GEN ** i] = rv for i in unroll(0, N_FIXED_CHALLENGE_ROUNDS): zerocheck_r[GEN ** (K_SKIP + i)] = FIXED_CHALLENGES[i] @@ -1503,8 +1702,8 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): flock_point_fs1[GEN ** (K_SKIP + N_FIXED_CHALLENGE_ROUNDS)] = fs[1] for xi in mul_range(GEN ** (K_SKIP + N_FIXED_CHALLENGE_ROUNDS), mr1cs_g): point_fs = [flock_point_fs0[xi], flock_point_fs1[xi]] - point_fs = squeeze(point_fs) - zerocheck_r[xi] = point_fs[0] + point_fs, zerocheck_challenge = squeeze(point_fs) + zerocheck_r[xi] = zerocheck_challenge xin = xi * GEN flock_point_fs0[xin] = point_fs[0] flock_point_fs1[xin] = point_fs[1] @@ -1515,8 +1714,7 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): for i in unroll(0, 2 * 2 ** K_SKIP): fs, w, cursor = fs_next(fs, cursor) zc_round1[GEN ** i] = w - fs = squeeze(fs) # cursor now sits at the multilinear round messages, walked below - zerocheck_z = fs[0] + fs, zerocheck_z = squeeze(fs) # cursor now sits at the multilinear round messages, walked below # interpolate P^C(z) on the Lambda domain (phi8 nodes 64..128): prefix/ # suffix numerator products with baked inverse denominators. lagrange_nums = StackBuf(2 ** K_SKIP) @@ -1543,10 +1741,9 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): fs, gamma_c, cursor = fs_next(fs, cursor) # (gamma_c, g_inf) per round, walked in order fs, g_inf, cursor = fs_next(fs, cursor) gamma_ab = (zc_running + r_eq * gamma_c) * ONE_PLUS_CHALLENGE_INV[i] # recover the g(alpha) evaluation from g(0)+g(1)=claim and the eq weight - fs = squeeze(fs) - rho_v = fs[0] + fs, rho_v = squeeze(fs) zerocheck_rhos[GEN ** i] = rho_v - zc_running = gamma_ab * (1 + rho_v) + gamma_c * rho_v + g_inf * rho_v * (1 + rho_v) + zc_running = gamma_ab + rho_v * (gamma_ab + gamma_c + (1 + rho_v) * g_inf) # rounds N_FIXED_CHALLENGE_ROUNDS.. at runtime count: K_LOG + tau_5 - K_SKIP rounds total (certified). nmlv_g = tau_blake3_g * GEN ** (K_LOG - K_SKIP) flock_round_size = mr1cs_rounds_g * GEN ** 2 @@ -1566,10 +1763,9 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): round_fs, gamma_c, cur_i = fs_next(round_fs, cur_i) round_fs, g_inf, cur_i = fs_next(round_fs, cur_i) gamma_ab = (round_running + r_eq * gamma_c) / (1 + r_eq) - round_fs = squeeze(round_fs) - rho_v = round_fs[0] + round_fs, rho_v = squeeze(round_fs) zerocheck_rhos[xi] = rho_v - round_running = gamma_ab * (1 + rho_v) + gamma_c * rho_v + g_inf * rho_v * (1 + rho_v) + round_running = gamma_ab + rho_v * (gamma_ab + gamma_c + (1 + rho_v) * g_inf) xin = xi * GEN flock_round_fs0[xin] = round_fs[0] flock_round_fs1[xin] = round_fs[1] @@ -1587,21 +1783,18 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): # ---- flock lincheck (matrix evaluation DEFERRED) ---- matrix_eval = StackBuf(1) hint_witness(matrix_eval[0:1], "matpart") - fs = squeeze(fs) - lincheck_alpha = fs[0] - fs = squeeze(fs) - lincheck_beta = fs[0] + fs, lincheck_alpha = squeeze(fs) + fs, lincheck_beta = squeeze(fs) lc_running = lincheck_alpha * a_eval + b_eval + lincheck_beta # lincheck seed: alpha*a + b + beta (batches the two matrix claims) lincheck_rs = HeapBuf(LINCHECK_ROUNDS) for i in unroll(0, LINCHECK_ROUNDS): fs, e1, cursor = fs_next(fs, cursor) # (e1, e_inf) per round, walked in order fs, ei, cursor = fs_next(fs, cursor) - fs = squeeze(fs) - rv = fs[0] + fs, rv = squeeze(fs) lincheck_rs[GEN ** i] = rv e0 = lc_running + e1 c1q = e0 + e1 + ei - lc_running = ei * rv * rv + c1q * rv + e0 # fold the degree-2 round poly at the challenge rv + lc_running = (ei * rv + c1q) * rv + e0 # fold the degree-2 round poly at the challenge rv z_partial = HeapBuf(2 ** K_SKIP) # post-sumcheck collapse: fetch + observe each word for i in unroll(0, 2 ** K_SKIP): fs, w, cursor = fs_next(fs, cursor) @@ -1616,8 +1809,7 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): lincheck_final = matrix_part + pin_term # running == deferred matrix eval + the const-pin column contribution assert lc_running == lincheck_final # fresh z_skip; w = (phi8 nodes 0..64). - fs = squeeze(fs) - lincheck_z_skip = fs[0] + fs, lincheck_z_skip = squeeze(fs) skip_nums = StackBuf(2 ** K_SKIP) lag64(lincheck_z_skip, skip_nums, 0) lincheck_w = 0 @@ -1625,112 +1817,105 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): lincheck_w += skip_nums[i] * LAGRANGE_INV_S[i] * z_partial[GEN ** i] # ---- stacked mixed opening: ring-switch fronts + claim combination ---- - s_hat_v = StackBuf(2 * FIELD_BITS) # the two ring-switch slices (end the stream), fetched + observed in the loop below - # Ring-switch claim 0 (ab): value lincheck_w, z_skip = lincheck_z_skip, x_outer[0] = lincheck_rs[LINCHECK_ROUNDS-1] - # (x_inner_rest is the REVERSED lincheck round vector). Claim 1 (c): value - # c_eval, z_skip = zerocheck_z, x_outer[0] = zerocheck_r[6]. + # The two ring-switch slices (ab, c) each carry PACKING = 2^LOG_PACKING = 64 + # entries (one per packing bit) and live in the opening STRUCT + # (RingSwitchProof), observed into the sponge HERE (never on the stream). + # Claim 0 (ab): value lincheck_w, z_skip = lincheck_z_skip. Claim 1 (c): + # value c_eval, z_skip = zerocheck_z. (The 128->64 half-fold the prover does + # in blake3_flock::ring_claim is already baked into the transmitted 64 values, + # so the verifier just checks the plain prefix-weighted inner product.) + s_hat_v = HeapBuf(2 * (2 ** K_SKIP)) + hint_witness(s_hat_v[0 : 2 * (2 ** K_SKIP)], "rs_shatv") transposed_claims = StackBuf(2) rs_eq_vals = StackBuf(2) - c_table = StackBuf(FIELD_BITS) + map_challenges = HeapBuf(6) + c_table = HeapBuf(BASE_FIELD_BITS) z_vals = HeapBuf(2 * QPKD_VARS_CAP) - r_dprime = StackBuf(LOG2_FIELD_BITS) for rs in unroll(0, 2): - for i in unroll(0, FIELD_BITS): - fs, w, cursor = fs_next(fs, cursor) - s_hat_v[FIELD_BITS * rs + i] = w - # claim check: weights[i] = lambda_{i&63}(z_skip) * eq(x_outer0, i>>6). + # observe this claim's 64 s_hat_v entries (mirror of verify_observe / + # observe_ext_slice) before the claim check and the shared map. + for i in unroll(0, (2 ** K_SKIP)): + fs = obs(fs, s_hat_v[GEN ** ((2 ** K_SKIP) * rs + i)]) + # claim check: value == sum_i prefix_weights[i] * s_hat_v[i], where + # prefix_weights[i] = lambda_i(z_skip) = lag numerator * LAGRANGE_INV_S[i]. if rs == 0: claim_z_skip = lincheck_z_skip - claim_x_outer_0 = lincheck_rs[GEN ** (LINCHECK_ROUNDS - 1)] claim_val = lincheck_w else: claim_z_skip = zerocheck_z - claim_x_outer_0 = zerocheck_r[GEN ** K_SKIP] claim_val = c_eval - claim_nums = StackBuf(2 ** K_SKIP) + claim_nums = StackBuf((2 ** K_SKIP)) lag64(claim_z_skip, claim_nums, 0) claim_check = 0 - for i in unroll(0, 2 ** K_SKIP): - lagrange_w = claim_nums[i] * LAGRANGE_INV_S[i] - claim_check += lagrange_w * ((1 + claim_x_outer_0) * s_hat_v[FIELD_BITS * rs + i] + claim_x_outer_0 * s_hat_v[FIELD_BITS * rs + 2 ** K_SKIP + i]) # claim = sum_i lambda_i(z_skip) * eq(x_outer0, i>>6) * s_hat_v[i] + for i in unroll(0, (2 ** K_SKIP)): + claim_check += claim_nums[i] * LAGRANGE_INV_S[i] * s_hat_v[GEN ** ((2 ** K_SKIP) * rs + i)] assert claim_check == claim_val - # ONE r'' shared by both claims (each slice was absorbed before the - # sample), so one eq tensor and one linearized coefficient table - # serve the whole batch. - for i in unroll(0, LOG2_FIELD_BITS): - fs = squeeze(fs) - rv = fs[0] - r_dprime[i] = rv - # Only eq(r'', i), i=0..6, is needed by the sparse correction to the - # reversed-monomial trace-dual basis. These indices share four zero high - # bits, so their common factor is computed once. - correction_weights = StackBuf(LOG2_FIELD_BITS) - correction_high = GEN ** 0 - for t in unroll(3, LOG2_FIELD_BITS): - correction_high *= 1 + r_dprime[t] - for i in unroll(0, LOG2_FIELD_BITS): - correction_weight = correction_high - for t in unroll(0, 3): - if (i // (2 ** t)) % 2 == 1: - correction_weight *= r_dprime[t] - else: - correction_weight *= 1 + r_dprime[t] - correction_weights[i] = correction_weight - # Factored c_k. The 128-term reversed-monomial sum is a seven-factor MLE - # product; only the first seven dual-basis elements add corrections. - for k in unroll(0, FIELD_BITS): - c_main = RS_COEFF_ORBITS[RS_COEFF_ORBIT_WIDTH * k] - for t in unroll(0, LOG2_FIELD_BITS): - c_main *= 1 + r_dprime[t] * RS_COEFF_ORBITS[RS_COEFF_ORBIT_WIDTH * k + 1 + t] - c_correction = 0 - for i in unroll(0, LOG2_FIELD_BITS): - c_correction += correction_weights[i] * RS_COEFF_ORBITS[RS_COEFF_ORBIT_WIDTH * k + 1 + LOG2_FIELD_BITS + i] - c_table[k] = c_main + c_correction - for rs in unroll(0, 2): - # Transposed claim T = sum_j x^j * L_w(shv_j). Both fixed 128-step - # dimensions are unrolled; only proof-size-dependent loops remain - # runtime loops in this verifier. - x_pow = GEN ** 0 - transposed_claim = 0 - for j in unroll(0, FIELD_BITS): - y_pow = s_hat_v[FIELD_BITS * rs + j] - lin_eval = 0 - for k in unroll(0, FIELD_BITS): # L_w(y) = sum_k c_k y^(2^k); y^(2^k) squares once per step - lin_eval += c_table[k] * y_pow - if k != FIELD_BITS - 1: - y_pow *= y_pow - transposed_claim += x_pow * lin_eval - if j != FIELD_BITS - 1: - x_pow *= 2 # x = the field element 2 (the polynomial x) - transposed_claims[rs] = transposed_claim - # z_vals for eval_rs_eq (the x_outer tail), used at the opening terminal. - if rs == 0: - for t in unroll(0, LINCHECK_ROUNDS - 1): - z_vals[GEN ** t] = lincheck_rs[GEN ** (LINCHECK_ROUNDS - 2 - t)] - zv_lo = z_vals * GEN ** (LINCHECK_ROUNDS - 1) - zr_hi = zerocheck_rhos * GEN ** LINCHECK_ROUNDS - for xt in mul_range(1, tau_blake3_g): - zv_lo[xt] = zr_hi[xt] - else: - # row 1 lives at the CAPACITY stride (QPKD_VARS_CAP); its length is the - # runtime qpkdv. - zv_hi = z_vals * GEN ** QPKD_VARS_CAP - zcr7 = zerocheck_r * GEN ** (K_SKIP + 1) - for xt in mul_range(1, tau_blake3_g * GEN ** (K_LOG - LOG2_FIELD_BITS)): - zv_hi[xt] = zcr7[xt] + # Compose six two-term F2-linear maps with shifts 32,16,8,4,2,1. Their + # expansion has all 64 Frobenius terms required for soundness, while direct + # application costs 63 squarings and only six general multiplications. + for stage in unroll(0, len(RING_MAP_SHIFTS)): + fs, map_challenge = squeeze(fs) + map_challenges[GEN ** stage] = map_challenge + # Expand the same composition once for the later transparent-weight + # evaluation. Before shift d, the populated coefficients are exactly at + # multiples of 2d; the new branch fills the adjacent d-offset entries. + c_table[GEN ** 0] = 1 + for stage in unroll(0, len(RING_MAP_SHIFTS)): + shift = RING_MAP_SHIFTS[stage] + map_challenge = map_challenges[GEN ** stage] + for slot in unroll(0, BASE_FIELD_BITS // (2 * shift)): + coefficient = c_table[GEN ** (slot * 2 * shift)] + for k in unroll(0, shift): + coefficient *= coefficient + c_table[GEN ** (slot * 2 * shift + shift)] = map_challenge * coefficient + # Evaluate both claims together and combine their 64 packing rows. + s_hat_row_0 = s_hat_v + s_hat_row_1 = s_hat_v * GEN ** (2 ** K_SKIP) + x_pow_chain = HeapBuf((2 ** K_SKIP) + 1) + x_pow_chain[GEN ** 0] = GEN ** 0 + t_chain_0 = HeapBuf((2 ** K_SKIP) + 1) + t_chain_1 = HeapBuf((2 ** K_SKIP) + 1) + t_chain_0[GEN ** 0] = 0 + t_chain_1[GEN ** 0] = 0 + for x_round in mul_range(1, GEN ** (2 ** K_SKIP)): + lin_eval_0 = s_hat_row_0[x_round] + lin_eval_1 = s_hat_row_1[x_round] + for stage in unroll(0, len(RING_MAP_SHIFTS)): + frobenius_0 = lin_eval_0 + frobenius_1 = lin_eval_1 + for k in unroll(0, RING_MAP_SHIFTS[stage]): + frobenius_0 *= frobenius_0 + frobenius_1 *= frobenius_1 + map_challenge = map_challenges[GEN ** stage] + lin_eval_0 += map_challenge * frobenius_0 + lin_eval_1 += map_challenge * frobenius_1 + x_pow = x_pow_chain[x_round] + t_chain_0[x_round * GEN] = t_chain_0[x_round] + x_pow * lin_eval_0 + t_chain_1[x_round * GEN] = t_chain_1[x_round] + x_pow * lin_eval_1 + x_pow_chain[x_round * GEN] = x_pow * 2 + transposed_claims[0] = t_chain_0[GEN ** (2 ** K_SKIP)] + transposed_claims[1] = t_chain_1[GEN ** (2 ** K_SKIP)] + # Suffix points for the two transparent weights. + for t in unroll(0, LINCHECK_ROUNDS): + z_vals[GEN ** t] = lincheck_rs[GEN ** (LINCHECK_ROUNDS - 1 - t)] + zv_lo = z_vals * GEN ** LINCHECK_ROUNDS + zr_hi = zerocheck_rhos * GEN ** LINCHECK_ROUNDS + for xt in mul_range(1, tau_blake3_g): + zv_lo[xt] = zr_hi[xt] + zv_hi = z_vals * GEN ** QPKD_VARS_CAP + zcr7 = zerocheck_r * GEN ** K_SKIP + for xt in mul_range(1, tau_blake3_g * GEN ** SLOT_STRIDE_LOG): + zv_hi[xt] = zcr7[xt] # gamma-combine the two transposed sumcheck claims (computed in-circuit). - fs = squeeze(fs) - gamma_ab = fs[0] - fs = squeeze(fs) - gamma_c = fs[0] + fs, gamma_ab = squeeze(fs) + fs, gamma_c = squeeze(fs) target = gamma_ab * transposed_claims[0] + gamma_c * transposed_claims[1] # gamma-batch the two ring-switch claims into the opening's target # ...then every pooled point claim, each observed. for j in unroll(0, N_CLAIMS): fs = obs(fs, claim_pool[GEN ** j]) gamma_pool = HeapBuf(N_CLAIMS) for j in unroll(0, N_CLAIMS): - fs = squeeze(fs) - gv = fs[0] + fs, gv = squeeze(fs) gamma_pool[GEN ** j] = gv target += gv * claim_pool[GEN ** j] @@ -1782,7 +1967,7 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): for k in unroll(0, SIZE_BITS): offset_bit = offset_row[GEN ** k] assert offset_bit * offset_bit == offset_bit - rebuilt_offset *= (1 + offset_bit * (g_squares[GEN ** k] + 1)) + rebuilt_offset *= (1 + offset_bit * gsq_plus[k]) assert rebuilt_offset == col_off_g[GEN ** c] for k in unroll(SIZE_BITS, SIZE_BITS + YR_LOG_CAP): offset_row[GEN ** k] = 0 @@ -1790,9 +1975,14 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): # ---- certify g^m: m = max(log2_ceil(sum_cols 2^kappa), PCS_MIN_MU) ---- # g_total is g^(sum 2^kappa) from the certified placement walk above. gmv = log2_ceil_in_the_exponent(g_total, g_logs_pow2, g_squares, PCS_MIN_MU, SIZE_BITS) # g^m - sel = gmv * LIG_MIN_SHIFT_INV # g^(m - MIN): the match_range arm index selecting the opening candidate - assert log(sel) < LIG_N_CANDIDATES - sumcheck_target, fold_challenges, final_msg, inner_total, yr_log_n_g, yr_pad_g, fold_cap_g, tail_challenges, yr_at_tail = match_range(log(sel), range(0, LIG_N_CANDIDATES), lambda m_idx: open_stacked(m_idx, fs[0], fs[1], target, commit_root_0, commit_root_1, cursor)) + size_sel = gmv * LIG_MIN_SHIFT_INV # g^(m - MIN) + assert log(size_sel) < LIG_N_LOG_SIZES + # Flatten (rate-1, m-MIN) in rate-major order. Both coordinates are + # transcript-bound and range-checked above, so a single compiled guest can + # dispatch independently for every inner proof in a mixed-rate batch. + config_sel = size_sel * rate_sel ** LIG_N_LOG_SIZES + assert log(config_sel) < LIG_N_CANDIDATES + sumcheck_target, fold_challenges, final_msg, inner_total, yr_log_n_g, yr_pad_g, fold_cap_g, tail_challenges, yr_at_tail = match_range(log(config_sel), range(0, LIG_N_CANDIDATES), lambda m_idx: open_stacked(m_idx, fs[0], fs[1], target, commit_root_0, commit_root_1, cursor)) # `stream` is a fixed-capacity witness transport. The shape fixes the exact # consumed prefix, whose every word is transcript-bound; the unused suffix # is outside the recursively verified proof and intentionally unconstrained. @@ -1818,6 +2008,36 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): prefix_mask_table[GEN ** (t * YR_LOG_CAP + k)] = 1 for k in unroll(t, YR_LOG_CAP): prefix_mask_table[GEN ** (t * YR_LOG_CAP + k)] = 0 + # ---- shared low-coordinate eq chains ---- + # A claim's low_eq is the prefix product prod_{k < low_len} (1 + p_k + ris_k) + # over its point buffer p: the FACTORS depend only on which buffer the claim + # reads (and, for the qpkd slots, on the ris shift), never on the claim, so + # every claim on one buffer multiplies the same factors in the same order and + # differs only in where it stops. Build one prefix-product chain per buffer + # and let each claim read the partial product at its own certified length. + # Chain entry t is written from inputs with index < t only, so a garbage tail + # (past a buffer's written extent) cannot corrupt any shorter prefix; the + # length pins below keep every claim's read inside the written span + # (low_len <= cplen <= the point buffer's extent, and nlow <= lenris). + zeta_eq_chain = HeapBuf(SIZE_BITS + 1) + zeta_eq_chain[GEN ** 0] = 1 + for xk in mul_range(1, g_bus_mu): + zeta_eq_chain[xk * GEN] = zeta_eq_chain[xk] * (1 + zeta[xk] + fold_challenges[xk]) + rho_eq_chain = HeapBuf(SIZE_BITS + 1) + rho_eq_chain[GEN ** 0] = 1 + for xk in mul_range(1, g_zc_n): + rho_eq_chain[xk * GEN] = rho_eq_chain[xk] * (1 + rho[xk] + fold_challenges[xk]) + # The qpkd variants read the same points against ris shifted past the slot + # coordinates, so they need their own chains. + ris_slot = fold_challenges * GEN ** SLOT_STRIDE_LOG + zeta_slot_eq_chain = HeapBuf(SIZE_BITS + 1) + zeta_slot_eq_chain[GEN ** 0] = 1 + for xk in mul_range(1, g_bus_mu): + zeta_slot_eq_chain[xk * GEN] = zeta_slot_eq_chain[xk] * (1 + zeta[xk] + ris_slot[xk]) + rho_slot_eq_chain = HeapBuf(SIZE_BITS + 1) + rho_slot_eq_chain[GEN ** 0] = 1 + for xk in mul_range(1, g_zc_n): + rho_slot_eq_chain[xk * GEN] = rho_slot_eq_chain[xk] * (1 + rho[xk] + ris_slot[xk]) claim_weights = HeapBuf(N_CLAIMS) for j in unroll(0, N_CLAIMS): claim_offset_bits = col_offset_bits * GEN ** ((SIZE_BITS + YR_LOG_CAP) * CLAIM_COMMITTED_COL[j]) @@ -1837,10 +2057,11 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): nlow = cplen_g # delta = 0 for pi else: cplen_g = claim_cplen_g[GEN ** j] + nlow = cplen_g if CLAIM_POINT_BUF[j] == POINT_BUF_QPKD: - nlow = cplen_g * GEN ** LOG2_FIELD_BITS # nlow = cplen + the qpkd slot coords - else: - nlow = cplen_g # nlow = cplen + nlow = cplen_g * GEN ** SLOT_STRIDE_LOG # nlow = cplen + the qpkd slot coords + if CLAIM_POINT_BUF[j] == POINT_BUF_QPKD_RHO: + nlow = cplen_g * GEN ** SLOT_STRIDE_LOG nover_g = claim_nover[GEN ** j] # nover <= YR_LOG_CAP: honest nover <= yr_log_n <= cap, and the y-slot # loop below selects prefix_mask_table row nover, so its log must be @@ -1857,37 +2078,31 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): # so it stays in [0, lenris): either seln == 0 (empty loop) or # nlow + seln == lenris (the honest overlap-free case). assert (nlow * seln + fold_cap_g) * (seln + 1) == 0 - low_chain = HeapBuf(SIZE_BITS + 1) + # low_eq: the shared chain's partial product at this claim's certified + # length, times the qpkd slot factors (the only per-claim part). if CLAIM_POINT_BUF[j] == POINT_BUF_ZETA: - zptr = zeta - low_chain[GEN ** 0] = 1 - for xk in mul_range(1, low_len_g): - low_chain[xk * GEN] = low_chain[xk] * (1 + zptr[xk] + fold_challenges[xk]) + low_eq = zeta_eq_chain[low_len_g] if CLAIM_POINT_BUF[j] == POINT_BUF_RHO: - rptr = rho - low_chain[GEN ** 0] = 1 - for xk in mul_range(1, low_len_g): - low_chain[xk * GEN] = low_chain[xk] * (1 + rptr[xk] + fold_challenges[xk]) + low_eq = rho_eq_chain[low_len_g] if CLAIM_POINT_BUF[j] == POINT_BUF_PI: + low_chain = HeapBuf(SIZE_BITS + 1) low_chain[GEN ** 0] = 1 low_chain[GEN ** 1] = 1 + rm + fold_challenges[GEN ** 0] for xk in mul_range(GEN, low_len_g): low_chain[xk * GEN] = low_chain[xk] * (1 + fold_challenges[xk]) + low_eq = low_chain[low_len_g] if CLAIM_POINT_BUF[j] == POINT_BUF_QPKD: qpkd_slot_eq = GEN ** 0 - for k in unroll(0, LOG2_FIELD_BITS): - sb3 = CLAIM_QPKD_SLOT_BITS[LOG2_FIELD_BITS * j + k] + for k in unroll(0, SLOT_STRIDE_LOG): + sb3 = CLAIM_QPKD_SLOT_BITS[SLOT_STRIDE_LOG * j + k] + qpkd_slot_eq *= (1 + sb3 + fold_challenges[GEN ** k]) + low_eq = qpkd_slot_eq * zeta_slot_eq_chain[low_len_g] + if CLAIM_POINT_BUF[j] == POINT_BUF_QPKD_RHO: + qpkd_slot_eq = GEN ** 0 + for k in unroll(0, SLOT_STRIDE_LOG): + sb3 = CLAIM_QPKD_SLOT_BITS[SLOT_STRIDE_LOG * j + k] qpkd_slot_eq *= (1 + sb3 + fold_challenges[GEN ** k]) - # A value column is referenced only by its own table's bus blocks, and - # those are settled by the zerocheck, so the instance point of a q_pkd - # slot claim is always the batch's rho (asserted where the descriptors - # are built). - zptr = rho - ris7 = fold_challenges * GEN ** LOG2_FIELD_BITS - low_chain[GEN ** 0] = qpkd_slot_eq - for xk in mul_range(1, low_len_g): - low_chain[xk * GEN] = low_chain[xk] * (1 + zptr[xk] + ris7[xk]) - low_eq = low_chain[low_len_g] + low_eq = qpkd_slot_eq * rho_slot_eq_chain[low_len_g] ris_hi = fold_challenges * nlow # Selector coordinates [nlow, lenris) are exactly the corresponding # certified placement-offset bits. @@ -1900,27 +2115,46 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): claim_weights[GEN ** j] = sel_chain[seln] * gamma_pool[GEN ** j] # eval_rs_eq per claim: E = sum_k c_k * prod_j (z_j^(2^k) + 1 + ris_j) # (the telescoped product formula; z powers evolve by squaring per k). - # QPKD_VARS_CAP = tau_5 + (K_LOG - LOG2_FIELD_BITS), exponent-additive from the certified announced log. Walk the runtime coordinates outside and the fixed 128 Frobenius powers inside: each coordinate loads its opening challenge once, evolves z by squaring in registers, and advances one contiguous 128-product row. This is the same product formula as the k-major form, without 128 separate runtime loops or a stored z-power table. - qpkdv_g = tau_blake3_g * GEN ** (K_LOG - LOG2_FIELD_BITS) - for rs in unroll(0, 2): - prod_chains = HeapBuf((qpkdv_g * GEN) ** FIELD_BITS) - z_row_src = z_vals * GEN ** (QPKD_VARS_CAP * rs) - for k in unroll(0, FIELD_BITS): - prod_chains[GEN ** k] = 1 - for x_round in mul_range(1, qpkdv_g): - zv = z_row_src[x_round] - oq = 1 + fold_challenges[x_round] - prod_row = prod_chains * x_round ** FIELD_BITS - prod_row_next = prod_row * GEN ** FIELD_BITS - for k in unroll(0, FIELD_BITS): - prod_row_next[GEN ** k] = prod_row[GEN ** k] * (zv + oq) - if k != FIELD_BITS - 1: - zv *= zv - prod_final = prod_chains * qpkdv_g ** FIELD_BITS - e_acc = 0 - for k in unroll(0, FIELD_BITS): - e_acc += c_table[k] * prod_final[GEN ** k] - rs_eq_vals[rs] = e_acc + # QPKD_VARS_CAP = tau_5 + SLOT_STRIDE_LOG, exponent-additive from the + # certified announced log. Walk the runtime coordinates OUTSIDE and the + # fixed FIELD_BITS Frobenius powers inside: each coordinate loads its + # opening challenge once and evolves z by squaring in registers, advancing + # one contiguous FIELD_BITS-wide product row. Same product formula as the + # k-major form, but with no stored z-power table (the dominant memory + # traffic) and no per-level buffer. + qpkdv_g = tau_blake3_g * GEN ** SLOT_STRIDE_LOG + # Evaluate both transparent weights in lockstep, sharing c_k and the + # verifier-point factor in every inner iteration. + z_row_src_1 = z_vals * GEN ** QPKD_VARS_CAP + prod_chains_0 = HeapBuf((qpkdv_g * GEN) ** BASE_FIELD_BITS) + prod_chains_1 = HeapBuf((qpkdv_g * GEN) ** BASE_FIELD_BITS) + for k in unroll(0, BASE_FIELD_BITS): + prod_chains_0[GEN ** k] = 1 + prod_chains_1[GEN ** k] = 1 + for x_round in mul_range(1, qpkdv_g): + zv_0 = z_vals[x_round] + zv_1 = z_row_src_1[x_round] + one_plus = 1 + fold_challenges[x_round] + prod_row_0 = prod_chains_0 * x_round ** BASE_FIELD_BITS + prod_row_1 = prod_chains_1 * x_round ** BASE_FIELD_BITS + prod_row_next_0 = prod_row_0 * GEN ** BASE_FIELD_BITS + prod_row_next_1 = prod_row_1 * GEN ** BASE_FIELD_BITS + for k in unroll(0, BASE_FIELD_BITS): + prod_row_next_0[GEN ** k] = prod_row_0[GEN ** k] * (zv_0 + one_plus) + prod_row_next_1[GEN ** k] = prod_row_1[GEN ** k] * (zv_1 + one_plus) + if k != BASE_FIELD_BITS - 1: + zv_0 *= zv_0 + zv_1 *= zv_1 + prod_final_0 = prod_chains_0 * qpkdv_g ** BASE_FIELD_BITS + prod_final_1 = prod_chains_1 * qpkdv_g ** BASE_FIELD_BITS + e_acc_0 = 0 + e_acc_1 = 0 + for k in unroll(0, BASE_FIELD_BITS): + ck = c_table[GEN ** k] + e_acc_0 += ck * prod_final_0[GEN ** k] + e_acc_1 += ck * prod_final_1[GEN ** k] + rs_eq_vals[0] = e_acc_0 + rs_eq_vals[1] = e_acc_1 # ring-switch weight: extend by the selector bits over the fold_challenges # coords [qpkdv, lenris). rs_weight = gamma_ab * rs_eq_vals[0] + gamma_c * rs_eq_vals[1] @@ -1939,21 +2173,16 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): rs_bit = rs_sel_bits[xk] rsw_chain[xk * GEN] = rsw_chain[xk] * (1 + rs_bit + ris_q[xk]) rs_weight = rsw_chain[rs_len_g] - # Every weight is evaluated at the one fold point, so a claim contributes - # cw_j times eq(its tail coordinates, tail_challenges): coord_k = m_k * ov_k - # + (1 + m_k) * bit_k with mask bits m_k = [k < NOVER], read from the baked - # prefix-mask row nover. The product unrolls to the global cap; coordinates - # at k >= yr_log_n are ASSERTED zero below and the tail challenges are pinned - # to zero there, so those factors are 1 and nothing reads an unwritten cell. - # The ring-switch slot is the same, with no overlaps, and comes from the same - # certified QPKD offset. The transmitted message enters once, at the end. + # Evaluate every transparent weight at the one terminal fold point. Claim + # j contributes cw_j * eq(slot_point_j, tail_challenges); the transmitted + # final message is evaluated once and multiplied into their combined weight. inner_sum = inner_total for j in unroll(0, N_CLAIMS): - slot_point = HeapBuf(YR_LOG_CAP) + overlap_ptr = rho * claim_low_len[GEN ** j] if CLAIM_POINT_BUF[j] == POINT_BUF_ZETA: overlap_ptr = zeta * claim_low_len[GEN ** j] - else: - overlap_ptr = rho * claim_low_len[GEN ** j] + if CLAIM_POINT_BUF[j] == POINT_BUF_QPKD: + overlap_ptr = zeta * claim_low_len[GEN ** j] # overlap_ptr[g^k] reads the claim point at low_len + k, which is written # only for k < nover (the [low_len, cplen) span); at k >= nover it points # into the unwritten point-buffer gap (prover-chosen free cells). The @@ -1965,10 +2194,12 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): mask_row = prefix_mask_table * claim_nover[GEN ** j] ** YR_LOG_CAP # row nover: g^(nover * cap) claim_offset_bits = col_offset_bits * GEN ** ((SIZE_BITS + YR_LOG_CAP) * CLAIM_COMMITTED_COL[j]) residual_offset_bits = claim_offset_bits * fold_cap_g + tail_eq = GEN ** 0 for k in unroll(0, YR_LOG_CAP): mask_bit = mask_row[GEN ** k] slot_bit = residual_offset_bits[GEN ** k] - slot_point[GEN ** k] = mask_bit * overlap_ptr[GEN ** k] + (1 + mask_bit) * slot_bit + slot_coord = mask_bit * overlap_ptr[GEN ** k] + (1 + mask_bit) * slot_bit + tail_eq *= (1 + slot_coord + tail_challenges[GEN ** k]) # zero-pin coords beyond final_msg's log-length (no over-cap weight): the # pointers start at yr_log_n. The zero asserts double as the # nover <= yr_log_n pin: a larger nover selects a row whose prefix @@ -1980,21 +2211,15 @@ def verify_sub(pi_0, pi_1, seed_0, seed_1, g_logs_pow2, g_squares, defer_out): for xk in mul_range(1, yr_pad_g): assert hi_mask[xk] == 0 assert hi_slot[xk] == 0 - tail_eq = GEN ** 0 - for k in unroll(0, YR_LOG_CAP): - tail_eq *= (1 + slot_point[GEN ** k] + tail_challenges[GEN ** k]) inner_sum += claim_weights[GEN ** j] * tail_eq - rs_slot_point = HeapBuf(YR_LOG_CAP) rs_yslot_bits = qpkd_offset_bits * fold_cap_g + rs_tail_eq = GEN ** 0 for k in unroll(0, YR_LOG_CAP): yb = rs_yslot_bits[GEN ** k] - rs_slot_point[GEN ** k] = yb + rs_tail_eq *= (1 + yb + tail_challenges[GEN ** k]) rs_hi = rs_yslot_bits * yr_log_n_g for xk in mul_range(1, yr_pad_g): assert rs_hi[xk] == 0 # zero-pin coords beyond final_msg's log-length - rs_tail_eq = GEN ** 0 - for k in unroll(0, YR_LOG_CAP): - rs_tail_eq *= (1 + rs_slot_point[GEN ** k] + tail_challenges[GEN ** k]) inner_sum += rs_weight * rs_tail_eq assert inner_sum * yr_at_tail == sumcheck_target @@ -2071,20 +2296,18 @@ def main(): gamma_bc = StackBuf(NSUB) bc_running = 0 for t in unroll(0, NSUB): - agg_fs = squeeze(agg_fs) - gv = agg_fs[0] + agg_fs, gv = squeeze(agg_fs) gamma_bc[t] = gv bc_running += gv * defer[GEN ** (t * DEFER_SIZE + BYTECODE_LOG + LOG2_BYTECODE_COLS)] bc_point = HeapBuf(BYTECODE_VARS) for rd in unroll(0, BYTECODE_VARS): agg_fs, msg_g1, c = fs_next(agg_fs, bc_sumcheck_msgs * GEN ** (2 * rd)) agg_fs, msg_ginf, c = fs_next(agg_fs, c) - agg_fs = squeeze(agg_fs) - rv = agg_fs[0] + agg_fs, rv = squeeze(agg_fs) bc_point[GEN ** rd] = rv g_zero = bc_running + msg_g1 c_one = g_zero + msg_g1 + msg_ginf - bc_running = msg_ginf * rv * rv + c_one * rv + g_zero # fold the degree-2 batching-sumcheck round at rv + bc_running = (msg_ginf * rv + c_one) * rv + g_zero # fold the degree-2 batching-sumcheck round at rv # terminal: W(r*) in-circuit; the reduced bytecode claim B(r*) is deferred. bc_weight = 0 for t in unroll(0, NSUB): @@ -2102,20 +2325,18 @@ def main(): gamma_mat = StackBuf(NSUB) mat_running = 0 for t in unroll(0, NSUB): - agg_fs = squeeze(agg_fs) - gv = agg_fs[0] + agg_fs, gv = squeeze(agg_fs) gamma_mat[t] = gv mat_running += gv * defer[GEN ** (t * DEFER_SIZE + BYTECODE_LOG + LOG2_BYTECODE_COLS + 3 + 2 ** K_SKIP + 2 * LINCHECK_ROUNDS)] mat_point = HeapBuf(2 * K_LOG) for rd in unroll(0, 2 * K_LOG): agg_fs, msg_g1, c = fs_next(agg_fs, mat_sumcheck_msgs * GEN ** (2 * rd)) agg_fs, msg_ginf, c = fs_next(agg_fs, c) - agg_fs = squeeze(agg_fs) - rv = agg_fs[0] + agg_fs, rv = squeeze(agg_fs) mat_point[GEN ** rd] = rv g_zero = mat_running + msg_g1 c_one = g_zero + msg_g1 + msg_ginf - mat_running = msg_ginf * rv * rv + c_one * rv + g_zero + mat_running = (msg_ginf * rv + c_one) * rv + g_zero # terminal weights: U_t(r*) = urow_t(r*_row) * wcol_t(r*_col), with # row_weight = (sum_i L_i(zz_t) eq(r*[0..6], i)) * eq(zrho_t, r*[6..K_LOG]) and # col_weight = (sum_i z_partial_t[i] eq(r*[K_LOG..K_LOG+6], i)) * prod_j (1 + lrr_j diff --git a/crates/rec_aggregation/guests/xmss_aggregate.py b/crates/rec_aggregation/guests/xmss_aggregate.py index db0cf7e5..7f4f5f07 100644 --- a/crates/rec_aggregation/guests/xmss_aggregate.py +++ b/crates/rec_aggregation/guests/xmss_aggregate.py @@ -1,5 +1,5 @@ # The in-VM XMSS aggregation verifier. The public input is the full 32-byte -# protocol-specific streaming accumulator (IV = g^{num_bytes} | 0^16) of +# protocol-specific streaming accumulator (IV = g^{num_bytes} | 0^24) of # message | tweaks | merkle_bits | public keys, # num_bytes = FIXED_BYTES + 32·n, with n (the signature count) hinted in the # exponent and range checked. The fixed part (FIXED_BLOCKS blocks) is hinted, @@ -9,7 +9,10 @@ # all). The IV size element is computed from n directly (the loop absorbs # exactly n pk blocks, so it needs no separate hint or consistency check). # -# Tweak table layout (word index in `tweak_table`): +# BLAKE3 operands use the canonical 128-bit subspace of F192, so every 16-byte +# native value (tweak, digest pair, chain tip, sibling, pp) is one cell with a +# zero top limb, and a 32-byte hash block is two cells. +# Tweak table layout (tweak index t at cell g^{t}): # 0 : encoding tweak # 1 + CHAIN_STEPS·i + s : chain tweak, chain i < V, step s < CHAIN_STEPS # WOTS_PK_TWEAK_IDX : wots-pk tweak @@ -27,24 +30,35 @@ CHAIN_STEPS = CHAIN_LENGTH - 1 # hash steps / tweaks per chain WOTS_PK_PAIRS = V / 2 # tip pairs hashed into the WOTS leaf -WORDS_PER_BLOCK = 2 # a 256-bit value = two field words … -BYTES_PER_BLOCK = 32 # … = 32 bytes (one accumulator block) +WORDS_PER_VALUE = 1 # a 16-byte native value = one BLAKE3 cell … +WORDS_PER_BLOCK = 2 # … and a 32-byte accumulator block = two +BYTES_PER_BLOCK = 32 -# Tweak table (word layout): encoding | V·CHAIN_STEPS chain | wots-pk | merkle. -N_TWEAK_WORDS = 1 + V * CHAIN_STEPS + 1 + LOG_LIFETIME -N_TWEAK_BLOCKS = N_TWEAK_WORDS / WORDS_PER_BLOCK -WOTS_PK_TWEAK_IDX = 1 + V * CHAIN_STEPS # word index of the wots-pk tweak -MERKLE_TWEAK_IDX = WOTS_PK_TWEAK_IDX + 1 # word index of merkle level 0 +# The tower generator Y. `acc_lo + acc_hi·Y` embeds 128 BLAKE3 bits in F192. +Y = 18446744073709551616 -MERKLE_BIT_WORDS = LOG_LIFETIME # one bit word per merkle level -MERKLE_BIT_BLOCKS = LOG_LIFETIME / WORDS_PER_BLOCK +# Tweak table (one 1-cell tweak per index): encoding | V·CHAIN_STEPS chain | +# wots-pk | merkle. +N_TWEAKS = 1 + V * CHAIN_STEPS + 1 + LOG_LIFETIME +N_TWEAK_CELLS = WORDS_PER_VALUE * N_TWEAKS +N_TWEAK_BLOCKS = N_TWEAKS / 2 # two tweaks per absorbed block +WOTS_PK_TWEAK_IDX = 1 + V * CHAIN_STEPS # tweak index of the wots-pk tweak +MERKLE_TWEAK_IDX = WOTS_PK_TWEAK_IDX + 1 # tweak index of merkle level 0 + +MERKLE_BIT_CELLS = WORDS_PER_VALUE * LOG_LIFETIME # one 1-cell bit word per level +MERKLE_BIT_BLOCKS = LOG_LIFETIME / 2 # Absorbed fixed preamble = message (1 block) | tweaks | merkle bits. FIXED_BLOCKS = 1 + N_TWEAK_BLOCKS + MERKLE_BIT_BLOCKS FIXED_BYTES = FIXED_BLOCKS * BYTES_PER_BLOCK -# Sub-hash IV byte counts (num_bytes = #blocks · 32). -WOTS_PK_BLOCKS = (2 + V) / 4 # prefix (tweak, pp) + V tips, four words per BLAKE3 block +# Digits packed per digest lane: W bits each in GF(2^64)'s monomial budget +# (the lane's leftover top bits are ground to zero by the signer). +DIGITS_PER_WORD = V / 2 + +TIP_CELLS = WORDS_PER_VALUE * V # the V chain tips, one cell each + +WOTS_PK_BLOCKS = (2 + V) / 4 # prefix (tweak, pp) + V tips, four cells per BLAKE3 block N_SIGS_BOUND = 2 ** 16 # range cap for the hinted batch count @@ -68,15 +82,16 @@ def main(): pk_bytes = pk_bytes * pk_bytes # g^{16·n_sigs} pk_bytes = pk_bytes * pk_bytes # g^{32·n_sigs} iv = StackBuf(WORDS_PER_BLOCK) - iv[0] = GEN ** FIXED_BYTES * pk_bytes # g^{FIXED_BYTES + 32·n_sigs} + iv[0] = GEN ** FIXED_BYTES * pk_bytes # cell 0 = g^{FIXED_BYTES + 32·n_sigs} | 0 iv[1] = 0 - # Buffers for the common (per-batch) data — message, the tweak table, the - # merkle-bit decomposition. Each block below is hinted, absorbed into the - # hash, AND stored, which binds it to the public input. + # Buffers for the common (per-batch) data — message, the tweak table (1 + # cell per tweak), the merkle-bit decomposition (1 cell per level). Each + # block below is hinted, absorbed into the hash, AND stored, which binds it + # to the public input. message = HeapBuf(WORDS_PER_BLOCK) - tweak_table = HeapBuf(N_TWEAK_WORDS) - merkle_bits = HeapBuf(MERKLE_BIT_WORDS) + tweak_table = HeapBuf(N_TWEAK_CELLS) + merkle_bits = HeapBuf(MERKLE_BIT_CELLS) msg_block = StackBuf(WORDS_PER_BLOCK) hint_witness(msg_block, "msg") @@ -85,51 +100,55 @@ def main(): state = StackBuf(WORDS_PER_BLOCK) blake3(iv, msg_block, state) - tweak_slot = 1 # tweak_table word cursor: g^0, g^2, g^4, … (2 words / block) + # Block t fills cells g^{2t}..g^{2t+1}: compile-time indexes, so every + # store is a single DEREF (the offset rides the beta immediate). for t in unroll(0, N_TWEAK_BLOCKS): block = StackBuf(WORDS_PER_BLOCK) hint_witness(block, "tweaks") - tweak_table[tweak_slot] = block[0] - tweak_table[tweak_slot * GEN] = block[1] - tweak_slot = tweak_slot * GEN ** WORDS_PER_BLOCK + tweak_table[GEN ** (WORDS_PER_BLOCK * t)] = block[0] + tweak_table[GEN ** (WORDS_PER_BLOCK * t + 1)] = block[1] next_state = StackBuf(WORDS_PER_BLOCK) blake3(state, block, next_state) state = next_state - bit_slot = 1 for u in unroll(0, MERKLE_BIT_BLOCKS): block = StackBuf(WORDS_PER_BLOCK) hint_witness(block, "merkle_bits") - merkle_bits[bit_slot] = block[0] - merkle_bits[bit_slot * GEN] = block[1] - bit_slot = bit_slot * GEN ** WORDS_PER_BLOCK + merkle_bits[GEN ** (WORDS_PER_BLOCK * u)] = block[0] + merkle_bits[GEN ** (WORDS_PER_BLOCK * u + 1)] = block[1] next_state = StackBuf(WORDS_PER_BLOCK) blake3(state, block, next_state) state = next_state # Per-signature buffers, sized in the exponent from n_sigs (see HeapBuf - # docs): the MD state and each pk pair take a 2-cell slot per signature - # (slot k at g^{2k}, g^{2k+1}), so n_sigs·n_sigs·g^2 cells. - agg_states = HeapBuf(n_sigs * n_sigs * GEN ** WORDS_PER_BLOCK) - pubkeys = HeapBuf(n_sigs * n_sigs * GEN ** WORDS_PER_BLOCK) + # docs): the MD state and each pk block take a 2-cell slot per signature + # (slot k at g^{2k}..g^{2k+1}), so n_sigs^2·g^2 cells. + n_sigs_2 = n_sigs * n_sigs # g^{2·n_sigs} + agg_states = HeapBuf(n_sigs_2 * GEN ** WORDS_PER_BLOCK) + pubkeys = HeapBuf(n_sigs_2 * GEN ** WORDS_PER_BLOCK) agg_states[1] = state[0] agg_states[GEN] = state[1] for j in mul_range(1, n_sigs): - slot = j * j # signature k occupies cells g^{2k}, g^{2k+1} - hint_witness(pubkeys[slot:slot + WORDS_PER_BLOCK], "pks") - blake3(agg_states[slot:slot + WORDS_PER_BLOCK], pubkeys[slot:slot + WORDS_PER_BLOCK], agg_states[slot * GEN ** WORDS_PER_BLOCK:slot * GEN ** WORDS_PER_BLOCK + WORDS_PER_BLOCK]) - verify_sig(message, tweak_table, merkle_bits, pubkeys * slot) + slot = j * j # signature k occupies cells g^{2k}..g^{2k+1} + # Name the two slot pointers once; the slices off them are then + # compile-time (beta) offsets, with no per-operand pointer MUL. + sig_state = agg_states * slot + sig_pk = pubkeys * slot + hint_witness(sig_pk[0:2], "pks") + blake3(sig_state[0:2], sig_pk[0:2], sig_state[2:4]) + verify_sig(message, tweak_table, merkle_bits, sig_pk) - # Publish the final MD state = the aggregation public input. - final_slot = n_sigs * n_sigs + # Publish the final MD state (two 128-bit cells) = the aggregation public input. + final_ptr = agg_states * n_sigs_2 public_input = GEN ** 0 - public_input[1] = agg_states[final_slot] - public_input[GEN] = agg_states[final_slot * GEN] + public_input[1] = final_ptr[1] + public_input[GEN] = final_ptr[GEN] return def verify_sig(message, tweak_table, merkle_bits, pk_ptr): - # pk_ptr[1] is the signer's merkle root, pk_ptr[GEN] its public parameter. + # pk_ptr[g^0] is the signer's merkle root, pk_ptr[g^1] its public parameter + # (one 128-bit cell each). pp = pk_ptr[GEN] # Encoding digest D = BLAKE3(tweak | pp | msg | randomness | zero-pad), 96 bytes: @@ -153,37 +172,51 @@ def verify_sig(message, tweak_table, merkle_bits, pk_ptr): # V WOTS chains. Per chain: the digit is hinted in the exponent (g^{e_i}), # range checked, and dispatched once — arm k walks the remaining - # CHAIN_STEPS-k steps and returns the tip plus the digit literal. The + # CHAIN_STEPS-k steps and returns the tip cell plus the digit literal. The # product of the digits is the target sum (g^{Σe_i}); the digits, weighted - # by CHAIN_LENGTH^i, reconstruct D. - tips = StackBuf(V) + # by CHAIN_LENGTH^i inside each 64-bit lane (DIGITS_PER_WORD digits per + # lane, GF(2^64)'s monomial budget, with each lane's leftover top bits + # ground to zero by the signer), reconstruct the two lanes of D's first + # cell, combined as `acc_lo + acc_hi·Y`. + tips = StackBuf(TIP_CELLS) digit_product = 1 - encoding_acc = 0 + chain_tweaks = tweak_table * GEN ** WORDS_PER_VALUE # chain i's tweaks start at cell (1+CHAIN_STEPS·i) + acc_lo = 0 + weight = 1 + for i in unroll(0, DIGITS_PER_WORD): + digit = StackBuf(1) + hint_witness(digit[0:1], "digits") + assert log(digit[0]) < CHAIN_LENGTH + chain_start = StackBuf(1) + hint_witness(chain_start, "chain_starts") + t, e = match_range(log(digit[0]), range(0, CHAIN_LENGTH), lambda k: walk(chain_start[0], chain_tweaks, pp, k)) + tips[i] = t + digit_product = digit_product * digit[0] + acc_lo = acc_lo + e * weight # e_i in its monomial subspace of lane 0 + weight = weight * CHAIN_LENGTH + chain_tweaks = chain_tweaks * GEN ** (WORDS_PER_VALUE * CHAIN_STEPS) + acc_hi = 0 weight = 1 - chain_tweaks = tweak_table * GEN # chain i's tweaks start at g^{1+CHAIN_STEPS·i} - for i in unroll(0, V): - # The encoding digit e_i for chain i, hinted in the exponent (g^{e_i}); - # log(digit[0]) = e_i, which the match dispatches on. + for i in unroll(DIGITS_PER_WORD, V): digit = StackBuf(1) hint_witness(digit[0:1], "digits") assert log(digit[0]) < CHAIN_LENGTH - # The signature's chain value for chain i — the start of the walk the - # arm hashes forward (CHAIN_STEPS-k steps) to the public-key tip. chain_start = StackBuf(1) - hint_witness(chain_start[0:1], "chain_starts") - tip, e = match_range(log(digit[0]), range(0, CHAIN_LENGTH), lambda k: walk(chain_start[0], chain_tweaks, pp, k)) - tips[i] = tip + hint_witness(chain_start, "chain_starts") + t, e = match_range(log(digit[0]), range(0, CHAIN_LENGTH), lambda k: walk(chain_start[0], chain_tweaks, pp, k)) + tips[i] = t digit_product = digit_product * digit[0] - encoding_acc = encoding_acc + e * weight # e_i in its monomial subspace + acc_hi = acc_hi + e * weight # e_i in its monomial subspace of lane 1 weight = weight * CHAIN_LENGTH - chain_tweaks = chain_tweaks * GEN ** CHAIN_STEPS + chain_tweaks = chain_tweaks * GEN ** (WORDS_PER_VALUE * CHAIN_STEPS) assert digit_product == GEN ** TARGET_SUM - assert encoding_acc == digest[0] + # Both lanes packed into D's first 128-bit cell. + assert acc_lo + acc_hi * Y == digest[0] # WOTS public-key leaf = standard BLAKE3 over prefix + 42 tips (704 bytes): # 11 full blocks, carrying the chaining value between instructions. pk_tweak_pp = StackBuf(WORDS_PER_BLOCK) - pk_tweak_pp[0] = tweak_table[GEN ** WOTS_PK_TWEAK_IDX] + pk_tweak_pp[0] = tweak_table[GEN ** (WORDS_PER_VALUE * WOTS_PK_TWEAK_IDX)] pk_tweak_pp[1] = pp leaf = StackBuf(WORDS_PER_BLOCK) blake3(pk_tweak_pp, tips[0:2], leaf, step=0) @@ -193,14 +226,14 @@ def verify_sig(message, tweak_table, merkle_bits, pk_ptr): leaf = next_leaf # Merkle path from the leaf to the root: the hinted slot bit orders the - # two children at each level; the tweak comes from the bound table. + # two children at each level; the tweak comes from the bound table. Level + # l reads bit cell l and tweak cell (MERKLE_TWEAK_IDX+l): compile-time + # (beta) indexes, one DEREF each. node = leaf[0] - level = 1 - merkle_tweaks = tweak_table * GEN ** MERKLE_TWEAK_IDX for l in unroll(0, LOG_LIFETIME): - bit = merkle_bits[level] + bit = merkle_bits[GEN ** (WORDS_PER_VALUE * l)] sibling = StackBuf(1) - hint_witness(sibling[0:1], "siblings") + hint_witness(sibling, "siblings") # Branchless child ordering: bit ∈ {0,1} (bound by the hash), so the # swap is a select, not a branch. m = bit·(node⊕sibling) is 0 when # bit=0 and node⊕sibling when bit=1, so children[0] = node⊕m is node @@ -211,33 +244,29 @@ def verify_sig(message, tweak_table, merkle_bits, pk_ptr): children[0] = node + m children[1] = sibling[0] + m merkle_tweak_pp = StackBuf(WORDS_PER_BLOCK) - merkle_tweak_pp[0] = merkle_tweaks[level] + merkle_tweak_pp[0] = tweak_table[GEN ** (WORDS_PER_VALUE * (MERKLE_TWEAK_IDX + l))] merkle_tweak_pp[1] = pp parent = StackBuf(WORDS_PER_BLOCK) blake3(merkle_tweak_pp, children, parent) node = parent[0] - level = level * GEN assert node == pk_ptr[1] return def walk(value, chain_tweaks, pp, k: Const): - # Walk WOTS chain steps k..CHAIN_STEPS-1: value' = H(tweak|pp, value|0), the - # step tweaks read off the bound subtable (cursor advanced to step k first). - tweak_cur = chain_tweaks - for a in unroll(0, k): - tweak_cur = tweak_cur * GEN + # Walk WOTS chain steps k..CHAIN_STEPS-1: value' = H(tweak|pp, value|0). + # Step s reads its tweak at cell s off the chain's subtable: a compile-time + # (beta) offset, one DEREF each; no cursor to advance. block = StackBuf(WORDS_PER_BLOCK) block[0] = value block[1] = 0 for s in unroll(k, CHAIN_STEPS): step_tweak = StackBuf(WORDS_PER_BLOCK) - step_tweak[0] = tweak_cur[1] + step_tweak[0] = chain_tweaks[GEN ** (WORDS_PER_VALUE * s)] step_tweak[1] = pp out = StackBuf(WORDS_PER_BLOCK) blake3(step_tweak, block, out, block_len=48) block = StackBuf(WORDS_PER_BLOCK) block[0] = out[0] block[1] = 0 - tweak_cur = tweak_cur * GEN return block[0], k diff --git a/crates/rec_aggregation/src/fibonacci.rs b/crates/rec_aggregation/src/fibonacci.rs index 17333342..9bb6ea75 100644 --- a/crates/rec_aggregation/src/fibonacci.rs +++ b/crates/rec_aggregation/src/fibonacci.rs @@ -6,15 +6,15 @@ use std::time::Instant; use lean_compiler::{compile, parse}; use lean_vm::cpu::{prove, verify}; use primitives::{ - field::{F128, g_pow}, + field::{F64, F192, g_pow}, pretty_f64, pretty_integer, }; /// Prove and verify Fibonacci-in-the-exponent over a `HeapBuf` (an unrolled /// `mul_range` recurrence), binding `g^{F(n)}` as the public input. Prints the /// benchmark report. -pub fn run_fibonacci(n: usize) { - let trace_span = tracing::info_span!("Fibonacci", n = %pretty_integer(n)).entered(); +pub fn run_fibonacci(n: usize, log_inv_rate: usize) { + let trace_span = tracing::info_span!("Fibonacci", n, log_inv_rate).entered(); let (src, pi) = fibonacci_program(n); let program = compile(&parse(&src).unwrap()); @@ -26,7 +26,7 @@ pub fn run_fibonacci(n: usize) { lean_vm::blake3_flock::warm_setup(0); let t = Instant::now(); - let (proof, stats) = prove(&program, pi); + let (proof, stats) = prove(&program, pi, log_inv_rate); let t_prove = t.elapsed(); let t = Instant::now(); verify(&program, &pi, &proof).unwrap(); @@ -38,11 +38,11 @@ pub fn run_fibonacci(n: usize) { drop(trace_span); println!( - "Fibonacci (in the exponent, i.e. modulo 2^128 - 1), N = {}", + "Fibonacci (in the exponent, i.e. modulo 2^64 - 1), N = {}", pretty_integer(n) ); println!(" cycles (VM steps) : {}", pretty_integer(stats.cycles)); - for (name, &c) in ["XOR", "MUL", "SET", "DEREF", "JUMP", "BLAKE3"] + for (name, &c) in ["XOR", "MUL", "SET", "DEREF", "JUMP", "BLAKE3", "PACK64X2"] .iter() .zip(&stats.counts) { @@ -57,18 +57,13 @@ pub fn run_fibonacci(n: usize) { " committed witness size : 2^{}", pretty_f64((stats.committed as f64).log2()) ); - println!( - " proof size : {} KiB", - pretty_f64(proof_bytes as f64 / 1024.0) - ); - println!( - " proving (incl. witness gen) : {} s", - pretty_f64(t_prove.as_secs_f64()) - ); - println!(" verifying : {} s", pretty_f64(t_verify.as_secs_f64())); + println!(" proof size : {:.1} KiB", proof_bytes as f64 / 1024.0); + println!(" proving (incl. witness gen) : {t_prove:?}"); + println!(" verifying : {t_verify:?}"); + let cycles_per_second = (stats.cycles as f64 / t_prove.as_secs_f64()).round() as u64; println!( " throughput : {} cycles/s", - pretty_f64(stats.cycles as f64 / t_prove.as_secs_f64()) + pretty_integer(cycles_per_second) ); } @@ -76,7 +71,7 @@ pub fn run_fibonacci(n: usize) { /// unrolled `mul_range` loop over a `HeapBuf`), with the result `g^{F(N)}` /// published into cell `m[0]`. Returns the zkDSL source and the public input /// `[g^{F(N)}, 0]`. -fn fibonacci_program(fib_n: usize) -> (String, [F128; 2]) { +fn fibonacci_program(fib_n: usize) -> (String, [F192; 2]) { const UNROLL: usize = 1000; assert!( fib_n >= UNROLL && fib_n.is_multiple_of(UNROLL), @@ -86,13 +81,13 @@ fn fibonacci_program(fib_n: usize) -> (String, [F128; 2]) { // Run the recurrence in the field (the same one the VM runs in the exponent) // to pin the result g^{F(N)}, the public input. - let (mut a, mut b) = (F128::ONE, g_pow(1)); // g^{F(0)}, g^{F(1)} + let (mut a, mut b) = (F64::ONE, g_pow(1)); // g^{F(0)}, g^{F(1)} for _ in 1..=fib_n { let c = a * b; a = b; b = c; // (a, b) = (g^{F(m)}, g^{F(m+1)}) } - let pi = [a, F128::ZERO]; // a = g^{F(N)}: the result, then 0 + let pi = [F192::from(a), F192::ZERO]; // a = g^{F(N)}: the result, then 0 // `K` blocks: each reads its boundary pair into locals, runs `UNROLL` // Fibonacci `MUL`s in registers, and writes the next pair (4 DEREFs per @@ -129,6 +124,6 @@ fn fibonacci_program(fib_n: usize) -> (String, [F128; 2]) { mod tests { #[test] fn fibonacci() { - super::run_fibonacci(200_000); + super::run_fibonacci(200_000, lean_vm::pcs::LOG_INV_RATE); } } diff --git a/crates/rec_aggregation/src/lib.rs b/crates/rec_aggregation/src/lib.rs index 2da5811e..280d8e50 100644 --- a/crates/rec_aggregation/src/lib.rs +++ b/crates/rec_aggregation/src/lib.rs @@ -4,6 +4,14 @@ //! `#[cfg(test)]` suites in each module drive the same entry points. pub mod fibonacci; +// The n→1 recursion harness dissects the proof stream word-for-word to build the +// guest's hint streams. It is fully ported to the F64-committed machine (each +// extension scalar is one F192 memory word / three committed K limbs, the +// sponge state is four `F64` lanes, and the opening is stacked Ligerito), and +// the guest (`guests/recursion.py`) +// replays the single-field tower verifier. Verified end-to-end by +// `recursion_2to1` (honest proofs accept) and `recursion_soundness_binds` +// (tampered hints reject). pub mod recursion; pub mod signers_cache; pub mod xmss_aggregation; diff --git a/crates/rec_aggregation/src/recursion.rs b/crates/rec_aggregation/src/recursion.rs index 664be830..48d5d473 100644 --- a/crates/rec_aggregation/src/recursion.rs +++ b/crates/rec_aggregation/src/recursion.rs @@ -4,26 +4,24 @@ //! statements + the three reduced claims (stacked bytecode, A0, B0) to its own //! public input (doc.tex §Recursive aggregation, §Deferred evaluation claims). //! -//! Zero hand-mirroring: the transcript trace of a REAL `cpu::verify` run -//! (`transcript::trace_start`/`trace_take`) is the guest's mechanical spec — -//! `gen_verify` walks it structurally (a `Walk` cursor; `Sponge::replay` yields -//! the checkpoint states) to extract every hint value, and the real -//! `cpu::layout` supplies every compile-time shape. `gen_agg` mirrors the -//! guest's aggregation transcript and runs the two batching-sumcheck provers -//! (dense for the bytecode, two-phase sparse for the flock matrices). +//! The transcript trace of a real `cpu::verify` run +//! (`transcript::trace_start`/`trace_take`) keeps the native and guest verifiers +//! synchronized: `gen_verify` walks it structurally, while `cpu::layout` +//! supplies every compile-time shape. `aggregate_deferred_claims` builds the guest's aggregation +//! transcript and the two batching-sumcheck proofs. //! [`RecursiveProof::verify`] is the only public acceptance path: it verifies //! the outer VM proof and evaluates every deferred fixed polynomial. use std::collections::BTreeMap; -use pcs::ligerito::log2_ceil; use lean_compiler::{compile, parse, parse_with_replacements}; use lean_vm::cpu::{Program, prove, verify}; use lean_vm::leaf::{Block, Coord}; use lean_vm::transcript::{Sponge, TraceOp, trace_start, trace_take}; +use pcs::ligerito::log2_ceil; +use primitives::multilinear::mle_eval; use primitives::{ - field::{F128, G, g_pow}, - multilinear::mle_eval, + field::{F64, F192, G, g_pow}, pretty_f64, pretty_integer, }; @@ -35,8 +33,40 @@ const RECURSION_AGG_LABEL: &[u8] = b"leanvm-b/recursion-aggregation/v1"; const RECURSION_STATEMENT_LABEL: &[u8] = b"leanvm-b/recursive-statement/v1"; /// A field element as the decimal `u128` literal the zkDSL parser accepts. -fn u(f: F128) -> u128 { - (f.lo as u128) | ((f.hi as u128) << 64) +fn u(f: F192) -> u128 { + assert_eq!(f.c2, 0, "u128 DSL literal cannot encode the top F192 limb"); + (f.c0 as u128) | ((f.c1 as u128) << 64) +} + +fn f192_literal(f: F192) -> String { + format!("f192({},{},{})", f.c0, f.c1, f.c2) +} + +/// Native replay of the VM's `blake3(cur, cur, nxt)` over two 128-bit words: +/// pack the two `F192` words into the four `F64` lanes the sponge compression +/// consumes, compress, and unpack. +/// +/// Word→lane packing confirmed against the VM's blake3 opcode (`cpu::mod` +/// `blake3_self_hash_aliased_operands`): a `[F64;4]` operand loaded from two +/// 128-bit words is `[w0.c0, w0.c1, w1.c0, w1.c1]` (word-major, lo=c0 then +/// hi=c1), and the two output words pack back the same way +/// (`mem[out] == cell(d[0], d[1])`, `cell(d[2], d[3])`). +fn vmhash_compress2(st: [F192; 2]) -> [F192; 2] { + let inb = [F64(st[0].c0), F64(st[0].c1), F64(st[1].c0), F64(st[1].c1)]; + let out = lean_vm::vmhash::compress(inb, inb); + [F192::new(out[0].0, out[1].0, 0), F192::new(out[2].0, out[3].0, 0)] +} + +/// Pack the sponge's four K lanes as two canonical 128-bit VM cells. +fn pack_state(s: [F64; 4]) -> [F192; 2] { + [F192::new(s[0].0, s[1].0, 0), F192::new(s[2].0, s[3].0, 0)] +} + +/// Pack a 32-byte Merkle node as the same canonical 128+128 cell pair used by +/// the VM's sole BLAKE3 representation. +fn pack_hash_state(hash: &[u8; 32]) -> [F192; 2] { + let w = |o: usize| u64::from_le_bytes(hash[o..o + 8].try_into().unwrap()); + [F192::new(w(0), w(8), 0), F192::new(w(16), w(24), 0)] } /// The non-trivial inner program: a runtime-bounded BLAKE3 hash chain seeded @@ -45,7 +75,7 @@ fn u(f: F128) -> u128 { /// witness hints ("n_hash", "iters"), so a single program (one bytecode, one /// digest) proves runs with wildly different opcode profiles and sizes - the /// exact genericity the recursion guest is built for. Exercises every table -/// (XOR/MUL/SET/DEREF/JUMP/BLAKE3). +/// (XOR/MUL/SET/DEREF/JUMP/BLAKE3/PACK64X2). fn inner_program() -> Program { let src = "from snark_lib import *\n\ def main():\n\ @@ -90,79 +120,84 @@ fn inner_program() -> Program { /// Prove one run of the inner program: `hashes` BLAKE3 compressions then /// `iters` product-loop steps (both runtime, driven by the witness hints). /// The witness generator replays both natively to supply the final-inverse -/// hint. Returns (program, proof, guest-cycle count). -fn prove_inner(pi: [F128; 2], hashes: usize, iters: usize) -> (Program, lean_vm::cpu::Proof, usize) { +/// hint. Returns (program, proof, guest-cycle count, committed witness size). +fn prove_inner( + pi: [F192; 2], + hashes: usize, + iters: usize, + log_inv_rate: usize, +) -> (Program, lean_vm::cpu::Proof, usize, usize) { assert!(hashes >= 1 && iters >= 1, "both loops run at least once"); let mut program = inner_program(); // Replay natively: the hash chain, then the product loop, to fetch the // final accumulator (nonzero, for the hinted-inverse assert). let mut st = [pi[0], pi[1]]; for _ in 0..hashes { - st = lean_vm::vmhash::compress(st, st); + st = vmhash_compress2(st); } let mut acc = st[0]; - let mut x = F128::ONE; - let g = primitives::field::g_pow(1); + let mut x = F192::ONE; + let g = F192::new(primitives::field::g_pow(1).0, 0, 0); // embedded base generator for _ in 0..iters { let b = acc * acc + st[1]; acc = b + x; x *= g; } let out = acc; - assert!(out != F128::ZERO, "inner accumulator must be nonzero"); + assert!(out != F192::ZERO, "inner accumulator must be nonzero"); program.set_witness("outinv", vec![vec![out.inv()]]); - program.set_witness("n_hash", vec![vec![g_pow(hashes)]]); - program.set_witness("iters", vec![vec![g_pow(iters)]]); - let (proof, stats) = prove(&program, pi); + program.set_witness("n_hash", vec![vec![F192::new(g_pow(hashes).0, 0, 0)]]); + program.set_witness("iters", vec![vec![F192::new(g_pow(iters).0, 0, 0)]]); + let (proof, stats) = prove(&program, pi, log_inv_rate); eprintln!( "[inner] cycles={} committed=2^{}", pretty_integer(stats.cycles), pretty_f64((stats.committed as f64).log2()) ); - (program, proof, stats.cycles) + (program, proof, stats.cycles, stats.committed) } /// The deferred-claim data the guest binds to the outer public input: the outer /// verifier checks each claim natively (doc.tex §Deferred evaluation claims; /// n_rec = 1 forwards fresh claims without batching). -struct SubDefer { - pi: [F128; 2], - kbc: usize, - zeta: Vec, - sb: Vec, - wbc: Vec, - lc_alpha: F128, - zz: F128, - zrho8: Vec, - lrr: Vec, - lcz: Vec, - matpart: F128, +struct DeferredSubproof { + public_input: [F192; 2], + bytecode_log: usize, + bytecode_row_point: Vec, + bytecode_selector_point: Vec, + bytecode_value: F192, + matrix_a_coefficient: F192, + skip_point: F192, + zerocheck_row_point: Vec, + lincheck_round_point: Vec, + lincheck_terminal_values: Vec, + matrix_claim: F192, } -/// The batched reduced claims the aggregation exports: one point + value on +/// The deferred claims the aggregation exports: one point and value on /// the stacked bytecode polynomial, one point + two values on the flock /// matrices (doc.tex §Deferred evaluation claims). #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] -struct ReducedClaims { - r_bc: Vec, - v_bc: F128, - r_m: Vec, - v_a: F128, - v_b: F128, +struct DeferredClaims { + bytecode_point: Vec, + bytecode_value: F192, + matrix_point: Vec, + matrix_a_value: F192, + matrix_b_value: F192, } /// Everything committed by the outer public input. Keeping this private makes /// the deferred claims an implementation detail of recursive verification. #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] struct RecursiveStatement { - sub_statements: Vec<[F128; 2]>, - reduced: ReducedClaims, + sub_statements: Vec<[F192; 2]>, + reduced: DeferredClaims, } impl RecursiveStatement { - fn public_input(&self, inner_environment: [F128; 2]) -> [F128; 2] { + fn public_input(&self, inner_environment: [F192; 2]) -> [F192; 2] { let mut sponge = Sponge::new(RECURSION_STATEMENT_LABEL, &[]); - sponge.observe(F128::new(self.sub_statements.len() as u64, 0)); + sponge.observe(F192::new(self.sub_statements.len() as u64, 0, 0)); for &v in &inner_environment { sponge.observe(v); } @@ -171,16 +206,16 @@ impl RecursiveStatement { sponge.observe(v); } } - for &v in &self.reduced.r_bc { + for &v in &self.reduced.bytecode_point { sponge.observe(v); } - sponge.observe(self.reduced.v_bc); - for &v in &self.reduced.r_m { + sponge.observe(self.reduced.bytecode_value); + for &v in &self.reduced.matrix_point { sponge.observe(v); } - sponge.observe(self.reduced.v_a); - sponge.observe(self.reduced.v_b); - sponge.state() + sponge.observe(self.reduced.matrix_a_value); + sponge.observe(self.reduced.matrix_b_value); + pack_state(sponge.state()) } } @@ -197,7 +232,7 @@ pub struct RecursiveProof { impl RecursiveProof { /// Statements aggregated by this proof, in transcript order. - pub fn sub_statements(&self) -> &[[F128; 2]] { + pub fn sub_statements(&self) -> &[[F192; 2]] { &self.statement.sub_statements } @@ -207,11 +242,12 @@ impl RecursiveProof { if statement.sub_statements.is_empty() { return Err(RecursiveVerifyError::EmptyBatch); } - let guest = recursion_guest(inner_program, statement.sub_statements.len()); + // Verification only reads the compiled guest; prover witness streams + // live on owned clones. + let guest = recursion_guest_arc(inner_program, statement.sub_statements.len()); let public_input = statement.public_input(lean_vm::cpu::fs_seed(inner_program)); - verify(&guest, &public_input, &self.outer_proof) - .map_err(RecursiveVerifyError::OuterProof)?; - check_reduced(inner_program, &statement.reduced) + verify(&guest, &public_input, &self.outer_proof).map_err(RecursiveVerifyError::OuterProof)?; + check_deferred_claims(inner_program, &statement.reduced) } } @@ -225,7 +261,7 @@ pub enum RecursiveVerifyError { MatrixBClaim, } -fn fold_lsb(t: &mut Vec, r: F128) { +fn fold_lsb(t: &mut Vec, r: F192) { let half = t.len() / 2; for i in 0..half { t[i] = t[2 * i] + r * (t[2 * i] + t[2 * i + 1]); @@ -235,10 +271,10 @@ fn fold_lsb(t: &mut Vec, r: F128) { /// Compressed product-sumcheck round message over γ-weighted table pairs: /// (g1, g∞) with g0 recovered from the running claim. -fn round_msg(pairs: &[(&[F128], &[F128], F128)]) -> (F128, F128) { - let (mut g1, mut gi) = (F128::ZERO, F128::ZERO); +fn round_msg(pairs: &[(&[F192], &[F192], F192)]) -> (F192, F192) { + let (mut g1, mut gi) = (F192::ZERO, F192::ZERO); for &(u, m, gamma) in pairs { - let (mut a1, mut ai) = (F128::ZERO, F128::ZERO); + let (mut a1, mut ai) = (F192::ZERO, F192::ZERO); for i in 0..u.len() / 2 { a1 += u[2 * i + 1] * m[2 * i + 1]; ai += (u[2 * i] + u[2 * i + 1]) * (m[2 * i] + m[2 * i + 1]); @@ -251,82 +287,90 @@ fn round_msg(pairs: &[(&[F128], &[F128], F128)]) -> (F128, F128) { /// The stacked bytecode polynomial of the inner program (leaf's canonical /// table, built from the real layout). -fn stacked_bytecode(program: &Program) -> Vec { +fn stacked_bytecode(program: &Program) -> Vec { // Public bytecode coordinates depend only on the program. The remaining // layout inputs affect private witness/table shapes, so fixed valid dummy // sizes are sufficient and avoid retaining a representative inner proof. let l = lean_vm::cpu::layout( &program.prog, 20, - [1usize << 10; 6], - [F128::ZERO; 2], + [1usize << 10; lean_vm::tables::N_TABLES], + [F192::ZERO; 2], ); lean_vm::leaf::stacked_bytecode_table(&l.push) } -/// The aggregation layer: mirror the guest's aggregation transcript, run the -/// two batching-sumcheck PROVERS (dense bytecode; two-phase sparse matrices), -/// and return the round-message hints, the terminal hints, the reduced claims, -/// and the outer public input. +/// Mirror the guest's aggregation transcript and prove the two batching +/// sumchecks: dense for bytecode and two-phase sparse for the matrices. Returns +/// the guest hints, deferred claims, and outer public input. #[allow(clippy::type_complexity)] -fn gen_agg( +fn aggregate_deferred_claims( program: &Program, - subs: &[SubDefer], -) -> (Vec<(String, Vec)>, [F128; 2], ReducedClaims) { + subs: &[DeferredSubproof], +) -> (Vec<(String, Vec)>, [F192; 2], DeferredClaims) { let nsub = subs.len(); - let kbc = subs[0].kbc; - let kbcv = kbc + 3; + let kbc = subs[0].bytecode_log; + let kbcv = kbc + lean_vm::leaf::N_BYTECODE_SELECTORS; let klog = flock::blake3::K_LOG; // ---- the aggregation transcript (mirrors the guest exactly) ---- let mut h = Sponge::new(RECURSION_AGG_LABEL, &[]); - h.observe(F128::new(nsub as u64, 0)); + h.observe(F192::new(nsub as u64, 0, 0)); for d in subs { - h.observe(d.pi[0]); - h.observe(d.pi[1]); - for &v in &d.zeta { + h.observe(d.public_input[0]); + h.observe(d.public_input[1]); + for &v in &d.bytecode_row_point { h.observe(v); } - for &v in &d.sb { + for &v in &d.bytecode_selector_point { h.observe(v); } - for &v in &d.wbc { + h.observe(d.bytecode_value); + h.observe(d.matrix_a_coefficient); + h.observe(d.skip_point); + for &v in &d.zerocheck_row_point { h.observe(v); } - h.observe(d.lc_alpha); - h.observe(d.zz); - for &v in &d.zrho8 { + for &v in &d.lincheck_round_point { h.observe(v); } - for &v in &d.lrr { + for &v in &d.lincheck_terminal_values { h.observe(v); } - for &v in &d.lcz { - h.observe(v); - } - h.observe(d.matpart); + h.observe(d.matrix_claim); } // ---- bytecode batching sumcheck (dense, 2^kbcv; ONE claim per sub, at // the shared push/pull point) ---- - let gbc: Vec = (0..nsub).map(|_| h.sample()).collect(); - let mut bt = stacked_bytecode(program); - let mut wt = vec![F128::ZERO; 1 << kbcv]; - let points: Vec> = subs + let gbc: Vec = (0..nsub).map(|_| h.sample()).collect(); + let mut bt: Vec = stacked_bytecode(program) + .into_iter() + .map(|x| F192::new(x.0, 0, 0)) + .collect(); + let mut wt = vec![F192::ZERO; 1 << kbcv]; + let points: Vec> = subs .iter() - .map(|d| d.zeta.iter().chain(&d.sb).copied().collect::>()) + .map(|d| { + d.bytecode_row_point + .iter() + .chain(&d.bytecode_selector_point) + .copied() + .collect::>() + }) .collect(); for (t, p) in points.iter().enumerate() { - let eqt = primitives::multilinear::build_eq(p); + let eqt = pcs::ligerito::build_eq_table_ext(p); for (w, &e) in wt.iter_mut().zip(eqt.iter()) { *w += gbc[t] * e; } } - let mut brun: F128 = (0..nsub).map(|t| gbc[t] * subs[t].wbc[0]).fold(F128::ZERO, |a, x| a + x); + let mut brun: F192 = (0..nsub) + .map(|t| gbc[t] * subs[t].bytecode_value) + .fold(F192::ZERO, |a, x| a + x); let mut bscr = Vec::new(); let mut r_bc = Vec::new(); for _ in 0..kbcv { - let (g1, gi) = round_msg(&[(&bt, &wt, F128::ONE)]); + let (g1, gi) = round_msg(&[(&bt, &wt, F192::ONE)]); h.observe(g1); h.observe(gi); let r = h.sample(); @@ -334,7 +378,7 @@ fn gen_agg( r_bc.push(r); let g0 = brun + g1; let c1 = g0 + g1 + gi; - brun = gi * r * r + c1 * r + g0; + brun = (gi * r + c1) * r + g0; fold_lsb(&mut bt, r); fold_lsb(&mut wt, r); } @@ -342,54 +386,69 @@ fn gen_agg( assert_eq!(brun, v_bc * wt[0], "bytecode sumcheck terminal"); // ---- matrix batching sumcheck (two-phase sparse, per the probe) ---- - let gmt: Vec = (0..nsub).map(|_| h.sample()).collect(); + let gmt: Vec = (0..nsub).map(|_| h.sample()).collect(); let (ma, mb) = flock::blake3::matrices(); // per-claim dense weight tables: rows = quirky eq, cols = eq(top rounds) x z_partial. - let mut us: Vec> = subs + let mut us: Vec> = subs .iter() - .map(|d| flock::lincheck::build_quirky_eq_table(d.zz, &d.zrho8, 6)) + .map(|d| flock::lincheck::build_quirky_eq_table(d.skip_point, &d.zerocheck_row_point, 6)) .collect(); - let ws: Vec> = subs + let ws: Vec> = subs .iter() .map(|d| { (0..1usize << klog) .map(|c| { - let mut w = d.lcz[c & 63]; - for (j, &rj) in d.lrr.iter().enumerate() { + let mut w = d.lincheck_terminal_values[c & 63]; + for (j, &rj) in d.lincheck_round_point.iter().enumerate() { let bit = (c >> (klog - 1 - j)) & 1; - w *= if bit == 1 { rj } else { F128::ONE + rj }; + w *= if bit == 1 { rj } else { F192::ONE + rj }; } w }) .collect() }) .collect(); - let contract_cols = |m: &flock::r1cs::SparseBinaryMatrix, w: &[F128]| -> Vec { + let contract_cols = |m: &flock::r1cs::SparseBinaryMatrix, w: &[F192]| -> Vec { m.rows .iter() - .map(|row| row.iter().map(|&j| w[j]).fold(F128::ZERO, |a, x| a + x)) + .map(|row| row.iter().map(|&j| w[j]).fold(F192::ZERO, |a, x| a + x)) .collect() }; - let mut ms: Vec> = Vec::new(); + let mut ms: Vec> = Vec::new(); for w in &ws { ms.push(contract_cols(ma, w)); ms.push(contract_cols(mb, w)); } - let ga: Vec = (0..nsub).map(|t| gmt[t] * subs[t].lc_alpha).collect(); - let gb: Vec = gmt.clone(); - let mut mrun: F128 = (0..nsub).map(|t| gmt[t] * subs[t].matpart).fold(F128::ZERO, |a, x| a + x); + let ga: Vec = (0..nsub).map(|t| gmt[t] * subs[t].matrix_a_coefficient).collect(); + let gb: Vec = gmt.clone(); + let mut mrun: F192 = (0..nsub) + .map(|t| gmt[t] * subs[t].matrix_claim) + .fold(F192::ZERO, |a, x| a + x); // sanity: the deferred matpart equals the bilinear form over the matrices. for (t, d) in subs.iter().enumerate() { - let direct = d.lc_alpha - * ms[2 * t].iter().zip(&us[t]).map(|(&m, &u)| m * u).fold(F128::ZERO, |a, x| a + x) - + ms[2 * t + 1].iter().zip(&us[t]).map(|(&m, &u)| m * u).fold(F128::ZERO, |a, x| a + x); - assert_eq!(direct, d.matpart, "matpart bilinear identity, sub {t}"); + let direct = d.matrix_a_coefficient + * ms[2 * t] + .iter() + .zip(&us[t]) + .map(|(&m, &u)| m * u) + .fold(F192::ZERO, |a, x| a + x) + + ms[2 * t + 1] + .iter() + .zip(&us[t]) + .map(|(&m, &u)| m * u) + .fold(F192::ZERO, |a, x| a + x); + assert_eq!(direct, d.matrix_claim, "matrix bilinear identity, sub {t}"); } let mut mscr = Vec::new(); let mut r_row = Vec::new(); for _ in 0..klog { - let pairs: Vec<(&[F128], &[F128], F128)> = (0..nsub) - .flat_map(|t| [(&us[t][..], &ms[2 * t][..], ga[t]), (&us[t][..], &ms[2 * t + 1][..], gb[t])]) + let pairs: Vec<(&[F192], &[F192], F192)> = (0..nsub) + .flat_map(|t| { + [ + (&us[t][..], &ms[2 * t][..], ga[t]), + (&us[t][..], &ms[2 * t + 1][..], gb[t]), + ] + }) .collect(); let (g1, gi) = round_msg(&pairs); h.observe(g1); @@ -399,7 +458,7 @@ fn gen_agg( r_row.push(r); let g0 = mrun + g1; let c1 = g0 + g1 + gi; - mrun = gi * r * r + c1 * r + g0; + mrun = (gi * r + c1) * r + g0; for u in us.iter_mut() { fold_lsb(u, r); } @@ -407,9 +466,9 @@ fn gen_agg( fold_lsb(m, r); } } - let eq_rstar = primitives::multilinear::build_eq(&r_row); - let contract_rows = |m: &flock::r1cs::SparseBinaryMatrix| -> Vec { - let mut out = vec![F128::ZERO; 1 << klog]; + let eq_rstar = pcs::ligerito::build_eq_table_ext(&r_row); + let contract_rows = |m: &flock::r1cs::SparseBinaryMatrix| -> Vec { + let mut out = vec![F192::ZERO; 1 << klog]; for (i, row) in m.rows.iter().enumerate() { let e = eq_rstar[i]; for &j in row { @@ -420,8 +479,8 @@ fn gen_agg( }; let mut acol = contract_rows(ma); let mut bcol = contract_rows(mb); - let mut wa = vec![F128::ZERO; 1 << klog]; - let mut wb = vec![F128::ZERO; 1 << klog]; + let mut wa = vec![F192::ZERO; 1 << klog]; + let mut wb = vec![F192::ZERO; 1 << klog]; for t in 0..nsub { let (sa, sb2) = (ga[t] * us[t][0], gb[t] * us[t][0]); for j in 0..1 << klog { @@ -431,8 +490,7 @@ fn gen_agg( } let mut r_col = Vec::new(); for _ in 0..klog { - let pairs: Vec<(&[F128], &[F128], F128)> = - vec![(&acol, &wa, F128::ONE), (&bcol, &wb, F128::ONE)]; + let pairs: Vec<(&[F192], &[F192], F192)> = vec![(&acol, &wa, F192::ONE), (&bcol, &wb, F192::ONE)]; let (g1, gi) = round_msg(&pairs); h.observe(g1); h.observe(gi); @@ -441,7 +499,7 @@ fn gen_agg( r_col.push(r); let g0 = mrun + g1; let c1 = g0 + g1 + gi; - mrun = gi * r * r + c1 * r + g0; + mrun = (gi * r + c1) * r + g0; for tb in [&mut acol, &mut bcol, &mut wa, &mut wb] { fold_lsb(tb, r); } @@ -450,18 +508,20 @@ fn gen_agg( assert_eq!(mrun, v_a * wa[0] + v_b * wb[0], "matrix sumcheck terminal"); // sanity for the GUEST's succinct terminal-weight formulas. { - let eqr = primitives::multilinear::build_eq(&r_row[..6]); - let eqc = primitives::multilinear::build_eq(&r_col[..6]); - let (mut wam, mut wbm) = (F128::ZERO, F128::ZERO); + let eqr = pcs::ligerito::build_eq_table_ext(&r_row[..6]); + let eqc = pcs::ligerito::build_eq_table_ext(&r_col[..6]); + let (mut wam, mut wbm) = (F192::ZERO, F192::ZERO); for (t, d) in subs.iter().enumerate() { - let lam = flock::zerocheck::multilinear::lagrange_weights_naive(6, d.zz); - let mut urow: F128 = (0..64).map(|i| lam[i] * eqr[i]).fold(F128::ZERO, |a, x| a + x); - for (k, &z) in d.zrho8.iter().enumerate() { - urow *= F128::ONE + z + r_row[6 + k]; + let lam = primitives::multilinear::lagrange_weights_naive(6, d.skip_point); + let mut urow: F192 = (0..64).map(|i| lam[i] * eqr[i]).fold(F192::ZERO, |a, x| a + x); + for (k, &z) in d.zerocheck_row_point.iter().enumerate() { + urow *= F192::ONE + z + r_row[6 + k]; } - let mut wcol: F128 = (0..64).map(|i| d.lcz[i] * eqc[i]).fold(F128::ZERO, |a, x| a + x); - for (j, &rj) in d.lrr.iter().enumerate() { - wcol *= F128::ONE + rj + r_col[klog - 1 - j]; + let mut wcol: F192 = (0..64) + .map(|i| d.lincheck_terminal_values[i] * eqc[i]) + .fold(F192::ZERO, |a, x| a + x); + for (j, &rj) in d.lincheck_round_point.iter().enumerate() { + wcol *= F192::ONE + rj + r_col[klog - 1 - j]; } let u = urow * wcol; wam += ga[t] * u; @@ -476,18 +536,18 @@ fn gen_agg( // baked into the guest), so one compiled guest serves any inner program. let seed = lean_vm::cpu::fs_seed(program); let mut e = Sponge::new(RECURSION_STATEMENT_LABEL, &[]); - e.observe(F128::new(subs.len() as u64, 0)); + e.observe(F192::new(subs.len() as u64, 0, 0)); e.observe(seed[0]); e.observe(seed[1]); for d in subs { - e.observe(d.pi[0]); - e.observe(d.pi[1]); + e.observe(d.public_input[0]); + e.observe(d.public_input[1]); } for &v in &r_bc { e.observe(v); } e.observe(v_bc); - let r_m: Vec = r_row.iter().chain(&r_col).copied().collect(); + let r_m: Vec = r_row.iter().chain(&r_col).copied().collect(); for &v in &r_m { e.observe(v); } @@ -503,46 +563,38 @@ fn gen_agg( ]; ( hints, - e.state(), - ReducedClaims { - r_bc, - v_bc, - r_m, - v_a, - v_b, + pack_state(e.state()), + DeferredClaims { + bytecode_point: r_bc, + bytecode_value: v_bc, + matrix_point: r_m, + matrix_a_value: v_a, + matrix_b_value: v_b, }, ) } /// Discharge the three fixed-polynomial claims deferred by the guest. -fn check_reduced(program: &Program, red: &ReducedClaims) -> Result<(), RecursiveVerifyError> { +fn check_deferred_claims(program: &Program, claims: &DeferredClaims) -> Result<(), RecursiveVerifyError> { let stacked = stacked_bytecode(program); let expected_bc = stacked.len().trailing_zeros() as usize; - if red.r_bc.len() != expected_bc { + if claims.bytecode_point.len() != expected_bc { return Err(RecursiveVerifyError::InvalidDeferredShape); } - if mle_eval(&stacked, &red.r_bc) != red.v_bc { + if mle_eval(&stacked, &claims.bytecode_point) != claims.bytecode_value { return Err(RecursiveVerifyError::BytecodeClaim); } - let (ma, mb) = flock::blake3::matrices(); let klog = flock::blake3::K_LOG; - if red.r_m.len() != 2 * klog { + if claims.matrix_point.len() != 2 * klog { return Err(RecursiveVerifyError::InvalidDeferredShape); } - let eq_r = primitives::multilinear::build_eq(&red.r_m[..klog]); - let eq_c = primitives::multilinear::build_eq(&red.r_m[klog..]); - let direct = |m: &flock::r1cs::SparseBinaryMatrix| -> F128 { - let mut acc = F128::ZERO; - for (i, row) in m.rows.iter().enumerate() { - let s = row.iter().map(|&j| eq_c[j]).fold(F128::ZERO, |a, x| a + x); - acc += eq_r[i] * s; - } - acc - }; - if direct(ma) != red.v_a { + let eq_r = pcs::ligerito::build_eq_table_ext(&claims.matrix_point[..klog]); + let eq_c = pcs::ligerito::build_eq_table_ext(&claims.matrix_point[klog..]); + let (v_a, v_b) = flock::blake3::bilinear_walk_pair(&eq_r, &eq_c); + if v_a != claims.matrix_a_value { return Err(RecursiveVerifyError::MatrixAClaim); } - if direct(mb) != red.v_b { + if v_b != claims.matrix_b_value { return Err(RecursiveVerifyError::MatrixBClaim); } Ok(()) @@ -553,15 +605,15 @@ fn check_reduced(program: &Program, red: &ReducedClaims) -> Result<(), Recursive /// a real `cpu::verify` run (zero hand-mirroring drift). fn gen_verify( program: &Program, - pi: [F128; 2], + pi: [F192; 2], proof: &lean_vm::cpu::Proof, summary: &lean_vm::cpu::VerifySummary, ops: &[TraceOp], -) -> (Vec<(String, Vec)>, SubDefer) { +) -> (Vec<(String, Vec)>, DeferredSubproof) { let l = lean_vm::cpu::layout( &program.prog, - proof.stream[0].lo as usize, - [1, 2, 3, 4, 5, 6].map(|i| proof.stream[i].lo as usize), + proof.stream[0].c0 as usize, + std::array::from_fn(|i| proof.stream[1 + i].c0 as usize), pi, ); let sides: [&[Block]; 3] = [&l.push, &l.pull, &l.count]; @@ -597,23 +649,24 @@ fn gen_verify( vi += 1; for c in &blk.coords { let (t, v, f) = match c { - Coord::Const(v) => (0u128, *v, *v), + Coord::Const(v) => (0u128, F192::new(v.0, 0, 0), F192::new(v.0, 0, 0)), Coord::Col(i) => { if framework && seen_claims.insert((*i, blk.kappa)) { nclaims += 1; } - (1, F128::ZERO, l.pad[*i]) + (1, F192::ZERO, F192::new(l.pad[*i].0, 0, 0)) } - Coord::GCol(i) => { + Coord::GCol(i, k) => { if framework && seen_claims.insert((*i, blk.kappa)) { nclaims += 1; } - (2, F128::ZERO, G * l.pad[*i]) + let gk = g_pow(*k as usize); + (2, F192::new(gk.0, 0, 0), F192::new((gk * l.pad[*i]).0, 0, 0)) } - Coord::Index => (3, F128::ZERO, F128::ZERO), + Coord::Index => (3, F192::ZERO, F192::ZERO), Coord::Public(_) => { nbcv += 1; - (4, F128::ZERO, F128::ZERO) + (4, F192::ZERO, F192::ZERO) } }; ct.push(t); @@ -632,24 +685,21 @@ fn gen_verify( seed.clone().replay(ops); // Grinding digests are the only trace-borne data (they are functions of - // sponge states): the first Pow is the bus grind; among the rest, fold - // grinds carry bits > 0 and query-phase grinds carry bits = 0. - let pows: Vec<(F128, u32, F128)> = ops + // sponge states): fold grinds carry bits > 0 and query-phase grinds carry + // bits = 0. + let pows: Vec<(F192, u32, F64)> = ops .iter() .filter_map(|op| match op { TraceOp::Pow { nonce, bits, digest } => Some((*nonce, *bits, *digest)), _ => None, }) .collect(); - let _gdig = pows[0].2; // digest bits now advice-decomposed in-guest - // Bus: the bytecode claims carry the push/pull ζ_lo points and sb. - let kbc = summary.bytecode_claims[0].point.len() - 3; - let zeta: Vec = summary.bytecode_claims[0].point[..kbc].to_vec(); - let sb: Vec = summary.bytecode_claims[0].point[kbc..].to_vec(); + let kbc = summary.bytecode_claims[0].point.len() - lean_vm::leaf::N_BYTECODE_SELECTORS; + let zeta: Vec = summary.bytecode_claims[0].point[..kbc].to_vec(); + let sb: Vec = summary.bytecode_claims[0].point[kbc..].to_vec(); let taus = l.taus; - // Flock replay data, all named struct fields. let n_log_b3 = l.taus[5]; let lcrounds = flock::blake3::K_LOG - 6; @@ -660,50 +710,37 @@ fn gen_verify( let lc_beta = summary.lc_claim.beta; let lrr = summary.lc_claim.r_rounds.clone(); - let evtot_e: usize = lean_vm::tables::tables().iter().map(|t| t.n_committed_columns()).sum(); - let ncl = nclaims + evtot_e + 1; // bus + constraint + the PI claim + let ncl = nclaims + evtot_e + 3; // bus + constraint + the three PI memory-limb claims // ---- the stacked opening: config + the opening summary ---- let stack_mu = l.m; - let vcfg = pcs::ligerito::LigeritoSecurityConfig::derive_config(stack_mu + 7) - .and_then(|s| s.to_config()) - .expect("stack ligerito config"); + let vcfg = pcs::ligerito::LigeritoSecurityConfig::derive_config_with_log_inv_rate( + stack_mu + pcs::LOG_PACKING, + summary.log_inv_rate, + ) + .and_then(|s| s.to_prover_verifier_configs()) + .expect("stack ligerito config") + .1; let log_n = stack_mu; let shapes = vcfg.level_shapes(log_n); let (nlev, r) = (shapes.levels, vcfg.level_steps); let (klvl, lmc, _yr_log_n) = (shapes.ks, shapes.log_msg_cols, shapes.yr_log_n); let queries = vcfg.queries.clone(); - // query packing: each squeezed word carries 128/depth positions. + // Query packing: each squeezed F192 word carries 192/depth positions. let depth: Vec = shapes.block_len.iter().map(|b| b.trailing_zeros() as usize).collect(); - let per: Vec = depth.iter().map(|&d| 128 / d).collect(); + let per: Vec = depth.iter().map(|&d| 192 / d).collect(); let fgb = |lvl: usize| vcfg.fold_grinding_bits.get(lvl).copied().unwrap_or(0) as i64; - // The Ligerito opening's scalars close the stream: start msg (2), per - // level the fold (nonce? + msg) words, then root (2) / yr words, one - // query-grind nonce, and an intro msg (2) at EVERY level including the last. - // The sumcheck then runs to the end, adding a message per tail round except - // the closing one, which needs none. - let lig_stream_words: usize = 2 - + (0..nlev) - .map(|lvl| { - let folds: usize = - (0..klvl[lvl]).map(|j| 2 + usize::from(fgb(lvl) - j as i64 > 0)).sum(); - folds - + if lvl == nlev - 1 { - (1 << shapes.yr_log_n) + 1 + 2 - } else { - 2 + 1 + 2 - } - }) - .sum::() - + 2 * (shapes.yr_log_n - 1); - // The lincheck rounds and z_partial sit at fixed offsets from the FLOCK - // tail (the stream up to the opening): [.. (e1,e_inf) x lcrounds | - // z_partial (64) | s_hat_v (2 x 128) | the opening's scalars]. - let ns = proof.stream.len() - lig_stream_words; - let lcr: Vec = proof.stream[ns - 256 - 64 - 2 * lcrounds..ns - 256 - 64].to_vec(); - let lcz: Vec = proof.stream[ns - 256 - 64..ns - 256].to_vec(); + // The K stacked opening lives ENTIRELY in `proof.openings` (structs, + // observed into the sponge for Fiat-Shamir, never `add_scalar`'d), and + // ring-switch `s_hat_v` is likewise observed from its struct, not streamed. + // So `proof.stream` ends with flock's reduction: the last 64 scalars are + // lincheck's `z_partial`, immediately preceded by the `(e1, e_inf)` pairs of + // the `lcrounds` lincheck sumcheck rounds. + let ns = proof.stream.len(); + let lcr: Vec = proof.stream[ns - 64 - 2 * lcrounds..ns - 64].to_vec(); + let lcz: Vec = proof.stream[ns - 64..ns].to_vec(); // matpart = the deferred weighted matrix evaluation: the lincheck running // claim minus (= plus, char 2) the const-pin contribution. @@ -714,23 +751,23 @@ fn gen_verify( let (e1, ei, rv) = (lcr[2 * i], lcr[2 * i + 1], lrr[i]); let e0 = lrun + e1; let c1q = e0 + e1 + ei; - lrun = ei * rv * rv + c1q * rv + e0; + lrun = (ei * rv + c1q) * rv + e0; } let mut pinw = lc_beta; for (j, &rv) in lrr.iter().enumerate() { let bit = (pincol >> (flock::blake3::K_LOG - 1 - j)) & 1; - pinw *= if bit == 1 { rv } else { F128::ONE + rv }; + pinw *= if bit == 1 { rv } else { F192::ONE + rv }; } pinw *= lcz[pincol % 64]; let matpart = lrun + pinw; let lig_raw = summary.opening.lig.query_squeezes.clone(); - // Grind sanity: in transcript order after the bus grind — per level, the - // fold grinds (bits > 0 per the config schedule) then ONE query-phase + // Grind sanity: in transcript order, per level, the fold grinds (bits > 0 + // per the config schedule) then ONE query-phase // grind. The nonces themselves ride the shared stream now (raw words); // the trace is only cross-checked here. let qbits: Vec = (0..nlev).map(|lvl| vcfg.grinding_bits[lvl] as u32).collect(); - let mut grinds = pows[1..].iter(); + let mut grinds = pows.iter(); for lvl in 0..nlev { for j in 0..klvl[lvl] { let bits = (fgb(lvl) - j as i64).max(0) as u32; @@ -746,12 +783,11 @@ fn gen_verify( // ---- hints ---- // bcv: the deferred bytecode evaluations at the SHARED push/pull point - // (leaf's own scan, coord order; both bytecode blocks carry the same six). + // (leaf's own scan, coord order; push and pull carry the same nine). let (kbc2, bcv) = lean_vm::leaf::public_evals(&l.push, &zeta); assert_eq!(kbc2, kbc); assert_eq!(bcv.len(), nbcv / 2); - let sb3: [F128; 3] = sb.clone().try_into().unwrap(); - let wbc = vec![lean_vm::leaf::stacked_bytecode_value(&bcv, &sb3)]; + let bytecode_value = lean_vm::leaf::stacked_bytecode_value(&bcv, &sb); // checkpoints: the verifier's phase-boundary sponge states (guest cvh). // ---- per-sub HINT data (the placeholder map is built once, elsewhere) ---- @@ -759,14 +795,14 @@ fn gen_verify( // sort_order[side_base + rank] = g^{side-local index of the rank-r block}. // The guest only perm-checks it and derives offsets; any aligned tiling is // sound, so this canonical order just has to match the committed leaf. - let mut sort_order: Vec = Vec::new(); + let mut sort_order: Vec = Vec::new(); let mut gbase = 0usize; for blocks in sides.iter() { let n = blocks.len(); let mut order: Vec = (0..n).collect(); order.sort_by(|&a, &b| blocks[b].kappa.cmp(&blocks[a].kappa).then(a.cmp(&b))); for &i in &order { - sort_order.push(g_pow(gbase + i)); // g^{global block index} + sort_order.push(F192::new(g_pow(gbase + i).0, 0, 0)); // g^{global block index} } gbase += n; } @@ -786,11 +822,14 @@ fn gen_verify( } let mut col_order = committed_globals.clone(); col_order.sort_by_key(|&global| l.placements[global].offset); - let col_sort_order: Vec = col_order.iter().map(|&global| g_pow(compact_col[global])).collect(); + let col_sort_order: Vec = col_order + .iter() + .map(|&global| F192::new(g_pow(compact_col[global]).0, 0, 0)) + .collect(); let sch = lean_vm::cpu::schema(); let b3base = sch.base[5]; let valcols: Vec = lean_vm::tables::BLAKE3_VALUE_COLS.iter().map(|&c| b3base + c).collect(); - let log_mem = proof.stream[0].lo as usize; + let log_mem = proof.stream[0].c0 as usize; // ---- Phase E2 hints (the stacked Ligerito opening) ---- let lig = &proof.openings[0]; @@ -802,46 +841,72 @@ fn gen_verify( let d = depth[lv]; let mut out = Vec::with_capacity(queries[lv]); for v in &lig_raw[lv] { - let bits = (v.lo as u128) | ((v.hi as u128) << 64); for j in 0..per[lv].min(queries[lv] - out.len()) { - out.push(((bits >> (j * d)) as usize) & (shapes.block_len[lv] - 1)); + let off = j * d; + let limbs = [v.c0, v.c1, v.c2]; + let (li, sh) = (off / 64, off % 64); + let mut chunk = limbs[li] >> sh; + if sh + d > 64 { + chunk |= limbs[li + 1] << (64 - sh); + } + out.push(chunk as usize & (shapes.block_len[lv] - 1)); } } out }) .collect(); - let rows_of = |lv: usize| -> &Vec> { - if lv == 0 { - &lig.initial_proof.opened_rows - } else if lv == r { - &lig.final_proof.opened_rows - } else { - &lig.level_proofs[lv - 1].opened_rows - } - }; let path_of = |lv: usize| -> &Vec<[u8; 32]> { if lv == 0 { - &lig.initial_proof.merkle_proof + &lig.ligerito.initial_proof.merkle_proof } else if lv == r { - &lig.final_proof.merkle_proof + &lig.ligerito.final_proof.merkle_proof } else { - &lig.level_proofs[lv - 1].merkle_proof + &lig.ligerito.recursive_proofs[lv - 1].merkle_proof } }; - let hb32 = |h: [u8; 32]| { - let wd = |o: usize| u64::from_le_bytes(h[o..o + 8].try_into().unwrap()); - [F128::new(wd(0), wd(8)), F128::new(wd(16), wd(24))] - }; - let (mut lrows_flat, mut lpaths_flat) = (Vec::new(), Vec::new()); + // Level 0 rows are embedded F64 values. For levels ≥1, each F192 word is + // flattened into three embedded limbs so the guest can reproduce the exact + // 24-byte Merkle-leaf preimage before reconstructing the field value. + let (mut lrows_flat, mut lpaths_flat): (Vec, Vec) = (Vec::new(), Vec::new()); for lv in 0..nlev { - let (rows_exp, path_exp) = - pcs::ligerito::expand_level_opening(shapes.block_len[lv], &positions[lv], rows_of(lv), numinter[lv], path_of(lv)) - .expect("expand stacked level opening"); - for row in &rows_exp { - lrows_flat.extend_from_slice(row); - } + let path_exp = if lv == 0 { + let (rows_exp, path_exp) = pcs::ligerito::expand_level_opening_base( + shapes.block_len[lv], + &positions[lv], + &lig.ligerito.initial_proof.opened_rows, + numinter[lv], + path_of(lv), + ) + .expect("expand base (level 0) stacked opening"); + for row in &rows_exp { + for &x in row { + lrows_flat.push(F192::new(x.0, 0, 0)); + } + } + path_exp + } else { + let rows_ref = if lv == r { + &lig.ligerito.final_proof.opened_rows + } else { + &lig.ligerito.recursive_proofs[lv - 1].opened_rows + }; + let (rows_exp, path_exp) = pcs::ligerito::expand_level_opening_ext( + shapes.block_len[lv], + &positions[lv], + rows_ref, + numinter[lv], + path_of(lv), + ) + .expect("expand ext (level ≥1) stacked opening"); + for row in &rows_exp { + for &x in row { + lrows_flat.extend([F192::new(x.c0, 0, 0), F192::new(x.c1, 0, 0), F192::new(x.c2, 0, 0)]); + } + } + path_exp + }; for &h in &path_exp { - lpaths_flat.extend_from_slice(&hb32(h)); + lpaths_flat.extend_from_slice(&pack_hash_state(&h)); } } // claim descriptors, in exact clv order. @@ -865,7 +930,7 @@ fn gen_verify( continue; // a table's coords are settled by the zerocheck } for c in &blk.coords { - if let Coord::Col(i) | Coord::GCol(i) = c { + if let Coord::Col(i) | Coord::GCol(i, _) = c { if !desc_seen.insert((*i, blk.kappa)) { continue; // deduped: pooled once at its first occurrence } @@ -880,22 +945,25 @@ fn gen_verify( let col = sch.base[t] + c; let pl = l.placements[col]; if pl.is_virtual() { - let nvt = 7 + taus[t]; - push_desc(3, nvt); + let nvt = lean_vm::blake3_flock::SLOT_STRIDE_LOG + taus[t]; + push_desc(4, nvt); } else { push_desc(1, taus[t]); } } } { - // PI claim on MEM: point = [r_m, 0, 0, ...]. Coords beyond lenris are - // const zero, so they fold into the y pattern (required-zero bits) - // instead of runtime overlap factors: cap the low span at lenris and - // retain those zero coordinates in the derived offset-bit span. - let pl = l.placements[lean_vm::cpu::MEM]; - let folded = pl.n_vars.saturating_sub(lenris); - let low = pl.n_vars - folded; - push_desc(2, low); + // PI claims on MEM: three lanes (MEM_LO, MEM_HI, MEM_TOP) at the SAME + // point [r_m, 0, 0, ...] but different columns. bind_pi_claim orders + // them [lo, hi, top]; the proof streams lo/hi and the guest derives top. + // Coords beyond lenris are const zero, so they fold into the y pattern + // instead of runtime overlap factors. + for &col in &[lean_vm::cpu::MEM_LO, lean_vm::cpu::MEM_HI, lean_vm::cpu::MEM_TOP] { + let pl = l.placements[col]; + let folded = pl.n_vars.saturating_sub(lenris); + let low = pl.n_vars - folded; + push_desc(2, low); + } } assert_eq!(cpbuf.len(), ncl, "descriptor count == pool size"); @@ -904,28 +972,136 @@ fn gen_verify( for &lmc_lv in lmc.iter().take(nlev) { let s2 = pcs::ligerito::eval_sk_at_vks(lmc_lv); for &v in &s2 { - svk_flat.push(v); - ivk_flat.push(if v == F128::ZERO { F128::ZERO } else { v.inv() }); + svk_flat.push(F192::new(v.0, 0, 0)); + ivk_flat.push(if v == F64::ZERO { + F192::ZERO + } else { + F192::new(v.inv().0, 0, 0) + }); } } - let deferred = SubDefer { - pi, - kbc, - zeta, - sb: sb.clone(), - wbc: wbc.clone(), - lc_alpha, - zz: zc_z, - zrho8: zrho[..lcrounds].to_vec(), - lrr: lrr.clone(), - lcz: lcz.clone(), - matpart, + let deferred = DeferredSubproof { + public_input: pi, + bytecode_log: kbc, + bytecode_row_point: zeta, + bytecode_selector_point: sb.clone(), + bytecode_value, + matrix_a_coefficient: lc_alpha, + skip_point: zc_z, + zerocheck_row_point: zrho[..lcrounds].to_vec(), + lincheck_round_point: lrr.clone(), + lincheck_terminal_values: lcz.clone(), + matrix_claim: matpart, }; let hints = vec![ ("stream".to_string(), { let mut v = proof.stream.clone(); - v.resize(stream_cap, F128::ZERO); + // Append the Ligerito opening's msg-cursor sequence, in EXACT + // F64-verifier order (see ligerito::recursive_verifier_with_basis_ + // succinct): the interleaved raw grind nonces + observed scalars + // (start_msg, per-fold [grind-nonce?, msg u0/u2], level roots as two + // hash_to_scalars, query-grind nonce, every level's intro msg, + // final yr, and the remaining tail-round messages). + // The guest's open_stacked reads these via `msg_cursor = cursor`, + // which sits at proof.stream.len() after the flock reduction — the + // ring-switch is struct-observed and no longer advances the cursor. + let lp = &proof.openings[0].ligerito; + let fb = |lvl: usize| -> u32 { vcfg.fold_grinding_bits.get(lvl).copied().unwrap_or(0) as u32 }; + let (mut tx, mut fni, mut qi, mut rri, mut oi) = (0usize, 0usize, 0usize, 0usize, 0usize); + let msg = |tx: &mut usize| -> [F192; 2] { + let m = lp.sumcheck_transcript[*tx]; + *tx += 1; + [m.u_0, m.u_2] + }; + // intro start_msg + v.extend_from_slice(&msg(&mut tx)); + // L0 fold rounds + for j in 0..vcfg.initial_k { + if fb(0).saturating_sub(j as u32) > 0 { + v.push(F192::new(lp.fold_grinding_nonces[fni], 0, 0)); + fni += 1; + } + v.extend_from_slice(&msg(&mut tx)); + } + // L1 root, its OOD claims, then the L0 query phase and induced + // basis introduction. + v.extend_from_slice(&pcs::merkle::hash_to_scalars(&lp.recursive_roots[rri])); + rri += 1; + for _ in 0..vcfg.ood_samples[1] { + v.push(lp.ood_values[oi]); + oi += 1; + v.extend_from_slice(&msg(&mut tx)); + } + v.push(F192::new(lp.grinding_nonces[qi], 0, 0)); + qi += 1; + v.extend_from_slice(&msg(&mut tx)); + // recursive levels 1..=r (loop index i = level-1) + for i in 0..vcfg.level_steps { + for j in 0..vcfg.level_ks[i] { + if fb(i + 1).saturating_sub(j as u32) > 0 { + v.push(F192::new(lp.fold_grinding_nonces[fni], 0, 0)); + fni += 1; + } + v.extend_from_slice(&msg(&mut tx)); + } + if i == vcfg.level_steps - 1 { + // last level: final message yr, then the query-grind nonce + // (the verifier reads grinding_nonces[qi] without advancing), + // its intro message, and every tail-round message except + // the closing round, which sends none. + v.extend_from_slice(&lp.final_proof.yr); + v.push(F192::new(lp.grinding_nonces[qi], 0, 0)); + v.extend_from_slice(&msg(&mut tx)); + for _ in 1..shapes.yr_log_n { + v.extend_from_slice(&msg(&mut tx)); + } + } else { + v.extend_from_slice(&pcs::merkle::hash_to_scalars(&lp.recursive_roots[rri])); + rri += 1; + for _ in 0..vcfg.ood_samples[i + 2] { + v.push(lp.ood_values[oi]); + oi += 1; + v.extend_from_slice(&msg(&mut tx)); + } + v.push(F192::new(lp.grinding_nonces[qi], 0, 0)); + qi += 1; + v.extend_from_slice(&msg(&mut tx)); + } + } + // Sanity: the reconstruction must consume the struct exactly. + assert_eq!( + tx, + lp.sumcheck_transcript.len(), + "lig_msgs: sumcheck_transcript not fully consumed" + ); + assert_eq!( + fni, + lp.fold_grinding_nonces.len(), + "lig_msgs: fold nonces not fully consumed" + ); + assert_eq!( + rri, + lp.recursive_roots.len(), + "lig_msgs: recursive_roots not fully consumed" + ); + assert_eq!(oi, lp.ood_values.len(), "lig_msgs: OOD values not fully consumed"); + assert!( + v.len() <= stream_cap, + "stream+lig_msgs {} exceeds stream_cap {stream_cap}", + v.len() + ); + v.resize(stream_cap, F192::ZERO); + v + }), + ("rs_shatv".to_string(), { + // The ring-switch slices: each claim's 64-entry s_hat_v, observed from + // the opening STRUCT (RingSwitchProof), not the stream. Order [ab, c]. + let lig = &proof.openings[0]; + let mut v = Vec::new(); + for rsw in &lig.ring_switches { + v.extend_from_slice(&rsw.s_hat_v); + } v }), ("bytecode_vals".to_string(), bcv), @@ -935,16 +1111,25 @@ fn gen_verify( ("sub_pis".to_string(), vec![pi[0], pi[1]]), // slacks bounding each claim'"'"'s reads to the written regions (so an // over-long hint cannot pull free padding): low_len <= mu_s/tau_t - // (zeta/rho) and low_len(+7 for qpkd) <= lenris (fold challenges). + // (zeta/rho) and low_len(+SLOT_STRIDE_LOG for qpkd) <= lenris. // per-claim overlap count, for the exact length pin: nover = the // amount by which the claim's total vars exceed the fold rounds. - ("claim_nover".to_string(), (0..ncl).map(|j| g_pow(nover_v[j])).collect()), + ( + "claim_nover".to_string(), + (0..ncl).map(|j| F192::new(g_pow(nover_v[j]).0, 0, 0)).collect(), + ), // the pi claim's low dimension is min(log_mem, lenris); certify it as // a min (<= both, == one) so pi is pinned like every other claim. - ("pi_cplen".to_string(), vec![g_pow(log_mem.min(lenris))]), + ( + "pi_cplen".to_string(), + vec![F192::new(g_pow(log_mem.min(lenris)).0, 0, 0)], + ), // the batched zerocheck's round count: max_t tau_t, certified in-guest as a // maximum (one of the taus, and dominating them all). - ("zc_tau_max".to_string(), vec![g_pow(*taus.iter().max().unwrap())]), + ( + "zc_tau_max".to_string(), + vec![F192::new(g_pow(*taus.iter().max().unwrap()).0, 0, 0)], + ), ("col_sort_order".to_string(), col_sort_order), ("sort_order".to_string(), sort_order.clone()), ]; @@ -956,16 +1141,24 @@ fn gen_verify( /// program), the merged per-sub witness entries, the outer statement, and the /// data to discharge the reduced claims. Splitting the build from the compile /// lets one compiled guest serve many batches (see `recursion_generic_many`). +/// The guest's stacked-size dispatch range: one `match_range` opening arm per +/// candidate `mu` in `MU_MIN..=MU_MAX` (mirrored by the soundness test's +/// residual-log cap). +const MU_MIN: usize = 22; +const MU_MAX: usize = 28; + struct Batch { - merged: Vec<(String, Vec>)>, + merged: Vec<(String, Vec>)>, program0: Program, statement: RecursiveStatement, nsub: usize, total_inner_cycles: usize, + inner_stats: Vec<(usize, usize)>, + outer_log_inv_rate: usize, } impl Batch { - fn public_input(&self) -> [F128; 2] { + fn public_input(&self) -> [F192; 2] { self.statement.public_input(lean_vm::cpu::fs_seed(&self.program0)) } @@ -976,7 +1169,7 @@ impl Batch { for (name, entries) in &self.merged { guest.set_witness(name, entries.clone()); } - let (outer_proof, stats) = prove(guest, self.public_input()); + let (outer_proof, stats) = prove(guest, self.public_input(), self.outer_log_inv_rate); ( RecursiveProof { statement: self.statement.clone(), @@ -990,24 +1183,27 @@ impl Batch { /// Prove `inner.len()` inner runs (same program, distinct statements + shapes), /// verify each inside the recursion guest, and assemble the aggregation inputs. /// `inner[k] = (hashes, iters)` sets sub k's opcode profile. -fn build_batch(inner: &[(usize, usize)]) -> Batch { +fn build_batch(inner: &[(usize, usize)], log_inv_rates: &[usize], outer_log_inv_rate: usize) -> Batch { assert!(!inner.is_empty(), "a recursion batch cannot be empty"); + assert_eq!(inner.len(), log_inv_rates.len(), "one PCS rate per inner proof"); let nsub = inner.len(); let mut total_inner_cycles = 0usize; + let mut inner_stats = Vec::with_capacity(nsub); let mut protos = Vec::new(); - for (k, &(hashes, iters)) in inner.iter().enumerate() { + for (k, (&(hashes, iters), &log_inv_rate)) in inner.iter().zip(log_inv_rates).enumerate() { let pi = [ - F128::new(0x1111_2222 + k as u64, 0x3333_4444), - F128::new(0x5555_6666, 0x7777_8888 + k as u64), + F192::new(0x1111_2222 + k as u64, 0x3333_4444, 0), + F192::new(0x5555_6666, 0x7777_8888 + k as u64, 0), ]; - let (program, proof, inner_cycles) = prove_inner(pi, hashes, iters); + let (program, proof, inner_cycles, inner_committed) = prove_inner(pi, hashes, iters, log_inv_rate); total_inner_cycles += inner_cycles; + inner_stats.push((inner_cycles, inner_committed)); trace_start(); let summary = verify(&program, &pi, &proof).expect("inner verifies"); let ops = trace_take(); protos.push((program, pi, proof, summary, ops)); } - let mut merged: Vec<(String, Vec>)> = Vec::new(); + let mut merged: Vec<(String, Vec>)> = Vec::new(); let mut subs = Vec::new(); for (program, pi, proof, summary, ops) in &protos { let (hints, defer) = gen_verify(program, *pi, proof, summary, ops); @@ -1025,13 +1221,16 @@ fn build_batch(inner: &[(usize, usize)]) -> Batch { } let (program0, _, _, _, _) = &protos[0]; // spi is main-level (one hint site): merge the statements into one entry. - let spi_all: Vec = subs.iter().flat_map(|d| [d.pi[0], d.pi[1]]).collect(); + let spi_all: Vec = subs + .iter() + .flat_map(|d| [d.public_input[0], d.public_input[1]]) + .collect(); let spi_pos = merged.iter().position(|(n, _)| n == "sub_pis").expect("spi hint"); merged[spi_pos].1 = vec![spi_all]; - let (agg_hints, gpi, reduced) = gen_agg(program0, &subs); + let (agg_hints, gpi, reduced) = aggregate_deferred_claims(program0, &subs); merged.extend(agg_hints.into_iter().map(|(n, v)| (n, vec![v]))); let statement = RecursiveStatement { - sub_statements: subs.iter().map(|d| d.pi).collect(), + sub_statements: subs.iter().map(|d| d.public_input).collect(), reduced, }; assert_eq!( @@ -1048,19 +1247,49 @@ fn build_batch(inner: &[(usize, usize)]) -> Batch { statement, nsub, total_inner_cycles, + inner_stats, + outer_log_inv_rate, } } +struct OpeningShape { + n_levels: usize, + yr_level: usize, + yr_log_len: usize, + folds: Vec, + log_message_columns: Vec, + queries: Vec, + tree_depths: Vec, + positions_per_squeeze: Vec, + squeezes: Vec, + interleaving: Vec, + query_grinding_bits: Vec, + fold_grinding_bits: Vec, + row_offsets: Vec, + path_offsets: Vec, + positions_offsets: Vec, + vanish_offsets: Vec, + fold_offsets: Vec, + residual_fold_offsets: Vec, + vanish_values: Vec, + vanish_inverses: Vec, + ood_samples: Vec, +} + /// The recursion program's placeholder map (the SHAPE-INDEPENDENT constants the /// generic guest is compiled from), built from the inner program's STRUCTURE and /// bytecode SIZE alone — no proof. Dummy layout sizes are fine: `rep` reads only the /// size-independent block/coord structure and `kbc = log2(bytecode)`, so the guest /// can be compiled BEFORE any inner proof exists. Because the map is a function of /// the inner bytecode size alone, one compiled guest serves every shape. -#[allow(clippy::type_complexity)] fn placeholder_map(program: &Program) -> BTreeMap { // Any valid sizes drive the layout — rep depends only on structure + kbc. - let l = lean_vm::cpu::layout(&program.prog, 20, [1usize << 10; 6], [F128::ZERO, F128::ZERO]); + let l = lean_vm::cpu::layout( + &program.prog, + 20, + [1usize << 10; lean_vm::tables::N_TABLES], + [F192::ZERO, F192::ZERO], + ); let kbc = program.prog.len().trailing_zeros() as usize; let sides: [&[Block]; 3] = [&l.push, &l.pull, &l.count]; let mumax = 40usize; @@ -1079,8 +1308,10 @@ fn placeholder_map(program: &Program) -> BTreeMap { // A table's blocks raise no claim any more: the batched zerocheck settles them //, so only the framework blocks stream column values. let sch_pm = lean_vm::cpu::schema(); - let owner_pm: Vec> = - lean_vm::cpu::block_kappa_sources(kbc).into_iter().map(|(src, _)| src.checked_sub(2)).collect(); + let owner_pm: Vec> = lean_vm::cpu::block_kappa_sources(kbc) + .into_iter() + .map(|(src, _)| src.checked_sub(2)) + .collect(); for blocks in sides.iter() { for blk in blocks.iter() { bc0.push(ct.len()); @@ -1091,7 +1322,7 @@ fn placeholder_map(program: &Program) -> BTreeMap { // One COORD_FRESH/COORD_CLAIM_SLOT entry PER coord (the guest // indexes them by global coord offset); only Col/GCol matter. let (mut fresh, mut slot, mut local) = (0usize, 0usize, 0usize); - if let Coord::Col(i) | Coord::GCol(i) = c { + if let Coord::Col(i) | Coord::GCol(i, _) = c { match owner { // A table's coord: the zerocheck reads it off that table's // column evaluations, at its local index. @@ -1113,21 +1344,29 @@ fn placeholder_map(program: &Program) -> BTreeMap { coord_slot.push(slot); coord_local.push(local); let (t, v, f) = match c { - Coord::Const(v) => (0u128, *v, *v), - Coord::Col(i) => (1, F128::ZERO, l.pad[*i]), - Coord::GCol(i) => (2, F128::ZERO, G * l.pad[*i]), - Coord::Index => (3, F128::ZERO, F128::ZERO), - Coord::Public(_) => { nbcv += 1; (4, F128::ZERO, F128::ZERO) } + Coord::Const(v) => (0u128, F192::new(v.0, 0, 0), F192::new(v.0, 0, 0)), + Coord::Col(i) => (1, F192::ZERO, F192::new(l.pad[*i].0, 0, 0)), + Coord::GCol(i, k) => { + let gk = g_pow(*k as usize); + (2, F192::new(gk.0, 0, 0), F192::new((gk * l.pad[*i]).0, 0, 0)) + } + Coord::Index => (3, F192::ZERO, F192::ZERO), + Coord::Public(_) => { + nbcv += 1; + (4, F192::ZERO, F192::ZERO) + } }; - ct.push(t); cval.push(u(v)); fpv.push(u(f)); + ct.push(t); + cval.push(u(v)); + fpv.push(u(f)); } } sblk.push(nblocks); } let evtot: usize = lean_vm::tables::tables().iter().map(|t| t.n_committed_columns()).sum(); - let ncl = nclaims + evtot + 1; // bus + constraint + the PI claim + let ncl = nclaims + evtot + 3; // bus + constraint + the three PI memory-limb claims - // ---- claim descriptors: buffer id + offset only (both structural) ---- + // ---- claim descriptor buffer ids (structural) ---- let sch = lean_vm::cpu::schema(); let b3base = sch.base[5]; let valcols: Vec = lean_vm::tables::BLAKE3_VALUE_COLS.iter().map(|&c| b3base + c).collect(); @@ -1153,7 +1392,7 @@ fn placeholder_map(program: &Program) -> BTreeMap { continue; // a table's coords are settled by the zerocheck } for c in &blk.coords { - if let Coord::Col(i) | Coord::GCol(i) = c { + if let Coord::Col(i) | Coord::GCol(i, _) = c { if !desc_seen.insert((*i, blk.kappa)) { continue; // deduped: pooled once at its first occurrence } @@ -1171,18 +1410,20 @@ fn placeholder_map(program: &Program) -> BTreeMap { for c in 0..table.n_committed_columns() { let col = sch.base[t] + c; let virtual_col = l.placements[col].is_virtual(); - cpbuf.push(if virtual_col { 3 } else { 1 }); + cpbuf.push(if virtual_col { 4 } else { 1 }); cpcol.push(if virtual_col { qpkd_compact } else { compact_col_pm[col] }); cpqslot.push(if virtual_col { - lean_vm::blake3_flock::VM_SLOTS[valcols.iter().position(|&v| v == col).unwrap()] + lean_vm::blake3_flock::SLOTS[valcols.iter().position(|&v| v == col).unwrap()] } else { 0 }); } } - cpbuf.push(2); // PI claim on MEM - cpcol.push(compact_col_pm[lean_vm::cpu::MEM]); - cpqslot.push(0); + for &col in &[lean_vm::cpu::MEM_LO, lean_vm::cpu::MEM_HI, lean_vm::cpu::MEM_TOP] { + cpbuf.push(2); + cpcol.push(compact_col_pm[col]); + cpqslot.push(0); + } assert_eq!(cpbuf.len(), ncl, "descriptor count == pool size"); assert_eq!(cpcol.len(), ncl, "every descriptor has a committed-column target"); assert_eq!(cpqslot.len(), ncl, "every descriptor has a fixed QPKD slot"); @@ -1190,25 +1431,36 @@ fn placeholder_map(program: &Program) -> BTreeMap { // ---- the placeholder map ---- let ints = |v: &[usize]| format!("[{}]", v.iter().map(|x| x.to_string()).collect::>().join(", ")); let us = |v: &[u128]| format!("[{}]", v.iter().map(|x| x.to_string()).collect::>().join(", ")); - let flds = |v: &[F128]| format!("[{}]", v.iter().map(|&x| u(x).to_string()).collect::>().join(", ")); + let flds = |v: &[F192]| { + format!( + "[{}]", + v.iter().map(|&x| f192_literal(x)).collect::>().join(", ") + ) + }; let mut rep = BTreeMap::new(); - let mut ps = |k: &str, v: String| { rep.insert(format!("{k}_PLACEHOLDER"), v); }; + let mut ps = |k: &str, v: String| { + rep.insert(format!("{k}_PLACEHOLDER"), v); + }; ps("STREAM_CAP", stream_cap.to_string()); ps("MIN_LOG_MEM", lean_vm::cpu::MIN_LOG_MEM.to_string()); - ps("INV_GEN", u(G.inv()).to_string()); - ps("LAGRANGE_INV_0", u(G.inv()).to_string()); - ps("LAGRANGE_INV_1", u((F128::ONE + G).inv()).to_string()); - ps("LAGRANGE_INV_2", u((G * (F128::ONE + G)).inv()).to_string()); + ps("INV_GEN", u(F192::new(G.inv().0, 0, 0)).to_string()); + ps("LAGRANGE_INV_0", u(F192::new(G.inv().0, 0, 0)).to_string()); + ps("LAGRANGE_INV_1", f192_literal((F192::ONE + F192::new(G.0, 0, 0)).inv())); + ps( + "LAGRANGE_INV_2", + f192_literal((F192::new(G.0, 0, 0) * (F192::ONE + F192::new(G.0, 0, 0))).inv()), + ); // The batched zerocheck sends its round polynomial WHOLE, at {0, 1, g, g^2}, so // it interpolates a cubic: one baked inverse denominator per node. { let q = primitives::multilinear::quad_nodes(); for i in 0..4 { - let den = (0..4).filter(|&j| j != i).fold(F128::ONE, |acc, j| acc * (q[i] + q[j])); - ps(&format!("LAG4_INV_{i}"), u(den.inv()).to_string()); + let den = (0..4).filter(|&j| j != i).fold(F192::ONE, |acc, j| acc * (q[i] + q[j])); + ps(&format!("LAG4_INV_{i}"), f192_literal(den.inv())); } } ps("MU_CAP", mumax.to_string()); + ps("REAL_IS_FULL_CUBE", l.taus.len().to_string()); ps("GKR_ROUNDS_CAP", (mumax * (mumax + 1) / 2 + mumax + 2).to_string()); ps("GKR_POINTS_CAP", ((mumax + 1) * mumax).to_string()); ps("SIDE_BLOCK_START", ints(&sblk)); @@ -1217,12 +1469,31 @@ fn placeholder_map(program: &Program) -> BTreeMap { // Push and pull emit bus blocks in matched pairs, so their baked kappa-source // segments are identical; the guest computes only push's side total and // aliases pull's mu to push's on this basis. - assert_eq!(bks[sblk[0]..sblk[1]], bks[sblk[1]..sblk[2]], "push/pull kappa sources must match"); - ps("BLOCK_KAPPA_SRC", ints(&bks.iter().map(|&(s, _)| s).collect::>())); - ps("BLOCK_KAPPA_ADJ", ints(&bks.iter().map(|&(_, a)| a).collect::>())); - ps("BLOCK_REAL_TABLE", ints(&bks.iter().map(|&(s, _)| if s >= 2 { s - 2 } else { 6 }).collect::>())); + assert_eq!( + bks[sblk[0]..sblk[1]], + bks[sblk[1]..sblk[2]], + "push/pull kappa sources must match" + ); + ps( + "BLOCK_KAPPA_SRC", + ints(&bks.iter().map(|&(s, _)| s).collect::>()), + ); + ps( + "BLOCK_KAPPA_ADJ", + ints(&bks.iter().map(|&(_, a)| a).collect::>()), + ); + ps( + "BLOCK_REAL_TABLE", + ints( + &bks.iter() + .map(|&(s, _)| if s >= 2 { s - 2 } else { l.taus.len() }) + .collect::>(), + ), + ); let mut block_side = Vec::new(); - for (s, blocks) in sides.iter().enumerate() { block_side.extend(std::iter::repeat_n(s, blocks.len())); } + for (s, blocks) in sides.iter().enumerate() { + block_side.extend(std::iter::repeat_n(s, blocks.len())); + } ps("BLOCK_SIDE", ints(&block_side)); ps("BLOCK_COORD_OFF", ints(&bc0)); ps("BLOCK_COORD_COUNT", ints(&bcn)); @@ -1233,7 +1504,15 @@ fn placeholder_map(program: &Program) -> BTreeMap { ps("COORD_CLAIM_SLOT", ints(&coord_slot)); ps("COORD_COL_LOCAL", ints(&coord_local)); ps("N_BUS_CLAIMS", nclaims.to_string()); - let idxc: Vec = (0..34).map(|i| { let mut g2k = G; for _ in 0..i { g2k = g2k * g2k; } u(F128::ONE + g2k) }).collect(); + let idxc: Vec = (0..34) + .map(|i| { + let mut g2k = F192::new(G.0, 0, 0); + for _ in 0..i { + g2k = g2k * g2k; + } + u(F192::ONE + g2k) + }) + .collect(); ps("INDEX_MLE_FACTORS", us(&idxc)); ps("N_CLAIMS", ncl.to_string()); ps("N_TABLES", l.taus.len().to_string()); @@ -1243,148 +1522,392 @@ fn placeholder_map(program: &Program) -> BTreeMap { // three leaf claims instead of trusted (lean_vm::cpu::eta_form_base). let n_id: Vec = lean_vm::tables::tables().iter().map(|t| t.n_constraints()).collect(); let form_base = lean_vm::cpu::eta_form_base(); - ps("ETA_OFFSET", ints(&lean_vm::constraints::eta_offsets(n_id.iter().copied()))); + ps( + "ETA_OFFSET", + ints(&lean_vm::constraints::eta_offsets(n_id.iter().copied())), + ); ps("ETA_FORM_BASE", form_base.to_string()); ps("N_ETA_POWS", (form_base + 3).to_string()); - let committed: Vec = - lean_vm::tables::tables().iter().map(|t| t.n_committed_columns()).collect(); + let committed: Vec = lean_vm::tables::tables() + .iter() + .map(|t| t.n_committed_columns()) + .collect(); ps("N_TABLE_COLS", ints(&committed)); ps("TABLE_COLS_CAP", (committed.iter().max().unwrap() + 1).to_string()); - // g^(push.mu - BUS_GRIND_SHIFT) is the bus PoW window - // (leaf::grand_product_grinding_bits: bits = mu - (127 - SECURITY_BITS)). - ps("BUS_GRIND_SHIFT", (127 - lean_vm::SECURITY_BITS).to_string()); const MINB3: usize = 3; - let fixed_challenges: Vec = flock::zerocheck::univariate_skip_optimized::small_challenges_ghash().into_iter().chain(flock::zerocheck::univariate_skip_optimized::medium_challenges_ghash()).collect(); + let fixed_challenges: Vec = flock::zerocheck::univariate_skip_optimized::small_challenges() + .into_iter() + .chain(flock::zerocheck::univariate_skip_optimized::medium_challenges()) + .collect(); ps("FIXED_CHALLENGES", flds(&fixed_challenges)); // Flock univariate skip: 6 skipped variables, then the fixed inner rounds. ps("K_SKIP", "6".to_string()); ps("N_FIXED_CHALLENGE_ROUNDS", fixed_challenges.len().to_string()); - let one_plus_challenge_inv: Vec = fixed_challenges.iter().map(|&c| (F128::ONE + c).inv()).collect(); + let one_plus_challenge_inv: Vec = fixed_challenges.iter().map(|&c| (F192::ONE + c).inv()).collect(); ps("ONE_PLUS_CHALLENGE_INV", flds(&one_plus_challenge_inv)); - let phi: Vec = primitives::field::phi8::PHI_8_TABLE[..128].to_vec(); + let phi: Vec = primitives::field::PHI_8_TABLE_192[..128].to_vec(); ps("PHI8_NODES", flds(&phi)); - let inv_den = |nodes: &[F128], node: F128, skip: F128| { let mut d = F128::ONE; for &s in nodes { if s != skip { d *= node + s; } } d.inv() }; - let ilam: Vec = (0..64).map(|i| inv_den(&phi[64..128], phi[64 + i], phi[64 + i])).collect(); - let icmb: Vec = (0..64).map(|i| inv_den(&phi[..128], phi[64 + i], phi[64 + i])).collect(); - let isdom: Vec = (0..64).map(|i| inv_den(&phi[..64], phi[i], phi[i])).collect(); + // Tower F192 = F64[Y]/(Y^3+Y+1), Y = new(0,1,0). Y_TOWER embeds Y for + // AIR lane reassembly; Y_INV helps derive the top PI-memory limb. + let y_tower = F192::new(0, 1, 0); + ps("Y_TOWER", u(y_tower).to_string()); + ps("Y_INV", f192_literal(y_tower.inv())); + // Coordinate basis e_i of F192 over F2 (spans the whole field): the 64 + // binary basis vectors in each of the three tower limbs. The guest uses + // these vectors to reconstruct a word from its 192 coordinate bits. + let coord_basis: Vec = (0..192) + .map(|i| match i / 64 { + 0 => F192::new(1u64 << i, 0, 0), + 1 => F192::new(0, 1u64 << (i - 64), 0), + 2 => F192::new(0, 0, 1u64 << (i - 128)), + _ => unreachable!(), + }) + .collect(); + ps("COORD_BASIS", flds(&coord_basis)); + let inv_den = |nodes: &[F192], node: F192, skip: F192| { + let mut d = F192::ONE; + for &s in nodes { + if s != skip { + d *= node + s; + } + } + d.inv() + }; + let ilam: Vec = (0..64) + .map(|i| inv_den(&phi[64..128], phi[64 + i], phi[64 + i])) + .collect(); + let icmb: Vec = (0..64) + .map(|i| inv_den(&phi[..128], phi[64 + i], phi[64 + i])) + .collect(); + let isdom: Vec = (0..64).map(|i| inv_den(&phi[..64], phi[i], phi[i])).collect(); ps("LAGRANGE_INV_LAMBDA", flds(&ilam)); ps("LAGRANGE_INV_COMBINED", flds(&icmb)); ps("LAGRANGE_INV_S", flds(&isdom)); ps("LINCHECK_ROUNDS", lcrounds.to_string()); - let pincol = flock::blake3::build_block_r1cs(taus[5].max(MINB3)).const_pin.expect("blake3 r1cs has a const pin"); + let pincol = flock::blake3::build_block_r1cs(taus[5].max(MINB3)) + .const_pin + .expect("blake3 r1cs has a const pin"); ps("PIN_COLUMN", pincol.to_string()); ps("K_LOG", flock::blake3::K_LOG.to_string()); + // The q_pkd Strided-claim slot stride is K_LOG - LOG_PACKING (= 8), so the + // qpkd point-claim slot must use THIS, not LOG2_FIELD_BITS. + ps("SLOT_STRIDE_LOG", lean_vm::blake3_flock::SLOT_STRIDE_LOG.to_string()); // ---- LIG candidate tables (fixed [minm, maxm] range; open_stacked config) ---- - let oshape = |m: usize| { - let vc = pcs::ligerito::LigeritoSecurityConfig::derive_config(m + 7) - .and_then(|s| s.to_config()) - .expect("candidate ligerito config"); + let oshape = |m: usize, log_inv_rate: usize| { + let vc = + pcs::ligerito::LigeritoSecurityConfig::derive_config_with_log_inv_rate(m + pcs::LOG_PACKING, log_inv_rate) + .and_then(|s| s.to_prover_verifier_configs()) + .expect("candidate ligerito config") + .1; let sh = vc.level_shapes(m); let (cn, cr) = (sh.levels, vc.level_steps); let (ck, cl, cyr) = (sh.ks.clone(), sh.log_msg_cols.clone(), sh.yr_log_n); let cq = vc.queries.clone(); let cd: Vec = sh.block_len.iter().map(|b| b.trailing_zeros() as usize).collect(); - let cp: Vec = cd.iter().map(|&d| 128 / d).collect(); + let cp: Vec = cd.iter().map(|&d| 192 / d).collect(); let cs: Vec = (0..cn).map(|i| cq[i].div_ceil(cp[i])).collect(); let cni: Vec = ck.iter().map(|&k| 1usize << k).collect(); let cqb: Vec = (0..cn).map(|lvl| vc.grinding_bits[lvl]).collect(); assert!( - cni.iter().all(|&n| n <= 64), - "recursive Ligerito guest supports Merkle rows of at most one 1024-byte BLAKE3 chunk" + cni.iter().enumerate().all(|(lv, &n)| { + let (bytes, whole_blocks) = if lv == 0 { + (8 * n, n % 8 == 0) + } else { + (24 * n, (3 * n) % 8 == 0) + }; + bytes <= 1024 && whole_blocks + }), + "recursive Ligerito guest supports whole-block Merkle rows of at most one 1024-byte BLAKE3 chunk" ); let cfgb = |lvl: usize| vc.fold_grinding_bits.get(lvl).copied().unwrap_or(0) as i64; let mut cfb: Vec = Vec::new(); - for (lvl, &k) in ck.iter().enumerate().take(cn) { for j in 0..k { cfb.push((cfgb(lvl) - j as i64).max(0) as usize); } } - let psum = |f: &dyn Fn(usize) -> usize| -> Vec { let mut o = Vec::with_capacity(cn); let mut acc = 0; for lv in 0..cn { o.push(acc); acc += f(lv); } o }; - let c_rowoff = psum(&|lv| cq[lv] * cni[lv]); + for (lvl, &k) in ck.iter().enumerate().take(cn) { + for j in 0..k { + cfb.push((cfgb(lvl) - j as i64).max(0) as usize); + } + } + let psum = |f: &dyn Fn(usize) -> usize| -> Vec { + let mut o = Vec::with_capacity(cn); + let mut acc = 0; + for lv in 0..cn { + o.push(acc); + acc += f(lv); + } + o + }; + let c_rowoff = psum(&|lv| cq[lv] * cni[lv] * if lv == 0 { 1 } else { 3 }); let c_pathoff = psum(&|lv| cq[lv] * cd[lv] * 2); - let c_sbitsoff = psum(&|lv| cs[lv] * 128); let c_qpoff = psum(&|lv| cs[lv] * cp[lv]); let c_svkoff = psum(&|lv| cl[lv] + 1); let c_foldbase = psum(&|lv| ck[lv]); let c_risstart: Vec = (0..cn).map(|k| c_foldbase[k] + ck[k]).collect(); let mut c_svk = Vec::new(); let mut c_ivk = Vec::new(); - for &cl_lv in cl.iter().take(cn) { for &v in &pcs::ligerito::eval_sk_at_vks(cl_lv) { c_svk.push(v); c_ivk.push(if v == F128::ZERO { F128::ZERO } else { v.inv() }); } } - (cn, cr, cyr, ck, cl, cq, cd, cp, cs, cni, cqb, cfb, c_rowoff, c_pathoff, c_sbitsoff, c_qpoff, c_svkoff, c_foldbase, c_risstart, c_svk, c_ivk) + for &cl_lv in cl.iter().take(cn) { + for &v in &pcs::ligerito::eval_sk_at_vks(cl_lv) { + c_svk.push(F192::new(v.0, 0, 0)); + c_ivk.push(if v == F64::ZERO { + F192::ZERO + } else { + F192::new(v.inv().0, 0, 0) + }); + } + } + OpeningShape { + n_levels: cn, + yr_level: cr, + yr_log_len: cyr, + folds: ck, + log_message_columns: cl, + queries: cq, + tree_depths: cd, + positions_per_squeeze: cp, + squeezes: cs, + interleaving: cni, + query_grinding_bits: cqb, + fold_grinding_bits: cfb, + row_offsets: c_rowoff, + path_offsets: c_pathoff, + positions_offsets: c_qpoff, + vanish_offsets: c_svkoff, + fold_offsets: c_foldbase, + residual_fold_offsets: c_risstart, + vanish_values: c_svk, + vanish_inverses: c_ivk, + ood_samples: vc.ood_samples, + } }; - let (minm, maxm) = (22usize, 28usize); - let cands: Vec<_> = (minm..=maxm).map(oshape).collect(); - let maxlev = cands.iter().map(|c| c.0).max().unwrap(); - let maxfolds = cands.iter().map(|c| c.11.len()).max().unwrap(); - let maxsvk = cands.iter().map(|c| c.19.len()).max().unwrap(); + let (minm, maxm) = (MU_MIN, MU_MAX); + let rates = pcs::ligerito::MIN_LOG_INV_RATE..=pcs::ligerito::MAX_LOG_INV_RATE; + let cands: Vec<_> = rates + .clone() + .flat_map(|r| (minm..=maxm).map(move |m| oshape(m, r))) + .collect(); + let maxlev = cands.iter().map(|c| c.n_levels).max().unwrap(); + let maxfolds = cands.iter().map(|c| c.fold_grinding_bits.len()).max().unwrap(); + let maxsvk = cands.iter().map(|c| c.vanish_values.len()).max().unwrap(); + let maxood = cands.iter().flat_map(|c| &c.ood_samples).copied().max().unwrap_or(0); ps("LIG_MAX_LEVELS", maxlev.to_string()); ps("LIG_MAX_TOTAL_FOLDS", maxfolds.to_string()); ps("LIG_MAX_VANISH_LEN", maxsvk.to_string()); + ps("LIG_MAX_OOD_SAMPLES", maxood.to_string()); ps("LIG_MIN_LOG_SIZE", minm.to_string()); let cks: Vec<(usize, usize)> = lean_vm::cpu::col_kappa_sources(kbc).into_iter().flatten().collect(); ps("N_COMMITTED_COLS", cks.len().to_string()); ps("COL_KAPPA_SRC", ints(&cks.iter().map(|&(s, _)| s).collect::>())); ps("COL_KAPPA_ADJ", ints(&cks.iter().map(|&(_, a)| a).collect::>())); ps("PCS_MIN_MU", lean_vm::pcs::MIN_MU.to_string()); - ps("LIG_LOG_MSG_COLS_CAP", cands.iter().map(|c| *c.4.iter().max().unwrap()).max().unwrap().to_string()); - ps("YR_LOG_CAP", cands.iter().map(|c| c.2).max().unwrap().to_string()); + ps( + "LIG_LOG_MSG_COLS_CAP", + cands + .iter() + .map(|c| *c.log_message_columns.iter().max().unwrap()) + .max() + .unwrap() + .to_string(), + ); + ps( + "YR_LOG_CAP", + cands.iter().map(|c| c.yr_log_len).max().unwrap().to_string(), + ); { - let pad = |v: &[usize], stride: usize| -> Vec { let mut o = v.to_vec(); o.resize(stride, 0); o }; - let flat = |f: &dyn Fn(&(usize, usize, usize, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec)) -> Vec, stride: usize| -> Vec { cands.iter().flat_map(|c| pad(&f(c), stride)).collect() }; - let scal = |f: &dyn Fn(&(usize, usize, usize, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec, Vec)) -> usize| -> Vec { cands.iter().map(f).collect() }; - ps("LIG_N_LEVELS", ints(&scal(&|c| c.0))); - ps("LIG_YR_LEVEL", ints(&scal(&|c| c.1))); - ps("LIG_YR_LOG_LEN", ints(&scal(&|c| c.2))); - ps("LIG_YR_LEN", ints(&scal(&|c| 1usize << c.2))); - ps("LIG_TOTAL_FOLDS", ints(&scal(&|c| c.3.iter().sum()))); - ps("LIG_MAX_QUERIES", ints(&scal(&|c| *c.5.iter().max().unwrap()))); - ps("LIG_MAX_SQUEEZES", ints(&scal(&|c| *c.8.iter().max().unwrap()))); - ps("LIG_MAX_LOG_MSG_COLS", ints(&scal(&|c| *c.4.iter().max().unwrap()))); - ps("LIG_MAX_INTERLEAVE", ints(&scal(&|c| *c.9.iter().max().unwrap()))); - ps("LIG_POSITIONS_LEN", ints(&scal(&|c| (0..c.0).map(|lv| c.8[lv] * c.7[lv]).sum()))); - ps("LIG_SUMCHECK_LEN", ints(&scal(&|c| 2 * (c.3.iter().sum::() + c.0)))); - ps("LIG_ROWS_LEN", ints(&scal(&|c| (0..c.0).map(|lv| c.5[lv] * c.9[lv]).sum()))); - ps("LIG_PATHS_LEN", ints(&scal(&|c| (0..c.0).map(|lv| c.5[lv] * c.6[lv] * 2).sum()))); - ps("LIG_FOLD_GRIND_LEN", ints(&scal(&|c| c.3.iter().sum::() * 128))); - ps("LIG_QUERY_GRIND_BITS", ints(&flat(&|c| c.10.clone(), maxlev))); - ps("LIG_QUERIES", ints(&flat(&|c| c.5.clone(), maxlev))); - ps("LIG_FOLDS", ints(&flat(&|c| c.3.clone(), maxlev))); - ps("LIG_INTERLEAVE", ints(&flat(&|c| c.9.clone(), maxlev))); - ps("LIG_LEAF_PAIRS", ints(&flat(&|c| c.9.iter().map(|&n| n / 2).collect(), maxlev))); - ps("LIG_LEAF_BLOCKS", ints(&flat(&|c| c.9.iter().map(|&n| n / 4).collect(), maxlev))); - ps("LIG_TREE_DEPTH", ints(&flat(&|c| c.6.clone(), maxlev))); - ps("LIG_SQUEEZES", ints(&flat(&|c| c.8.clone(), maxlev))); - ps("LIG_POSITIONS_OFF", ints(&flat(&|c| c.15.clone(), maxlev))); - ps("LIG_LOG_QUERIES", ints(&flat(&|c| c.5.iter().map(|&q| log2_ceil(q)).collect(), maxlev))); - ps("LIG_LOG_MSG_COLS", ints(&flat(&|c| c.4.clone(), maxlev))); - ps("LIG_RESIDUAL_FOLD_OFF", ints(&flat(&|c| c.18.clone(), maxlev))); - ps("LIG_RESIDUAL_PREFIX_LEN", ints(&flat(&|c| c.4.iter().map(|&m2| m2 - c.2).collect(), maxlev))); - ps("LIG_FOLDS_OFF", ints(&flat(&|c| c.17.clone(), maxlev))); - ps("LIG_ROWS_OFF", ints(&flat(&|c| c.12.clone(), maxlev))); - ps("LIG_PATHS_OFF", ints(&flat(&|c| c.13.clone(), maxlev))); - ps("LIG_VANISH_OFF", ints(&flat(&|c| c.16.clone(), maxlev))); - ps("LIG_FOLD_GRIND_BITS", ints(&flat(&|c| c.11.clone(), maxfolds))); + let pad = |v: &[usize], stride: usize| -> Vec { + let mut o = v.to_vec(); + o.resize(stride, 0); + o + }; + let flat = |f: &dyn Fn(&OpeningShape) -> Vec, stride: usize| -> Vec { + cands.iter().flat_map(|c| pad(&f(c), stride)).collect() + }; + let scal = |f: &dyn Fn(&OpeningShape) -> usize| -> Vec { cands.iter().map(f).collect() }; + ps("LIG_N_LEVELS", ints(&scal(&|c| c.n_levels))); + ps("LIG_YR_LEVEL", ints(&scal(&|c| c.yr_level))); + ps("LIG_YR_LOG_LEN", ints(&scal(&|c| c.yr_log_len))); + ps("LIG_YR_LEN", ints(&scal(&|c| 1usize << c.yr_log_len))); + ps("LIG_TOTAL_FOLDS", ints(&scal(&|c| c.folds.iter().sum()))); + ps("LIG_MAX_QUERIES", ints(&scal(&|c| *c.queries.iter().max().unwrap()))); + ps("LIG_MAX_SQUEEZES", ints(&scal(&|c| *c.squeezes.iter().max().unwrap()))); + ps( + "LIG_MAX_LOG_MSG_COLS", + ints(&scal(&|c| *c.log_message_columns.iter().max().unwrap())), + ); + ps( + "LIG_MAX_INTERLEAVE", + ints(&scal(&|c| *c.interleaving.iter().max().unwrap())), + ); + // StackBuf cap for the packed leaf row AND the raw-limb `lanes` scratch + // that shares it (`open_stacked`). Level 0 packs 2 base-field lanes per + // cell (n/2 cells). Deeper levels first load 3 raw tower limbs per word + // into `lanes` (3n cells), then pack them into the 3n/2-cell leaf row, + // so `lanes` (3n) is the binding size there. Sizing the deeper term at + // 3n/2 happened to hold only while L0's n/2 dominated (small folds); + // it under-provisions once a deeper interleave exceeds L0's. + let packed_cells = |c: &Vec| -> usize { + c.iter() + .enumerate() + .map(|(lv, &n)| if lv == 0 { n / 2 } else { 3 * n }) + .max() + .unwrap() + }; + ps( + "LIG_PACKED_ROW_CAP", + cands + .iter() + .map(|c| packed_cells(&c.interleaving)) + .max() + .unwrap() + .to_string(), + ); + ps( + "LIG_POSITIONS_LEN", + ints(&scal(&|c| { + (0..c.n_levels) + .map(|level| c.squeezes[level] * c.positions_per_squeeze[level]) + .sum() + })), + ); + ps( + "LIG_SUMCHECK_LEN", + ints( + &cands + .iter() + .map(|c| 2 * (c.folds.iter().sum::() + c.n_levels + c.ood_samples.iter().sum::())) + .collect::>(), + ), + ); + ps( + "LIG_ROWS_LEN", + ints(&scal(&|c| { + (0..c.n_levels) + .map(|level| c.queries[level] * c.interleaving[level] * if level == 0 { 1 } else { 3 }) + .sum() + })), + ); + ps( + "LIG_PATHS_LEN", + ints(&scal(&|c| { + (0..c.n_levels) + .map(|level| c.queries[level] * c.tree_depths[level] * 2) + .sum() + })), + ); + ps( + "LIG_QUERY_GRIND_BITS", + ints(&flat(&|c| c.query_grinding_bits.clone(), maxlev)), + ); + ps( + "LIG_OOD_SAMPLES", + ints( + &cands + .iter() + .flat_map(|shape| pad(&shape.ood_samples, maxlev)) + .collect::>(), + ), + ); + ps("LIG_QUERIES", ints(&flat(&|c| c.queries.clone(), maxlev))); + ps("LIG_FOLDS", ints(&flat(&|c| c.folds.clone(), maxlev))); + ps("LIG_INTERLEAVE", ints(&flat(&|c| c.interleaving.clone(), maxlev))); + ps( + "LIG_LEAF_PAIRS", + ints(&flat( + &|c| { + c.interleaving + .iter() + .enumerate() + .map(|(lv, &n)| if lv == 0 { n / 4 } else { 3 * n / 4 }) + .collect() + }, + maxlev, + )), + ); + // 64-byte BLAKE3 blocks per leaf row: level 0's committed rows are + // base-field F64 (8 bytes/lane); deeper levels are native F192 + // (24 bytes/word, received as three embedded K limbs each). Rows are + // whole blocks only (asserted at candidate construction). + ps( + "LIG_LEAF_BLOCKS", + ints(&flat( + &|c| { + c.interleaving + .iter() + .enumerate() + .map(|(lv, &n)| if lv == 0 { n / 8 } else { 3 * n / 8 }) + .collect() + }, + maxlev, + )), + ); + ps("LIG_TREE_DEPTH", ints(&flat(&|c| c.tree_depths.clone(), maxlev))); + ps("LIG_SQUEEZES", ints(&flat(&|c| c.squeezes.clone(), maxlev))); + ps( + "LIG_POSITIONS_OFF", + ints(&flat(&|c| c.positions_offsets.clone(), maxlev)), + ); + ps( + "LIG_LOG_QUERIES", + ints(&flat( + &|c| c.queries.iter().map(|&queries| log2_ceil(queries)).collect(), + maxlev, + )), + ); + ps( + "LIG_LOG_MSG_COLS", + ints(&flat(&|c| c.log_message_columns.clone(), maxlev)), + ); + ps( + "LIG_RESIDUAL_FOLD_OFF", + ints(&flat(&|c| c.residual_fold_offsets.clone(), maxlev)), + ); + ps( + "LIG_RESIDUAL_PREFIX_LEN", + ints(&flat( + &|c| { + c.log_message_columns + .iter() + .map(|&columns| columns - c.yr_log_len) + .collect() + }, + maxlev, + )), + ); + ps("LIG_FOLDS_OFF", ints(&flat(&|c| c.fold_offsets.clone(), maxlev))); + ps("LIG_ROWS_OFF", ints(&flat(&|c| c.row_offsets.clone(), maxlev))); + ps("LIG_PATHS_OFF", ints(&flat(&|c| c.path_offsets.clone(), maxlev))); + ps("LIG_VANISH_OFF", ints(&flat(&|c| c.vanish_offsets.clone(), maxlev))); + ps( + "LIG_FOLD_GRIND_BITS", + ints(&flat(&|c| c.fold_grinding_bits.clone(), maxfolds)), + ); let mut svk2 = Vec::new(); let mut ivk2 = Vec::new(); for c in &cands { - let mut s = c.19.clone(); - let mut iv = c.20.clone(); - s.resize(maxsvk, F128::ZERO); - iv.resize(maxsvk, F128::ZERO); + let mut s = c.vanish_values.clone(); + let mut iv = c.vanish_inverses.clone(); + s.resize(maxsvk, F192::ZERO); + iv.resize(maxsvk, F192::ZERO); svk2.extend(s); ivk2.extend(iv); } ps("LIG_VANISH_VALS", flds(&svk2)); ps("LIG_VANISH_INVS", flds(&ivk2)); } - ps("LIG_N_CANDIDATES", (maxm - minm + 1).to_string()); - ps("LIG_MIN_SHIFT_INV", u(g_pow(minm).inv()).to_string()); + let n_log_sizes = maxm - minm + 1; + let n_rates = pcs::ligerito::MAX_LOG_INV_RATE - pcs::ligerito::MIN_LOG_INV_RATE + 1; + ps("LIG_N_LOG_SIZES", n_log_sizes.to_string()); + ps("LIG_N_RATES", n_rates.to_string()); + ps("LIG_N_CANDIDATES", (n_log_sizes * n_rates).to_string()); + ps("LIG_MIN_SHIFT_INV", u(F192::new(g_pow(minm).inv().0, 0, 0)).to_string()); ps("CLAIM_POINT_BUF", ints(&cpbuf)); ps("CLAIM_COMMITTED_COL", ints(&cpcol)); + let slot_stride_log = lean_vm::blake3_flock::SLOT_STRIDE_LOG; let cpqbits: Vec = cpqslot .iter() - .flat_map(|&slot| (0..7).map(move |k| (slot >> k) & 1)) + .flat_map(|&slot| (0..slot_stride_log).map(move |k| (slot >> k) & 1)) .collect(); ps("CLAIM_QPKD_SLOT_BITS", ints(&cpqbits)); ps("QPKD_COMMITTED_COL", qpkd_compact.to_string()); - ps("QPKD_VARS_CAP", (33 + flock::blake3::K_LOG - 7).to_string()); + ps("QPKD_VARS_CAP", (33 + slot_stride_log).to_string()); ps("BYTECODE_LOG", kbc.to_string()); // The stacked bytecode: nbcv/2 encoding columns per side, packed along // log2_ceil(cols) selector bits. The defer region is 2*kbc points + sel @@ -1396,30 +1919,70 @@ fn placeholder_map(program: &Program) -> BTreeMap { ps("LOG2_BYTECODE_COLS", log2_bc_cols.to_string()); ps("DEFER_SIZE", (kbc + log2_bc_cols + 2 * lcrounds + 68).to_string()); ps("BYTECODE_VARS", (kbc + log2_bc_cols).to_string()); - let label_state = Sponge::new(b"leanvm-b", &[]).state(); + let label_state = pack_state(Sponge::new(b"leanvm-b", &[]).state()); ps("TRANSCRIPT_SEED_0", u(label_state[0]).to_string()); ps("TRANSCRIPT_SEED_1", u(label_state[1]).to_string()); - let agg_state = Sponge::new(RECURSION_AGG_LABEL, &[]).state(); + let agg_state = pack_state(Sponge::new(RECURSION_AGG_LABEL, &[]).state()); ps("AGG_SEED_0", u(agg_state[0]).to_string()); ps("AGG_SEED_1", u(agg_state[1]).to_string()); - let statement_state = Sponge::new(RECURSION_STATEMENT_LABEL, &[]).state(); + let statement_state = pack_state(Sponge::new(RECURSION_STATEMENT_LABEL, &[]).state()); ps("STATEMENT_SEED_0", u(statement_state[0]).to_string()); ps("STATEMENT_SEED_1", u(statement_state[1]).to_string()); - let rs_coeff_orbits: Vec = - pcs::ring_switch::eq_linearized_orbit_constants().iter().flatten().copied().collect(); - ps("RS_COEFF_ORBITS", flds(&rs_coeff_orbits)); + // Closed-form ring-switch coefficients: the guest bakes both Frobenius + // orbits in, so it needs neither a runtime orbit table nor a 63-term rep } -/// Compile the canonical recursion guest for this program and batch arity. -/// Both proving and verification use this function so they cannot drift. -fn recursion_guest(inner_program: &Program, nsub: usize) -> Program { +/// Return the process-cached recursion guest for this program and batch arity. +fn recursion_guest_arc(inner_program: &Program, nsub: usize) -> std::sync::Arc { + use std::sync::{Arc, Mutex, OnceLock}; + + type Key = ([u64; 6], usize); + static CACHE: OnceLock>>> = OnceLock::new(); + const GUEST_CACHE_CAP: usize = 8; + + let seed = lean_vm::cpu::fs_seed(inner_program); + let key = ( + [seed[0].c0, seed[0].c1, seed[0].c2, seed[1].c0, seed[1].c1, seed[1].c2], + nsub, + ); + let cache = CACHE.get_or_init(Default::default); + if let Some(guest) = cache.lock().expect("recursion guest cache poisoned").get(&key) { + return Arc::clone(guest); + } + let mut replacements = placeholder_map(inner_program); replacements.insert("NSUB_PLACEHOLDER".to_string(), nsub.to_string()); - compile( + // `DBG_PLACEHOLDERS=path`: dump the baked guest constants, to read alongside + // a `DBG_PROF_DUMP` profile (the guest's shape is entirely in these). + if let Ok(path) = std::env::var("DBG_PLACEHOLDERS") { + let dump: String = replacements.iter().map(|(k, v)| format!("{k} = {v}\n")).collect(); + std::fs::write(&path, dump).expect("write DBG_PLACEHOLDERS"); + } + let guest = Arc::new(compile( &parse_with_replacements(include_str!("../guests/recursion.py"), &replacements) .expect("the repository recursion guest must parse"), - ) + )); + + // `DBG_DISASM=path`: dump the guest's disassembly, to read alongside a + // `DBG_PROF_DUMP` per-pc profile. + if let Ok(path) = std::env::var("DBG_DISASM") { + std::fs::write(&path, lean_compiler::disassemble(&guest.prog)).expect("write DBG_DISASM"); + } + + let mut map = cache.lock().expect("recursion guest cache poisoned"); + if let Some(cached) = map.get(&key) { + return Arc::clone(cached); + } + if map.len() < GUEST_CACHE_CAP { + map.insert(key, Arc::clone(&guest)); + } + guest +} + +/// Return an owned guest whose witness streams may be mutated by the prover. +fn recursion_guest(inner_program: &Program, nsub: usize) -> Program { + (*recursion_guest_arc(inner_program, nsub)).clone() } /// Run an `inner.len()`→1 recursive aggregation and verify the outer proof; @@ -1430,9 +1993,19 @@ fn recursion_guest(inner_program: &Program, nsub: usize) -> Program { /// map needs only that size); /// 3. prove the inner proofs (and extract their hints); /// 4. prove the recursion, verify, discharge the three reduced claims. -/// When `enable_tracing` is true, tracing starts after the inner proofs so the -/// emitted tree profiles the recursive aggregation itself. -pub fn run_recursion(inner: &[(usize, usize)], enable_tracing: bool) -> RecursiveProof { +pub fn run_recursion(inner: &[(usize, usize)], log_inv_rate: usize, enable_tracing: bool) -> RecursiveProof { + let rates = vec![log_inv_rate; inner.len()]; + run_recursion_with_rates(inner, &rates, log_inv_rate, enable_tracing) +} + +/// Run recursion with one transcript-bound PCS rate per inner proof. The guest +/// bytecode is independent of these values and supports mixed-rate batches. +fn run_recursion_with_rates( + inner: &[(usize, usize)], + log_inv_rates: &[usize], + outer_log_inv_rate: usize, + enable_tracing: bool, +) -> RecursiveProof { // 1 + 2: the recursion program is generic — its map needs only the inner // bytecode size — so it is compiled FIRST, before any inner proof. let program = inner_program(); @@ -1441,21 +2014,15 @@ pub fn run_recursion(inner: &[(usize, usize)], enable_tracing: bool) -> Recursiv let t_compile = t.elapsed(); // The recursion program size + compile time, BEFORE any inner proving. let real_instrs: usize = guest.fn_ranges.iter().map(|(_, _, len)| *len as usize).sum(); - eprintln!( - "recursion program: {} instructions (2^{} padded), compiled in {} s", - pretty_integer(real_instrs), - pretty_integer(guest.prog.len().trailing_zeros()), - pretty_f64(t_compile.as_secs_f64()) - ); // 3: prove the inner proofs and extract the recursion witness (hints). - let batch = build_batch(inner); + let batch = build_batch(inner, log_inv_rates, outer_log_inv_rate); let nsub = batch.nsub; let total_inner_cycles = batch.total_inner_cycles; if enable_tracing { primitives::init_tracing(); } let trace_span = - tracing::info_span!("Recursive aggregation", n = %pretty_integer(nsub)).entered(); + tracing::info_span!("Recursive aggregation", n = nsub, log_inv_rate = outer_log_inv_rate).entered(); let t = std::time::Instant::now(); let (recursive_proof, stats) = batch.prove(&mut guest); let t_prove = t.elapsed(); @@ -1464,11 +2031,23 @@ pub fn run_recursion(inner: &[(usize, usize)], enable_tracing: bool) -> Recursiv .verify(&batch.program0) .expect("complete recursive proof verifies"); let t_verify = t.elapsed(); - let proof_bytes = bincode::serialized_size(&recursive_proof).expect("recursive proof is serializable"); - // tracing-forest renders the tree when its root span closes. Close it - // before printing the benchmark report so the complete trace appears first. + // tracing-forest renders a tree when its root span closes. Close it before + // printing any benchmark/status output so the complete trace appears first. drop(trace_span); + println!( + "recursion program: {} instructions (2^{} padded), compiled in {t_compile:?}", + pretty_integer(real_instrs), + guest.prog.len().trailing_zeros() + ); + for &(cycles, committed) in &batch.inner_stats { + println!( + "[inner] cycles={} committed=2^{}", + pretty_integer(cycles), + pretty_f64((committed as f64).log2()) + ); + } + let proof_bytes = bincode::serialized_size(&recursive_proof).expect("recursive proof is serializable"); let pow = |x: usize| { if x == 0 { " -".into() @@ -1476,24 +2055,23 @@ pub fn run_recursion(inner: &[(usize, usize)], enable_tracing: bool) -> Recursiv format!("2^{}", pretty_f64((x as f64).log2())) } }; + let nsub_pretty = pretty_integer(nsub); println!( - "\nrecursion {}\u{2192}1: {} inner proofs of {} cycles each", - pretty_integer(nsub), - pretty_integer(nsub), + "\nrecursion {nsub_pretty}\u{2192}1: {nsub_pretty} inner proofs of {} cycles each", pretty_integer(total_inner_cycles / nsub) ); + let guest_cycles = pretty_integer(stats.cycles); println!( - " guest cycles (VM steps) : {:>14} = {:>9} ({} / inner cycle)", - pretty_integer(stats.cycles), + " guest cycles (VM steps) : {guest_cycles:>14} = {:>9} ({} / inner cycle)", pow(stats.cycles), pretty_f64(stats.cycles as f64 / total_inner_cycles as f64) ); - for (name, &c) in ["XOR", "MUL", "SET", "DEREF", "JUMP", "BLAKE3"].iter().zip(&stats.counts) { - println!( - " {name:<6} instructions : {:>14} = {:>9}", - pretty_integer(c), - pow(c) - ); + for (name, &c) in ["XOR", "MUL", "SET", "DEREF", "JUMP", "BLAKE3", "PACK64X2"] + .iter() + .zip(&stats.counts) + { + let count = pretty_integer(c); + println!(" {name:<6} instructions : {count:>14} = {:>9}", pow(c)); } println!( " committed witness size : 2^{}", @@ -1519,26 +2097,24 @@ pub fn run_recursion(inner: &[(usize, usize)], enable_tracing: bool) -> Recursiv recursive_proof } -/// THE recursion test: two ~1M-cycle inner proofs (log_mem 21, committed -/// 2^24.6, an m=33 stacked opening each), verified and aggregated by one -/// guest into one outer proof, whose three reduced claims are then discharged -/// natively. +/// End-to-end recursion test: two ordinary proofs are verified and aggregated +/// by one guest, then its three reduced claims are discharged natively. #[test] fn recursion_2to1() { - run_recursion(&[(8, 1 << 15), (8, 1 << 15)], false); + run_recursion(&[(8, 1 << 15), (8, 1 << 15)], lean_vm::pcs::LOG_INV_RATE, false); } /// THE genericity milestone: ONE compiled guest bytecode verifies two inner -/// proofs of DIFFERENT sizes in the same aggregation (the placeholder map -/// depends only on the inner bytecode size, so one map covers both shapes). +/// proofs of DIFFERENT sizes and rates in the same aggregation (the placeholder +/// map depends only on the inner bytecode size, so one map covers both shapes). #[test] fn recursion_2to1_mixed() { - run_recursion(&[(4, 1 << 13), (64, 1 << 15)], false); + run_recursion_with_rates(&[(4, 1 << 13), (64, 1 << 15)], &[1, 4], 3, false); } /// One compiled guest bytecode proves MANY inner runs with wildly different /// opcode profiles and sizes, without recompilation. The configs span four -/// committed sizes (m in {22,23,24,25} - four distinct match_range opening +/// committed sizes (m in {23,24,25,26} - four distinct match_range opening /// arms) and four BLAKE3 log-instance-counts (tau_5 in {3,4,5,6} - different /// r1cs statement digests, flock reduction sizes, and pin prefixes). The /// guest is compiled ONCE from the placeholder map, which is a function of the @@ -1552,16 +2128,16 @@ fn recursion_soundness_binds() { // hints reject; and all commitment-placement descriptors are absent from the // witness. Ignored because it runs several full inner+outer proofs. let cfg: &[(usize, usize)] = &[(4, 1 << 12)]; - let batch = build_batch(cfg); + let batch = build_batch(cfg, &[lean_vm::pcs::LOG_INV_RATE], lean_vm::pcs::LOG_INV_RATE); let mut guest = recursion_guest(&batch.program0, cfg.len()); let public_input = batch.public_input(); - let run = |g: &mut Program, merged: &[(String, Vec>)]| -> bool { + let run = |g: &mut Program, merged: &[(String, Vec>)]| -> bool { for (name, entries) in merged { g.set_witness(name, entries.clone()); } std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let (proof, _) = prove(g, public_input); + let (proof, _) = prove(g, public_input, lean_vm::pcs::LOG_INV_RATE); verify(g, &public_input, &proof).is_ok() })) .unwrap_or(false) @@ -1571,23 +2147,19 @@ fn recursion_soundness_binds() { assert!( batch.merged.iter().all(|(name, _)| !matches!( name.as_str(), - "claim_sel_bits" - | "claim_yslot_bits" - | "claim_qpkd_slot_bits" - | "rs_sel_bits" - | "rs_yslot_bits" + "claim_sel_bits" | "claim_yslot_bits" | "claim_qpkd_slot_bits" | "rs_sel_bits" | "rs_yslot_bits" )), "claim and ring placement descriptors must be derived, not hinted" ); // each tamper flips one hint to a definitely-invalid value. - let tampers: Vec<(&str, usize, F128)> = vec![ - ("fs_seed", 0, F128::ONE), // wrong proving environment: own_pi (public input) must reject - ("stream", 0, F128::new((lean_vm::cpu::MIN_LOG_MEM - 1) as u64, 0)), // native memory floor - ("stream", 1, F128::new(1u64 << 32, 0)), // native row counts are strictly below 2^32 - ("claim_nover", 0, g_pow(5)), // wrong overlap: exact length pin must reject - ("pi_cplen", 0, g_pow(2)), // wrong pi dimension: min-cert must reject - ("zc_tau_max", 0, g_pow(2)), // not the max tau: the max-cert must reject + let tampers: Vec<(&str, usize, F192)> = vec![ + ("fs_seed", 0, F192::ONE), // wrong proving environment: own_pi must reject + ("stream", 0, F192::new((lean_vm::cpu::MIN_LOG_MEM - 1) as u64, 0, 0)), // native memory floor + ("stream", 1, F192::new(1u64 << 32, 0, 0)), // native row counts are strictly below 2^32 + ("claim_nover", 0, F192::new(g_pow(5).0, 0, 0)), + ("pi_cplen", 0, F192::new(g_pow(2).0, 0, 0)), + ("zc_tau_max", 0, F192::new(g_pow(2).0, 0, 0)), ]; for &(stream, idx, val) in &tampers { let mut merged = batch.merged.clone(); @@ -1612,10 +2184,16 @@ fn recursion_soundness_binds() { // tie-break, so the reconstructed commitment offsets must reject it. { let mut merged = batch.merged.clone(); - let pos = merged.iter().position(|(n, _)| n == "col_sort_order").expect("col_sort_order"); + let pos = merged + .iter() + .position(|(n, _)| n == "col_sort_order") + .expect("col_sort_order"); assert!(merged[pos].1[0].len() >= 2); merged[pos].1[0].swap(0, 1); - assert!(!run(&mut guest, &merged), "non-canonical col_sort_order must be rejected"); + assert!( + !run(&mut guest, &merged), + "non-canonical col_sort_order must be rejected" + ); } // (The overlap mask is no longer a hint: the guest selects a baked // prefix-mask row by the pinned nover, so the point-reuse y-slot @@ -1628,12 +2206,12 @@ fn recursion_soundness_binds() { fn recursion_generic_many() { // (hashes, iters) per inner run - deliberately diverse profiles. let configs: &[(usize, usize)] = &[ - (4, 1 << 12), // m=22, tau_5=3 - (8, 1 << 13), // m=23, tau_5=3 - (16, 1 << 14), // m=24, tau_5=4 - (8, 1 << 15), // m=25, tau_5=3 - (32, 1 << 13), // m=23, tau_5=5 - (64, 1 << 13), // m=23, tau_5=6 + (4, 1 << 12), // m=23, tau_5=3 + (8, 1 << 13), // m=24, tau_5=3 + (16, 1 << 14), // m=25, tau_5=4 + (8, 1 << 15), // m=26, tau_5=3 + (32, 1 << 13), // m=24, tau_5=5 + (64, 1 << 13), // m=24, tau_5=6 ]; // The recursion program is generic: compile it ONCE, from the inner program's // size alone, BEFORE any inner proof exists. Genericity is then shown directly @@ -1641,7 +2219,7 @@ fn recursion_generic_many() { let mut guest = recursion_guest(&inner_program(), 1); eprintln!("guest compiled ONCE ({} instrs)", pretty_integer(guest.prog.len())); for &cfg in configs { - let batch = build_batch(&[cfg]); + let batch = build_batch(&[cfg], &[lean_vm::pcs::LOG_INV_RATE], lean_vm::pcs::LOG_INV_RATE); let (recursive_proof, _) = batch.prove(&mut guest); recursive_proof .verify(&batch.program0) @@ -1649,7 +2227,7 @@ fn recursion_generic_many() { eprintln!( " verified: hashes={:>2}, iters=2^{}", pretty_integer(cfg.0), - pretty_integer((cfg.1 as f64).log2() as u32) + (cfg.1 as f64).log2() as u32 ); } eprintln!( diff --git a/crates/rec_aggregation/src/signers_cache.rs b/crates/rec_aggregation/src/signers_cache.rs index f45bd04a..334057fa 100644 --- a/crates/rec_aggregation/src/signers_cache.rs +++ b/crates/rec_aggregation/src/signers_cache.rs @@ -15,17 +15,17 @@ //! Cache location: `/target/signers-cache/` (already git-ignored). //! The filename carries a footprint of everything that determines the signers — //! not just the declared parameters but a known-answer of the hash construction -//! itself (see [`hash_fingerprint`]), so a branch that changes the digests +//! itself (see `hash_fingerprint`), so a branch that changes the digests //! (e.g. a change to the standard BLAKE3 input encoding) without touching a //! single constant still lands in a fresh file rather than mis-loading the //! other branch's signers. The WOTS encoding *predicate* is fingerprinted the -//! same way (see [`encoding_fingerprint`]): two branches can agree on every +//! same way (see `encoding_fingerprint`): two branches can agree on every //! constant and every hash digest yet lay the digest out into digits //! differently, which silently invalidates every ground randomness. As a last //! line of defense, loaded signers are re-verified and the pool truncated at -//! the first invalid one ([`try_load_cache`]), so a stale cache that slips +//! the first invalid one (`try_load_cache`), so a stale cache that slips //! past the footprint regenerates instead of panicking downstream. Bump -//! [`SCHEMA_VERSION`] to force regeneration by hand. +//! `SCHEMA_VERSION` to force regeneration by hand. use std::collections::hash_map::DefaultHasher; use std::fs; @@ -35,9 +35,9 @@ use std::path::PathBuf; use std::sync::Mutex; use std::time::Instant; +use primitives::{pretty_f64, pretty_integer}; use rand::SeedableRng; use rand::rngs::StdRng; -use primitives::{pretty_f64, pretty_integer}; use xmss::*; /// A cached signer: its public key and one signature over [`message`] at [`SLOT`]. @@ -236,8 +236,7 @@ mod tests { let msg = message(); for (i, (pk, sig)) in large.iter().enumerate() { - xmss_verify(pk, &msg, sig, SLOT) - .unwrap_or_else(|e| panic!("cached signer {i} failed to verify: {e:?}")); + xmss_verify(pk, &msg, sig, SLOT).unwrap_or_else(|e| panic!("cached signer {i} failed to verify: {e:?}")); } } } diff --git a/crates/rec_aggregation/src/xmss_aggregation.rs b/crates/rec_aggregation/src/xmss_aggregation.rs index 5fb2dd79..7631ed4f 100644 --- a/crates/rec_aggregation/src/xmss_aggregation.rs +++ b/crates/rec_aggregation/src/xmss_aggregation.rs @@ -12,22 +12,36 @@ use std::collections::BTreeMap; use lean_compiler::{compile, parse_file_with_replacements}; use lean_vm::cpu::{prove, verify}; use primitives::{ - field::{F128, g_pow}, + field::{F64, F192, g_pow}, pretty_f64, pretty_integer, }; use xmss::*; use crate::signers_cache; -fn word(bytes: &[u8]) -> F128 { - F128::new( - u64::from_le_bytes(bytes[..8].try_into().unwrap()), - u64::from_le_bytes(bytes[8..16].try_into().unwrap()), - ) +fn word(bytes: &[u8]) -> F64 { + F64(u64::from_le_bytes(bytes[..8].try_into().unwrap())) } -fn pair(a: &[u8], b: &[u8]) -> Vec { - vec![word(a), word(b)] +/// A K-embedded F192 cell for count/digit hints, which are g-powers. +fn cell(w: F64) -> F192 { + F192::from(w) +} + +/// A 16-byte native value in the canonical BLAKE3 subspace of F192: `c0` +/// carries bytes 0..8, `c1` bytes 8..16, and `c2` is zero. +fn val16(b: &[u8]) -> F192 { + F192::new(word(&b[..8]).0, word(&b[8..16]).0, 0) +} + +/// A 16-byte native value as ONE cell. +fn pair(b: &[u8]) -> Vec { + vec![val16(b)] +} + +/// A 32-byte hash block as two canonical 128-bit BLAKE3 cells. +fn quad(b: &[u8]) -> Vec { + vec![val16(&b[..16]), val16(&b[16..32])] } /// Protocol-specific streaming binding used only by the runtime-sized XMSS @@ -49,8 +63,8 @@ fn aggregate_binding(mut state: [u8; STATE_LEN], data: &[u8]) -> [u8; STATE_LEN] /// natively with the `xmss` crate, runs the in-VM aggregation verifier /// (`guests/xmss_aggregate.py`) over all signatures, proves, verifies, and /// prints the benchmark report. -pub fn run_xmss_aggregation(n: usize) { - let trace_span = tracing::info_span!("XMSS aggregation", n = %pretty_integer(n)).entered(); +pub fn run_xmss_aggregation(n: usize, log_inv_rate: usize) { + let trace_span = tracing::info_span!("XMSS aggregation", n, log_inv_rate).entered(); // Pin rayon workers to performance cores (QoS) before any parallel work runs, // so fork-join stages are not held up by efficiency-core stragglers. Thread @@ -61,7 +75,7 @@ pub fn run_xmss_aggregation(n: usize) { // Generated once and cached to disk; see `signers_cache`. let signers = signers_cache::get_signers(n); - // The 328-word tweak table (word index — see the program header). The + // The 328-tweak table (tweak index — see the program header). The // Merkle parent index is `slot >> (level+1)` computed in u64 (a u32 shift // by 32 at the top level would mask, not zero). let mut tweaks: Vec = vec![make_tweak(TWEAK_TYPE_ENCODING, 0, slot)]; @@ -94,9 +108,10 @@ pub fn run_xmss_aggregation(n: usize) { let num_bytes = data.len(); assert_eq!(num_bytes, 5792 + 32 * n); let mut iv = [0u8; STATE_LEN]; - iv[..16].copy_from_slice(&g_pow(num_bytes).to_le_bytes()); + iv[..8].copy_from_slice(&g_pow(num_bytes).0.to_le_bytes()); let state = aggregate_binding(iv, &data); - let want = [word(&state[..16]), word(&state[16..])]; + // The guest publishes the final binding state's two 128-bit cells (32 bytes). + let want = [val16(&state[..16]), val16(&state[16..32])]; // The XMSS instance parameters, injected into the program's placeholders; // every derived size (tweak-table width, IV byte counts, …) is computed @@ -114,17 +129,41 @@ pub fn run_xmss_aggregation(n: usize) { ) .expect("parse"), ); - program.set_witness("n_pks", vec![vec![g_pow(n)]]); - program.set_witness("msg", vec![pair(&message[..16], &message[16..])]); - program.set_witness("tweaks", tweaks.chunks(2).map(|c| pair(&c[0], &c[1])).collect()); - let bit_word = |l: usize| F128::new(((slot >> l) & 1) as u64, 0); + program.set_witness("n_pks", vec![vec![cell(g_pow(n))]]); + program.set_witness("msg", vec![quad(&message)]); + program.set_witness( + "tweaks", + tweaks + .chunks(2) + .map(|c| { + let mut block = pair(&c[0]); + block.extend(pair(&c[1])); + block + }) + .collect(), + ); + // A merkle slot-bit as one 16-byte cell (bit in the low byte, rest zero). + let bit_word = |l: usize| vec![cell(F64(((slot >> l) & 1) as u64))]; program.set_witness( "merkle_bits", - (0..LOG_LIFETIME / 2).map(|u| vec![bit_word(2 * u), bit_word(2 * u + 1)]).collect(), + (0..LOG_LIFETIME / 2) + .map(|u| { + let mut block = bit_word(2 * u); + block.extend(bit_word(2 * u + 1)); + block + }) + .collect(), ); program.set_witness( "pks", - signers.iter().map(|(pk, _)| pair(&pk.merkle_root, &pk.public_param)).collect(), + signers + .iter() + .map(|(pk, _)| { + let mut block = pair(&pk.merkle_root); + block.extend(pair(&pk.public_param)); + block + }) + .collect(), ); // Per-signature streams, signature-major order. let (mut rand_s, mut digits_s, mut chain_starts_s, mut sib_s) = (vec![], vec![], vec![], vec![]); @@ -132,12 +171,11 @@ pub fn run_xmss_aggregation(n: usize) { let wots = &sig.wots_signature; let mut rnd = [0u8; STATE_LEN]; rnd[..RANDOMNESS_LEN].copy_from_slice(&wots.randomness); - rand_s.push(pair(&rnd[..16], &rnd[16..])); - let encoding = - wots_encode(&message, slot, &pk.public_param, &wots.randomness).expect("encoding"); - digits_s.extend(encoding.iter().map(|&e| vec![g_pow(e as usize)])); - chain_starts_s.extend(wots.chain_tips.iter().map(|t| vec![word(t)])); - sib_s.extend(sig.merkle_proof.iter().map(|s| vec![word(s)])); + rand_s.push(quad(&rnd)); + let encoding = wots_encode(&message, slot, &pk.public_param, &wots.randomness).expect("encoding"); + digits_s.extend(encoding.iter().map(|&e| vec![cell(g_pow(e as usize))])); + chain_starts_s.extend(wots.chain_tips.iter().map(|t| pair(t))); + sib_s.extend(sig.merkle_proof.iter().map(|s| pair(s))); } program.set_witness("rand", rand_s); program.set_witness("digits", digits_s); @@ -156,7 +194,7 @@ pub fn run_xmss_aggregation(n: usize) { lean_vm::blake3_flock::warm_setup(181 + 146 * n); let t = Instant::now(); - let (proof, stats) = prove(&program, want); + let (proof, stats) = prove(&program, want, log_inv_rate); let t_prove = t.elapsed(); let t = Instant::now(); verify(&program, &want, &proof).expect("XMSS aggregation verifies in-VM"); @@ -164,7 +202,8 @@ pub fn run_xmss_aggregation(n: usize) { // 181 fixed blocks + per signature: 1 (pk absorb) + 145 (the native // verifier's constant). - let bad = [want[0], want[1] + F128::ONE]; + assert_eq!(stats.counts[5], 181 + 146 * n, "BLAKE3 instruction count"); + let bad = [want[0], want[1] + F192::ONE]; assert!(verify(&program, &bad, &proof).is_err()); let proof_bytes = bincode::serialized_size(&proof).expect("proof is serializable"); @@ -187,9 +226,12 @@ pub fn run_xmss_aggregation(n: usize) { pow(stats.cycles), per(stats.cycles) ); - for (name, &c) in ["XOR", "MUL", "SET", "DEREF", "JUMP", "BLAKE3"].iter().zip(&stats.counts) { + for (name, &c) in ["XOR", "MUL", "SET", "DEREF", "JUMP", "BLAKE3", "PACK64X2"] + .iter() + .zip(&stats.counts) + { println!( - " {name:<6} instructions : {:>14} = {:>9} ({:>12} / XMSS)", + " {name:<8} instructions : {:>14} = {:>9} ({:>12} / XMSS)", pretty_integer(c), pow(c), per(c) @@ -212,7 +254,10 @@ pub fn run_xmss_aggregation(n: usize) { " proving (incl. witness gen) : {} s", pretty_f64(t_prove.as_secs_f64()) ); - println!(" verifying : {} s", pretty_f64(t_verify.as_secs_f64())); + println!( + " verifying : {} s", + pretty_f64(t_verify.as_secs_f64()) + ); println!( " throughput : {} XMSS/s", pretty_f64(n as f64 / t_prove.as_secs_f64()) @@ -224,7 +269,10 @@ mod tests { /// Batch size overridable: `LEANVM_XMSS_N=820 cargo test … -- --nocapture`. #[test] fn aggregate_xmss() { - let n = std::env::var("LEANVM_XMSS_N").ok().and_then(|s| s.parse().ok()).unwrap_or(3); - super::run_xmss_aggregation(n); + let n = std::env::var("LEANVM_XMSS_N") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(3); + super::run_xmss_aggregation(n, lean_vm::pcs::LOG_INV_RATE); } } diff --git a/crates/rec_aggregation/tests/hash_chain.rs b/crates/rec_aggregation/tests/hash_chain.rs index 1a82129f..ea43e84a 100644 --- a/crates/rec_aggregation/tests/hash_chain.rs +++ b/crates/rec_aggregation/tests/hash_chain.rs @@ -4,9 +4,10 @@ //! h_i)` (the previous value fed as both 256-bit operands). The program mirrors //! the Fibonacci demo's strategy: a `mul_range` loop *in the exponent* on the //! outside, an unrolled block of `BLAKE3` steps on the inside, with the chain -//! state carried through a `HeapBuf` (write-once memory). The final `h_N` is -//! published into the public input (`m[0], m[1]`); write-once memory forces the -//! proven result to equal it. +//! state carried through a `HeapBuf` (write-once memory). The four 64-bit digest +//! lanes of the final `h_N` are packed into two canonical 128-bit BLAKE3 cells +//! embedded in the F192 public input; +//! write-once memory forces the proven result to equal it. //! //! `N` and the unroll factor are read from the environment (`LEANVM_HASH_N`, //! `LEANVM_HASH_UNROLL`) so this doubles as a benchmark — e.g. @@ -16,46 +17,45 @@ use std::time::Instant; -use lean_vm::blake3_flock::warm_setup; use lean_compiler::{compile, parse}; +use lean_vm::blake3_flock::warm_setup; use lean_vm::cpu::{prove, verify}; -use primitives::{field::F128, pretty_f64, pretty_integer}; +use primitives::{ + field::{F64, F192}, + pretty_f64, pretty_integer, +}; -/// One compression step `c = BLAKE3(a, b)` (the VM's `blake3` builtin): the four +/// One compression step `c = BLAKE3(a, b)` (the VM's `blake3` builtin): the eight /// input words are laid little-endian into 64 bytes, BLAKE3-hashed, and the -/// 32-byte digest split into two `F128` words. Matches `cpu::blake3_compress`. -fn compress(a: [F128; 2], b: [F128; 2]) -> [F128; 2] { +/// 32-byte digest split into four `F64` words. Matches `cpu::blake3_compress`. +fn compress(a: [F64; 4], b: [F64; 4]) -> [F64; 4] { let mut input = [0u8; 64]; - for (slot, w) in input.chunks_exact_mut(16).zip([a[0], a[1], b[0], b[1]]) { - slot[..8].copy_from_slice(&w.lo.to_le_bytes()); - slot[8..].copy_from_slice(&w.hi.to_le_bytes()); + for (slot, w) in input.chunks_exact_mut(8).zip(a.into_iter().chain(b)) { + slot.copy_from_slice(&w.0.to_le_bytes()); } let d = blake3::hash(&input); let d = d.as_bytes(); - let word = |b: &[u8]| { - F128::new( - u64::from_le_bytes(b[..8].try_into().unwrap()), - u64::from_le_bytes(b[8..16].try_into().unwrap()), - ) - }; - [word(&d[..16]), word(&d[16..])] + std::array::from_fn(|k| F64(u64::from_le_bytes(d[8 * k..8 * k + 8].try_into().unwrap()))) } /// Build the zkDSL source for an `n`-step chain unrolled `unroll` per outer /// iteration (`k = n / unroll` iterations). Layout in the heap `buff`: the chain -/// value after `j·unroll` steps sits at cells `2j, 2j+1` (g-powers `g^{2j}, -/// g^{2j+1}`). Each outer step loads that pair into a size-2 `StackBuf`, runs -/// `unroll` `BLAKE3`s in the stack — each output pair feeds the next with **no -/// copies** (a self-hash `blake3(h, h, out)` aliases one pair into both input -/// operands) — then writes the result pair two cells along. +/// value after `j·unroll` steps sits at cells `2j, 2j+1`. Each outer step loads +/// that pair into a size-2 `StackBuf`, runs `unroll` `BLAKE3`s in the stack — +/// each output pair feeds the next with **no copies** (a self-hash aliases one +/// pair into both input operands) — then writes the result pair two cells along. fn chain_source(n: usize, unroll: usize) -> String { - assert!(unroll >= 1 && n.is_multiple_of(unroll), "N must be a positive multiple of UNROLL"); + assert!( + unroll >= 1 && n.is_multiple_of(unroll), + "N must be a positive multiple of UNROLL" + ); let k = n / unroll; let two_k = 2 * k; let mut body = String::new(); - // Block `j`'s boundary pair sits at cells `g^{2j}, g^{2j+1}`; the loop counter - // `i = gʲ` is the block index (×g each iteration), so the pair base is `b = i·i`. + // A 256-bit BLAKE3 value occupies two canonical 128-bit cells. Block `j`'s + // boundary value sits at cells `g^{2j}..g^{2j+1}`; the loop counter `i = gʲ` + // is the block index (×g each iteration), so the value base is `b = i²`. // Load the current chain value into a size-2 StackBuf (heap read straight // into the two consecutive stack cells). body.push_str(" b = i * i\n"); @@ -68,9 +68,10 @@ fn chain_source(n: usize, unroll: usize) -> String { body.push_str(&format!(" h{s} = StackBuf(2)\n")); body.push_str(&format!(" blake3(h{p}, h{p}, h{s})\n", p = s - 1)); } - // Write the block's result back to the next array pair. - body.push_str(&format!(" buff[b * GEN ** 2] = h{unroll}[0]\n")); - body.push_str(&format!(" buff[b * GEN ** 3] = h{unroll}[1]\n")); + // Write the block's result back to the next value (two cells along). + for w in 0..2 { + body.push_str(&format!(" buff[b * GEN ** {}] = h{unroll}[{w}]\n", 2 + w)); + } format!( "def main():\n\ @@ -90,19 +91,21 @@ fn chain_source(n: usize, unroll: usize) -> String { #[test] fn blake3_hash_chain() { - let env = |key: &str, default: usize| { - std::env::var(key).ok().and_then(|s| s.parse().ok()).unwrap_or(default) - }; + let env = |key: &str, default: usize| std::env::var(key).ok().and_then(|s| s.parse().ok()).unwrap_or(default); let unroll = env("LEANVM_HASH_UNROLL", 4); let n = env("LEANVM_HASH_N", 8); - assert!(n.is_multiple_of(unroll), "LEANVM_HASH_N must be a multiple of LEANVM_HASH_UNROLL"); + assert!( + n.is_multiple_of(unroll), + "LEANVM_HASH_N must be a multiple of LEANVM_HASH_UNROLL" + ); // Reference chain in O(1) memory: a rolling value, no array of intermediates. - let mut h = [F128::ZERO; 2]; + let mut h = [F64::ZERO; 4]; for _ in 0..n { h = compress(h, h); } - let pi = h; // the published final value h_N + // The two published BLAKE3 cells of h_N (top F192 limb zero). + let pi = [F192::new(h[0].0, h[1].0, 0), F192::new(h[2].0, h[3].0, 0)]; let program = compile(&parse(&chain_source(n, unroll)).expect("parse")); @@ -112,7 +115,7 @@ fn blake3_hash_chain() { warm_setup(n); let t = Instant::now(); - let (proof, stats) = prove(&program, pi); + let (proof, stats) = prove(&program, pi, lean_vm::pcs::LOG_INV_RATE); let t_prove = t.elapsed(); let t = Instant::now(); verify(&program, &pi, &proof).expect("hash-chain proof verifies"); @@ -126,7 +129,10 @@ fn blake3_hash_chain() { pretty_integer(unroll) ); println!(" cycles (VM steps) : {}", pretty_integer(stats.cycles)); - for (name, &c) in ["XOR", "MUL", "SET", "DEREF", "JUMP", "BLAKE3"].iter().zip(&stats.counts) { + for (name, &c) in ["XOR", "MUL", "SET", "DEREF", "JUMP", "BLAKE3", "PACK64X2"] + .iter() + .zip(&stats.counts) + { let pow = if c == 0 { "0".to_string() } else { @@ -135,26 +141,21 @@ fn blake3_hash_chain() { println!(" {name:<6} instructions : {pow}"); } println!( - " committed witness size : 2^{}", - pretty_f64((stats.committed as f64).log2()) + " committed witness size : 2^{:.3}", + (stats.committed as f64).log2() ); let proof_bytes = bincode::serialized_size(&proof).expect("proof is serializable"); - println!( - " proof size : {} KiB", - pretty_f64(proof_bytes as f64 / 1024.0) - ); - println!( - " proving (incl. witness gen) : {} s", - pretty_f64(t_prove.as_secs_f64()) - ); - println!(" verifying : {} s", pretty_f64(t_verify.as_secs_f64())); + println!(" proof size : {:.1} KiB", proof_bytes as f64 / 1024.0); + println!(" proving (incl. witness gen) : {t_prove:?}"); + println!(" verifying : {t_verify:?}"); + let hashes_per_second = (n as f64 / t_prove.as_secs_f64()).round() as u64; println!( " throughput : {} hashes/s", - pretty_f64(n as f64 / t_prove.as_secs_f64()) + pretty_integer(hashes_per_second) ); // A wrong public input must be rejected. let mut bad = pi; - bad[0] += F128::ONE; + bad[0] += F192::ONE; assert!(verify(&program, &bad, &proof).is_err()); } diff --git a/crates/xmss/src/hash.rs b/crates/xmss/src/hash.rs index 27596805..77988d29 100644 --- a/crates/xmss/src/hash.rs +++ b/crates/xmss/src/hash.rs @@ -43,13 +43,7 @@ pub fn make_tweak(tweak_type: u8, sub_position: u32, index: u32) -> Tweak { /// Standard BLAKE3 of `tweak | pp | payload`. This is one compression for /// chain steps (48 bytes total) and Merkle nodes (64 bytes total). -pub fn tweak_hash( - pp: &PublicParam, - tweak_type: u8, - sub_position: u32, - index: u32, - payload: &[u8], -) -> Digest { +pub fn tweak_hash(pp: &PublicParam, tweak_type: u8, sub_position: u32, index: u32, payload: &[u8]) -> Digest { let mut hasher = blake3::Hasher::new(); hasher.update(&make_tweak(tweak_type, sub_position, index)); hasher.update(pp); @@ -58,13 +52,7 @@ pub fn tweak_hash( } /// Standard BLAKE3 of the exact-length `tweak | pp | data` byte string. -pub fn tweak_hash_many( - pp: &PublicParam, - tweak_type: u8, - sub_position: u32, - index: u32, - data: &[u8], -) -> Digest { +pub fn tweak_hash_many(pp: &PublicParam, tweak_type: u8, sub_position: u32, index: u32, data: &[u8]) -> Digest { let mut hasher = blake3::Hasher::new(); hasher.update(&make_tweak(tweak_type, sub_position, index)); hasher.update(pp); diff --git a/crates/xmss/src/lib.rs b/crates/xmss/src/lib.rs index 7e43421a..80256350 100644 --- a/crates/xmss/src/lib.rs +++ b/crates/xmss/src/lib.rs @@ -1,7 +1,8 @@ +// CREDIT: https://github.com/leanEthereum/leanVM (XMSS construction). //! XMSS over BLAKE3 (inspired by leanVM's `xmss` crate, byte-oriented). //! //! Every hash is standard BLAKE3 of the exact byte string -//! `tweak | pp | payload`, truncated to n = 128 bits. See [`hash`] for the +//! `tweak | pp | payload`, truncated to n = 128 bits. See the `hash` module for the //! constructions and per-call compression counts. #![cfg_attr(not(test), warn(unused_crate_dependencies))] @@ -60,10 +61,7 @@ pub mod array_serialization { use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::marker::PhantomData; - pub fn serialize( - data: &[T; N], - ser: S, - ) -> Result { + pub fn serialize(data: &[T; N], ser: S) -> Result { let mut tup = ser.serialize_tuple(N)?; for elem in data { tup.serialize_element(elem)?; diff --git a/crates/xmss/src/wots.rs b/crates/xmss/src/wots.rs index 8a6be26b..5663c641 100644 --- a/crates/xmss/src/wots.rs +++ b/crates/xmss/src/wots.rs @@ -87,13 +87,7 @@ impl WotsPublicKey { /// One chain step (1 compression). The position `chain_index * CHAIN_LENGTH + /// step` identifies the edge from chain value `step` to `step + 1`. -pub fn chain_step( - public_param: &PublicParam, - slot: u32, - chain_index: usize, - step: usize, - x: &Digest, -) -> Digest { +pub fn chain_step(public_param: &PublicParam, slot: u32, chain_index: usize, step: usize, x: &Digest) -> Digest { let position = (chain_index * CHAIN_LENGTH + step) as u32; tweak_hash(public_param, TWEAK_TYPE_CHAIN, position, slot, x) } @@ -107,7 +101,9 @@ pub fn iterate_hash( chain_index: usize, start_step: usize, ) -> Digest { - (0..n).fold(*a, |acc, j| chain_step(public_param, slot, chain_index, start_step + j, &acc)) + (0..n).fold(*a, |acc, j| { + chain_step(public_param, slot, chain_index, start_step + j, &acc) + }) } pub fn find_randomness_for_wots_encoding( @@ -128,14 +124,17 @@ pub fn find_randomness_for_wots_encoding( /// The target-sum encoding. `D = MD(msg | randomness | zeros)` under the /// encoding tweak, truncated to 16 bytes: 2 standard BLAKE3 compressions over -/// the 96-byte exact input. `D`'s 128 -/// bits, split little-endian into 42 chunks -/// of 3 bits, are the encoding; valid iff the 2 leftover top bits (126, 127) -/// are zero AND the chunks sum to [`TARGET_SUM`]. Grinding the top bits to -/// zero makes `D = sum(e_i * 2^{3i})` exactly, so the digest decomposes into -/// the chunks with no slack term — in-circuit this is checked over GF(2^128) -/// by XORing telescoping per-step constants along the chain walks (see the -/// `encoding_check_telescopes` test). +/// the 96-byte exact input. `D`'s two +/// little-endian 64-bit words each hold 21 chunks of 3 bits (the VM's word +/// width budgets the monomial encoding at 64 bits per word: `g^k = x^k` only +/// for `k < 64`): digit `i < 21` sits at bits `3i` of word 0, digit `i >= 21` +/// at bits `3(i-21)` of word 1. The encoding is valid iff the leftover top +/// bit of EACH word (bits 63 and 127) is zero AND the chunks sum to +/// [`TARGET_SUM`]. Grinding the top bits to zero makes each digest word +/// exactly `sum(e_i * 2^{3i})` of its 21 digits, so both words decompose into +/// the chunks with no slack term — in-circuit this is checked over GF(2^64) +/// per word by accumulating the dispatched digit literals against `8^i` +/// monomial weights (see `tests/xmss_aggregate.py`). pub fn wots_encode( message: &Message, slot: u32, @@ -147,13 +146,14 @@ pub fn wots_encode( data[MESSAGE_LEN..][..RANDOMNESS_LEN].copy_from_slice(randomness); let digest = tweak_hash_many(public_param, TWEAK_TYPE_ENCODING, 0, slot, &data); - if digest[DIGEST_LEN - 1] >> 6 != 0 { - return None; // the 2 leftover bits must be zero + if digest[7] >> 7 != 0 || digest[DIGEST_LEN - 1] >> 7 != 0 { + return None; // the leftover top bit of each 64-bit word must be zero } let bit = |j: usize| (digest[j / 8] >> (j % 8)) & 1; + let pos = |i: usize| if i < V / 2 { W * i } else { 64 + W * (i - V / 2) }; let mut encoding = [0u8; V]; for (i, e) in encoding.iter_mut().enumerate() { - *e = (0..W).fold(0, |acc, k| acc | (bit(W * i + k) << k)); + *e = (0..W).fold(0, |acc, k| acc | (bit(pos(i) + k) << k)); } (encoding.iter().map(|&x| x as usize).sum::() == TARGET_SUM).then_some(encoding) } diff --git a/crates/xmss/src/xmss.rs b/crates/xmss/src/xmss.rs index b1bdbef3..450e60bc 100644 --- a/crates/xmss/src/xmss.rs +++ b/crates/xmss/src/xmss.rs @@ -66,12 +66,13 @@ fn prf(seed: &[u8; 32], domain: u32, a: u64, b: u64) -> Digest { msg[..4].copy_from_slice(&domain.to_le_bytes()); msg[4..12].copy_from_slice(&a.to_le_bytes()); msg[12..20].copy_from_slice(&b.to_le_bytes()); - blake3::keyed_hash(seed, &msg).as_bytes()[..DIGEST_LEN].try_into().unwrap() + blake3::keyed_hash(seed, &msg).as_bytes()[..DIGEST_LEN] + .try_into() + .unwrap() } fn gen_wots_secret_key(seed: &[u8; 32], slot: u32, public_param: &PublicParam) -> WotsSecretKey { - let pre_images = - std::array::from_fn(|i| prf(seed, PRF_DOMAINSEP_WOTS_SECRET_KEY, slot as u64, i as u64)); + let pre_images = std::array::from_fn(|i| prf(seed, PRF_DOMAINSEP_WOTS_SECRET_KEY, slot as u64, i as u64)); WotsSecretKey::new(pre_images, public_param, slot) } @@ -85,13 +86,7 @@ fn gen_random_node(seed: &[u8; 32], level: usize, index: u64) -> Digest { } /// Merkle parent at `level` (1 compression: both children fill one block). -pub fn merkle_node( - public_param: &PublicParam, - level: usize, - index: u64, - left: &Digest, - right: &Digest, -) -> Digest { +pub fn merkle_node(public_param: &PublicParam, level: usize, index: u64, left: &Digest, right: &Digest) -> Digest { let mut data = [0u8; 2 * DIGEST_LEN]; data[..DIGEST_LEN].copy_from_slice(left); data[DIGEST_LEN..].copy_from_slice(right); @@ -235,8 +230,7 @@ pub fn xmss_sign( if slot < secret_key.slot_start || slot > secret_key.slot_end { return Err(XmssSignatureError::SlotOutOfRange); } - let (randomness, ..) = - find_randomness_for_wots_encoding(message, slot, &secret_key.public_param, rng); + let (randomness, ..) = find_randomness_for_wots_encoding(message, slot, &secret_key.public_param, rng); let wots_secret_key = gen_wots_secret_key(&secret_key.seed, slot, &secret_key.public_param); let wots_signature = wots_secret_key .sign_with_randomness(message, slot, &secret_key.public_param, randomness) @@ -285,7 +279,12 @@ impl XmssSecretKey { /// subtree, or `gen_random_node`. fn merkle_sibling(&self, level: usize, neighbour_index: u64, sub: &BottomSubtree) -> Digest { let (lo, hi, level_base, layers) = if level >= self.split_level { - (self.slot_start as u64, self.slot_end as u64, self.split_level, &self.top) + ( + self.slot_start as u64, + self.slot_end as u64, + self.split_level, + &self.top, + ) } else { let (lo, hi) = subtree_bounds( self.slot_start as u64, @@ -324,7 +323,11 @@ pub fn xmss_verify( for (level, neighbour) in signature.merkle_proof.iter().enumerate() { let is_left = ((slot as u64 >> level) & 1) == 0; let parent_index = (slot as u64) >> (level + 1); - let (left, right) = if is_left { (current, *neighbour) } else { (*neighbour, current) }; + let (left, right) = if is_left { + (current, *neighbour) + } else { + (*neighbour, current) + }; current = merkle_node(&pub_key.public_param, level + 1, parent_index, &left, &right); } if current == pub_key.merkle_root { diff --git a/misc/doc.tex b/misc/doc.tex index 814c19e3..9d53efaa 100644 --- a/misc/doc.tex +++ b/misc/doc.tex @@ -2,21 +2,16 @@ \usepackage[margin=1in]{geometry} \usepackage{amsmath, amssymb} +\usepackage{tabularx} \usepackage{xcolor} \usepackage[colorlinks=true, linkcolor=blue!50!black, citecolor=blue!50!black, urlcolor=blue!50!black]{hyperref} \newcommand{\F}{\mathbb{F}} +\newcommand{\K}{K} % F_{2^64}: addresses, counters, committed lanes +\newcommand{\E}{E} % F_{2^192} = K[y]/(y^3+y+1): words and challenges +\newcommand{\pkd}{\mathrm{pkd}} \newcommand{\eq}{\operatorname{eq}} \newcommand{\cube}[1]{\{0,1\}^{#1}} -\newcommand{\cin}{\mathit{cin}} -\newcommand{\oA}{\widehat A} -\newcommand{\oB}{\widehat B} -\newcommand{\oC}{\widehat C} -\newcommand{\ocout}{\widehat{c}_{\mathrm{out}}} -\newcommand{\ocin}{\widehat{c}_{\mathrm{in}}} -\newcommand{\oQ}{\widehat Q} -\newcommand{\II}{\widehat I} -\newcommand{\D}{D} % VM macros \newcommand{\gen}{\mathsf{g}} @@ -33,7 +28,6 @@ \newcommand{\dsep}[1]{\mathsf{#1}} \newcommand{\opc}[1]{\mathsf{op}_{\mathsf{#1}}} \newcommand{\srcsel}[1]{\mathsf{src}(#1)} -\newcommand{\todo}[1]{\textcolor{red}{\textbf{[TODO:} #1\textbf{]}}} \title{\bf leanVM-b} \author{} @@ -48,11 +42,18 @@ \section{VM specification}\label{sec:vm} -\paragraph{Field.} The machine works over the binary field $\F_{2^{128}}$ in the GHASH representation, $\F_2[x]/(x^{128}+x^7+x^2+x+1)$ (the AES-GCM field). \emph{Every} machine value (a memory word, an address, a register, or a counter) is an element of this field, and the only arithmetic is the field's own: addition is the bitwise \textsc{xor} of two $128$-bit values, and multiplication is the GHASH product. +\paragraph{Field.} +\[ + \K=\F_{2^{64}}=\F_2[x]/(x^{64}+x^4+x^3+x+1), + \qquad + \E=\K[y]/(y^3+y+1), + \qquad |\E|=2^{192}. +\] +A \emph{memory word}, including a cell value, immediate, or hash value, is an element $v=v_0+v_1y+v_2y^2\in\E$ with $v_0,v_1,v_2\in\K$. Each address, program counter $\pc$, frame pointer $\fp$, and read counter is in $\K$. Addition is bitwise \textsc{xor}; multiplication is in $\K$ or $\E$. Protocol randomness is sampled from $\E$ (\S\ref{sec:prelim}). Each physical committed column (\S\ref{sec:rs}) is $\K$-valued, so a data word occupies three columns. -\paragraph{Addresses as powers of a generator.} Fix a generator $\gen$ of the multiplicative group $\F_{2^{128}}^{\times}$, of order $2^{128}-1$. Memory addresses and bytecode addresses (the program counter) are powers of $\gen$: the $i$-th address is the field element $\gen^{\,i}$. Its successor is one field multiplication: $\gen^{\,i+1}\;=\;\gen\cdot\gen^{\,i}$. +\paragraph{Addresses as powers of a generator.} Fix a generator $\gen$ of the multiplicative group $\K^{\times}$, of order $2^{64}-1$. Memory addresses and bytecode addresses (the program counter) are powers of $\gen$: the $i$-th address is the field element $\gen^{\,i}$. Its successor is one field multiplication: $\gen^{\,i+1}\;=\;\gen\cdot\gen^{\,i}$. -\paragraph{Memory.} Memory is read-only (write-once): an array of $2^{h}$ cells, each holding one field element, addressed by the $2^{h}$ powers $\gen^{0},\gen^{1},\dots,\gen^{2^{h}-1}$. The size exponent $h$ is fixed per execution, with $16\le h\le 32$. Each cell is set once; every access thereafter reads it. +\paragraph{Memory.} Memory is immutable from the program's point of view: an array of $2^{h}$ cells, each holding one $192$-bit word (an $\E$-element), addressed by the $2^{h}$ powers $\gen^{0},\gen^{1},\dots,\gen^{2^{h}-1}$ (the addresses are $\K$-elements). The size exponent $h$ is fixed per execution, with $16\le h\le 32$. Witness generation may initialize an unset cell once; writing an initialized cell asserts that the value is unchanged. \paragraph{Program.} The program is a fixed, public sequence of instructions, addressed like memory by powers of $\gen$: the program counter $\gen^{\,i}$ selects the $i$-th instruction. An instruction is an opcode together with its operands, described below. @@ -64,27 +65,34 @@ \section{VM specification}\label{sec:vm} \] Since addresses are exponents of $\gen$, the frame is the contiguous block $\gen^{\mathrm{base}},\gen^{\mathrm{base}+1},\dots$, and adding the offset $j$ to the base exponent is, in the field, a single multiplication $\fp\cdot o$. -\paragraph{Instruction set.} The machine implements six instructions. Below, $o,o_A,\dots$ are fp-relative references and $k$ an immediate operand. +\paragraph{Instruction set.} The machine implements seven instructions. Below, $o,o_A,\dots$ are fp-relative references and $k$ an immediate operand. \begin{center} -\begin{tabular}{lll} +\begin{tabularx}{\textwidth}{@{}llX@{}} \hline Instruction & Operands & Semantics\\ \hline -\texttt{XOR} & $[o_A,o_B,o_C]$ & $\loc{o_C}=\loc{o_A}+\loc{o_B}$ \quad(field addition, \textsc{xor})\\ -\texttt{MUL\_NATIVE} & $[o_A,o_B,o_C]$ & $\loc{o_C}=\loc{o_A}\cdot\loc{o_B}$ \quad(field product in $\F_{2^{128}}$)\\ -\texttt{SET\_CONSTANT} & $[o,k]$ & $\loc{o}=k$\\ +\texttt{XOR} & $[o_A,o_B,o_C]$ & $\loc{o_C}=\loc{o_A}+\loc{o_B}$ \quad($192$-bit field addition, \textsc{xor})\\ +\texttt{MUL\_NATIVE} & $[o_A,o_B,o_C]$ & $\loc{o_C}=\loc{o_A}\cdot\loc{o_B}$ in $\E$\\ +\texttt{SET\_CONSTANT} & $[o,k]$ & $\loc{o}=k$ \quad($k$ a $192$-bit immediate)\\ \texttt{DEREF} & $[\alpha,\beta,\gamma;\,s]$ & $\mem[\loc{\alpha}\cdot\beta]=\srcsel{s}$\\ \texttt{JUMP} & $[o_c,o_d,o_f]$ & conditional jump (below)\\ -\texttt{BLAKE3} & $[o_0,o_1,o_2,o_3,o_V,o_C,\mu]$ & $\bigl(\loc{o_C},\loc{\gen\cdot o_C}\bigr)=\mathrm{Compress}\bigl(\loc{o_V},\loc{\gen\cdot o_V};\loc{o_0},\ldots,\loc{o_3};\mu\bigr)$ (below)\\ +\texttt{PACK64X2} & $[o_A,o_B,o_C]$ & $\loc{o_C}=a_0+b_0y$ for $\loc{o_A}=a_0,\loc{o_B}=b_0\in\K$\\ +\texttt{BLAKE3} & $[o_0,o_1,o_2,o_3,o_V,o_C,\mu]$ & standard BLAKE3 compression (below)\\ \hline -\end{tabular} +\end{tabularx} \end{center} -\texttt{BLAKE3} is the standard BLAKE3 compression, consuming a $64$-byte message block and a $32$-byte chaining value and producing $32$ bytes. The four $128$-bit message words are addressed \emph{independently} by references $o_0,\dots,o_3$ (word $i$ at $\loc{o_i}=\fp\cdot o_i$), while $o_V$ names two \emph{consecutive} chaining-value words $\loc{o_V},\loc{\gen\cdot o_V}$. The $256$-bit result is written to the consecutive cells $\loc{o_C},\loc{\gen\cdot o_C}$. The immediate $\mu\in\F_{2^{128}}$ packs the standard compression metadata in little-endian order, +\texttt{PACK64X2} groups the low limbs of two physical $\E$-valued cells into one canonical $128$-bit cell. Its memory interactions read the sources as $(a_0,0,0)$ and $(b_0,0,0)$ and the destination as $(a_0,b_0,0)$. Bus balance therefore proves that both source words lie in $\K$; this is not merely a witness-generation check. The instruction is useful when a program receives serialized $\K$-limbs through an $\E$-valued interface. + +\texttt{BLAKE3} is the standard BLAKE3 compression, consuming a $64$-byte message block and a $32$-byte chaining value and producing $32$ bytes. Each $128$-bit chunk occupies one full $\E$ memory cell through the canonical embedding $a_0+a_1y\mapsto a_0+a_1y+0y^2$; using a cell as a BLAKE3 operand therefore constrains its top limb to zero. The four message chunks are addressed \emph{independently} by references $o_0,\dots,o_3$ (chunk $i$ at $\loc{o_i}=\fp\cdot o_i$), while $o_V$ names two \emph{consecutive} chaining-value cells $\loc{o_V},\loc{\gen\cdot o_V}$. The $256$-bit result is written to the consecutive cells $\loc{o_C},\loc{\gen\cdot o_C}$, so each row accesses eight cells. The $128$-bit immediate $\mu$ (the two low $\K$-lanes of a $192$-bit word, top lane zero) packs the standard compression metadata in little-endian order, \[ \mu=\mathit{counter}_{64}\;\Vert\;\mathit{block\_len}_{32}\;\Vert\;\mathit{flags}_{32}. \] -For a one-block $64$-byte hash, the chaining value is the standard IV and $\mu=(0,64,\textsc{chunk-start}|\textsc{chunk-end}|\textsc{root})$. Within one $1024$-byte chunk, a longer standard BLAKE3 hash passes each compression result back through $o_V$ and chooses the length and flags prescribed by BLAKE3. Still longer inputs additionally use BLAKE3's chunk counters and binary tree of parent compressions. The BLAKE3 table (\S\ref{sec:tab-blake3}) wires its memory and bytecode operands into a flock~\cite{flock} R1CS proof. +For a one-block $64$-byte hash, the chaining value is the standard IV and +\[ + \mu=(0,64,\textsc{chunk-start}\mathbin{|}\textsc{chunk-end}\mathbin{|}\textsc{root}). +\] +Within one $1024$-byte chunk, a longer standard BLAKE3 hash passes each compression result back through $o_V$ and chooses the length and flags prescribed by BLAKE3. Still longer inputs additionally use BLAKE3's chunk counters and binary tree of parent compressions. The BLAKE3 table (\S\ref{sec:tab-blake3}) wires its memory and bytecode operands into a flock~\cite{flock} R1CS proof. \texttt{DEREF} stores, at the dereferenced address $\loc{\alpha}\cdot\beta$, a value chosen by a \emph{store mode} $s\in\{\texttt{deref\_cell}[\gamma],\texttt{deref\_pc},\texttt{deref\_fp}\}$: \[ @@ -97,45 +105,42 @@ \section{VM specification}\label{sec:vm} \] The \texttt{deref\_pc} and \texttt{deref\_fp} modes let a caller save its return address and frame so a callee can later return and restore them, the basis for function calls. The skip of two is the call's return target: place the \texttt{DEREF} that saves $\gen^{2}\cdot\pc$ immediately before the \texttt{JUMP} that transfers control, and $\gen^{2}\cdot\pc$ names the instruction just after that \texttt{JUMP}. -\texttt{JUMP} reads a condition $c=\loc{o_c}$, an arbitrary field element, and branches on whether it is zero. When $c\neq0$ it transfers control, $\pc\gets\loc{o_d}$ and $\fp\gets\loc{o_f}$; when $c=0$ it falls through, $\pc\gets\gen\cdot\pc$ and $\fp\gets\fp$. +\texttt{JUMP} reads a condition $c=\loc{o_c}$, an arbitrary $192$-bit word, and branches on whether it is zero. When $c\neq0$ it transfers control, $\pc\gets\loc{o_d}$ and $\fp\gets\loc{o_f}$; when $c=0$ it falls through, $\pc\gets\gen\cdot\pc$ and $\fp\gets\fp$. +\paragraph{Words used as addresses.} A memory word is $192$-bit, while $\pc$, $\fp$, and addresses are in $\K$. Three reads produce addresses or registers: the \texttt{DEREF} pointer $\loc{\alpha}$, the \texttt{JUMP} destination $\loc{o_d}$, and the \texttt{JUMP} frame $\loc{o_f}$. Each must have both nonconstant tower limbs zero. The address and selection constraints (\S\ref{sec:tab-deref}, \S\ref{sec:tab-jump}) equate these words, possibly multiplied by a $\K$-operand, to a $\K$ column. -\iffalse % Deferred future instructions (u32, BLAKE3): commented out for now. -\noindent The remaining instructions are listed to fix the eventual ISA but are deferred to a later version. -\begin{center} -\begin{tabular}{lll} -\hline -Instruction (later) & Operands & Semantics\\ -\hline -\texttt{u32\_ADD} & $[o_A,o_B,o_C]$ & integer add mod $2^{32}$\\ -\texttt{IS\_NONZERO} & $[o_x,o_b]$ & $\loc{o_b}=[\,\loc{o_x}\neq 0\,]$\\ -\texttt{u32\_MUL} & $[o_A,o_B,o_C,o_D]$ & integer multiply, double word: $a\cdot b=c+2^{32}d$\\ -\hline -\end{tabular} -\end{center} - -\noindent These operate on $32$-bit integers (\texttt{u32}) rather than native field elements, and so require integer addition with carries, which the field arithmetic does not provide. Their full semantics are fixed later. \todo{specify the \texttt{u32\_MUL} semantics later.} -\fi % end deferred future instructions \section{Proving primitives}\label{sec:prelim} -The proving field is $\F_{2^{128}}$ throughout. +Every physical committed column is $\K$-valued (\S\ref{sec:vm}); every challenge, sumcheck, zerocheck, GKR value, and evaluation claim is $\E$-valued, where +\[ + \K=\F_2[x]/(x^{64}+x^4+x^3+x+1), + \qquad + \E=\K[y]/(y^3+y+1)\cong\F_{2^{192}}. +\] +Thus $|\E|=2^{192}$. The embedding and mixed product are +\[ + \K\hookrightarrow\E,\quad e\mapsto e+0\cdot y+0\cdot y^2, + \qquad + k(e_0+e_1y+e_2y^2)=ke_0+(ke_1)y+(ke_2)y^2. +\] +Committed columns and their truth tables are $\K$-valued; extensions are evaluated at $\E$-points; protocol randomness is sampled from $\E$. A $192$-bit \emph{value} column is represented by $V_0,V_1,V_2:\cube\ell\to\K$ with $V=V_0+V_1y+V_2y^2$. Address, register, and counter columns are single $\K$-columns. -\textbf{Multilinear extension (MLE).} A function $b:\cube{\ell}\to\F_{2^{128}}$, such as a committed column or a product-tree layer of \S\ref{sec:gkr}, has a unique multilinear extension $\widetilde b:\F_{2^{128}}^{\ell}\to\F_{2^{128}}$, the polynomial of degree at most $1$ in each variable that agrees with $b$ on the boolean hypercube. The equality polynomial $\eq(r,X)=\prod_i(1+r_i+X_i)$ is the multilinear that is $1$ at $X=r$ and $0$ at every other Boolean point. +\textbf{Multilinear extension (MLE).} A function $b:\cube{\ell}\to\K$ (a committed column) or $b:\cube{\ell}\to\E$ (a product-tree layer of \S\ref{sec:gkr}) has a unique multilinear extension $\widetilde b:\E^{\ell}\to\E$, the polynomial of degree at most $1$ in each variable that agrees with $b$ on the boolean hypercube. Define $\eq(r,X)=\prod_i(1+r_i+X_i)$. For Boolean $r$, this is $1$ at $X=r$ and $0$ at the other Boolean points. For arbitrary $r\in\E^\ell$, it gives the interpolation weights: $\widetilde b(r)=\sum_{x\in\cube\ell}b(x)\eq(r,x)$. -\textbf{Tables and columns.} A \emph{table} is a group of columns of the same size. Each column is \emph{committed} or \emph{virtual}: a virtual column is an $\F_{2^{128}}$-linear function of the committed columns of its table, carried with no commitment of its own. An $\F_{2^{128}}$-linear map commutes with the multilinear extension, $\widetilde{L(x)}(\zeta)=L\bigl(\widetilde x(\zeta)\bigr)$, so one opening of the committed columns evaluates the virtual one. +\textbf{Tables and columns.} A \emph{table} is a group of columns of the same size. Each column is \emph{committed} or \emph{virtual}: a virtual column is a $\K$-linear function of the committed columns of its table, carried with no commitment of its own. Such a map, extended $\E$-linearly, commutes with the multilinear extension, $\widetilde{L(x)}(\zeta)=L\bigl(\widetilde x(\zeta)\bigr)$, so one opening of the committed columns evaluates the virtual one. -\textbf{Sumcheck.} Sumcheck reduces a claim $\sum_{x\in\cube\ell}P(x)=s$ to a single evaluation $P(\rho)$ at a random $\rho$, over $\ell$ rounds; a false claim passes with probability at most $\deg P\cdot\ell/|\F_{2^{128}}|$. +\textbf{Sumcheck.} Sumcheck reduces a claim $\sum_{x\in\cube\ell}P(x)=s$ to a single evaluation $P(\rho)$ at a random $\rho\in\E^{\ell}$, over $\ell$ rounds; a false claim passes with probability at most $\deg P\cdot\ell/|\E|$. -\textbf{Zerocheck.} To prove $P$ vanishes on $\cube\ell$, the verifier samples $r$ and runs sumcheck on $\sum_{x}P(x)\,\eq(r,x)=0$. +\textbf{Zerocheck.} To prove $P$ vanishes on $\cube\ell$, the verifier samples $r\in\E^{\ell}$ and runs sumcheck on $\sum_{x}P(x)\,\eq(r,x)=0$. -\textbf{Inner-product PCS.} An \emph{inner-product commitment scheme} lets a prover commit to a multilinear $g:\cube{M}\to\F_{2^{128}}$ and later prove a weighted sum $\sum_{w\in\cube M} W(w)\,\widehat g(w)=c$ against any weight $W$ the verifier can evaluate. A point evaluation $\widehat g(\mathbf r)$ is the special case $W=\eq(\mathbf r,\cdot)$. WHIR and BaseFold/Ligerito are such schemes; we use one (\S\ref{sec:rs-dense}). +\textbf{Inner-product PCS.} An \emph{inner-product commitment scheme} lets a prover commit to a multilinear $g:\cube{M}\to\K$ and later prove a weighted sum $\sum_{w\in\cube M} W(w)\,\widehat g(w)=c$ against any weight $W:\cube{M}\to\E$ the verifier can evaluate: the commitment is over $\K$, the opening over $\E$. A point evaluation $\widehat g(\mathbf r)$ at $\mathbf r\in\E^{M}$ is the special case $W=\eq(\mathbf r,\cdot)$. WHIR and BaseFold/Ligerito are such schemes; we use one (\S\ref{sec:rs-dense}). \section{Committing the witness}\label{sec:rs} \subsection{Stacking}\label{sec:stacking} -No column is committed on its own. The columns are $\F_{2^{128}}$-valued multilinears over boolean hypercubes of various sizes, concatenated into one witness $q$. Order the $T$ columns $\widehat P_1,\dots,\widehat P_T$ by size, $\kappa_1\ge\dots\ge\kappa_T$ variables, and lay their truth tables end to end, +No column is committed on its own. The columns are $\K$-valued multilinears over boolean hypercubes of various sizes, concatenated into one witness $q$; one region of the stack is flock's packed Boolean witness $q_\pkd$, $64$ bits per $\K$-word (\S\ref{sec:tab-blake3}). Order the $T$ columns $\widehat P_1,\dots,\widehat P_T$ by size, $\kappa_1\ge\dots\ge\kappa_T$ variables, and lay their truth tables end to end, \[ q \;=\; P_1 \,\Vert\, P_2 \,\Vert\,\dots\,\Vert\, P_T \,\Vert\, 0, \] @@ -149,14 +154,14 @@ \subsection{Stacking}\label{sec:stacking} \] a weighted sum over $\widehat q$ with weight $W(w)=\eq((\zeta,\mathsf{sel}_i),w)$. -\paragraph{Batching.} Every claim the protocol raises against a column is such a weighted sum $\sum_w W_j(w)\,\widehat q(w)=c_j$. The verifier folds the $J$ of them with a random $\lambda$ into one weight $W_\lambda=\sum_j\lambda^j W_j$ and target $C_\lambda=\sum_j\lambda^j c_j$, and the inner-product PCS (\S\ref{sec:rs-dense}) proves +\paragraph{Batching.} Every claim the protocol raises against a column is a weighted sum $\langle W_j,\widehat q\rangle=c_j$ with $\E$-valued weight and target. After the claimed values are bound, the verifier samples independent $\lambda_j\in\E$ and defines $W_\lambda=\sum_j\lambda_jW_j$ and $C_\lambda=\sum_j\lambda_jc_j$. The inner-product PCS (\S\ref{sec:rs-dense}) proves \[ \sum_{w\in\cube{M}} W_\lambda(w)\,\widehat q(w)=C_\lambda \] \subsection{Inner-product PCS}\label{sec:rs-dense} -It is Ligerito~\cite{cryptoeprint:2025/1187} over $\F_{2^{128}}$, in the unique-decoding regime, at rate $1/2$ and $120$-bit round-by-round soundness. +It is Ligerito~\cite{cryptoeprint:2025/1187}, instantiated by default at rate $1/2$ in the Johnson list-decoding regime with a per-level optimized slack $\eta$ and $128$-bit round-by-round soundness. A proof may instead select rate $1/2^r$ for $1\le r\le4$; the verifier checks and transcript-binds this choice. The level-zero opening claim is sampled after its commitment and therefore also binds one element of the nearby-codeword list; no separate out-of-domain evaluation is needed there. Each recursive commitment is bound by one fresh out-of-domain evaluation. Since all challenges lie in $\E=\F_{2^{192}}$, the proximity-gap and binding errors clear $128$ bits without proof-of-work grinding; the query phase grinds $17$ bits per level, so queries only close the remaining $111$. The witness is Reed--Solomon encoded over $\K$; level-zero symbols are $8$ bytes and $2^6$ interleaved symbols form each $512$-byte Merkle leaf. Openings are over $\E$: the level-zero fold pairs $\K$-symbols with $\E$-challenges, and all deeper levels are $\E$-valued. \section{M3 model}\label{sec:m3} @@ -164,7 +169,7 @@ \section{M3 model}\label{sec:m3} \paragraph{Constraints.} A constraint is a polynomial in the columns that must vanish on every row, including the padding rows. -\paragraph{The bus.} All interactions share a single \emph{bus}, a channel carrying tuples of $m$ field elements. A table flushes by wiring its columns into a tuple and either \emph{pushing} it onto the bus or \emph{pulling} it off, once per row. A few fixed \emph{boundary} tuples, belonging to no table, are pushed or pulled directly. The width $m$ is that of the widest tuple, eleven (the bytecode tuple: separator, $\pc$, count, opcode, and seven operand/immediate slots); shorter tuples are zero-padded. +\paragraph{The bus.} All interactions share a single \emph{bus}, a channel carrying tuples of $m$ field elements. A table flushes by wiring its columns into a tuple and either \emph{pushing} it onto the bus or \emph{pulling} it off, once per row. A few fixed \emph{boundary} tuples, belonging to no table, are pushed or pulled directly. The width $m$ is that of the widest tuple, twelve (the bytecode tuple: separator, $\pc$, count, opcode, and eight operand/immediate slots); shorter tuples are zero-padded. \paragraph{Domain separation.} The first coordinate of every tuple is a \emph{domain separator}, a fixed constant naming its interaction: $\dsep{ST}$ for state, $\dsep{MEM}$ for memory, $\dsep{BC}$ for bytecode. Since everything shares the one bus, the separators are what keep the three kinds of tuple from cancelling against one another: a memory tuple can never meet a bytecode tuple. The interactions below are three uses of this one channel, told apart by this coordinate. @@ -178,13 +183,13 @@ \subsection{Proving the local constraints}\label{sec:air} Table $T_j$ has $2^{\tau_j}$ rows and constraints $C_{j,1},\dots,C_{j,s_j}$: polynomials of degree at most $d$ in a row's column values, required to vanish on every row ($d=2$ throughout). Write $\sigma=\sum_j s_j$ for the total number of constraints and $n=\max_j\tau_j$ for the tallest table's height. -\paragraph{One challenge and one point.} The verifier samples $\eta\in\F_{2^{128}}$ and fixes a point $r\in\F_{2^{128}}^{n}$ (in the assembled protocol $r$ is not freshly sampled but taken from the bus, see \emph{Nonzero claims} below and \S\ref{sec:leafstack}). Table $T_j$ mixes its constraints with the powers of $\eta$ from $o_j=s_1+\dots+s_{j-1}$ on, and is tested at the length-$\tau_j$ prefix of $r$: +\paragraph{One challenge and one point.} The verifier samples $\eta\in\E$ and fixes a point $r\in\E^{n}$ (in the assembled protocol $r$ is not freshly sampled but taken from the bus, see \emph{Nonzero claims} below and \S\ref{sec:leafstack}). Table $T_j$ mixes its constraints with the powers of $\eta$ from $o_j=s_1+\dots+s_{j-1}$ on, and is tested at the length-$\tau_j$ prefix of $r$: \[ C_j\;=\;\sum_{i=1}^{s_j}\eta^{\,o_j+i-1}\,C_{j,i}, \qquad S_j\;=\;\widetilde{C_j}\bigl(r_{<\tau_j}\bigr)\;=\;\sum_{x\in\cube{\tau_j}}C_j(x)\,\eq(r_{<\tau_j},x). \] -The ranges of $\eta$-powers are \emph{disjoint}, so no table's violation can be cancelled by another's, and proving $\sum_j S_j=0$ proves every $C_{j,i}$ vanishes on every row of its own table, up to a Schwartz--Zippel error $(n+\sigma)/|\F_{2^{128}}|$ over $r$ and $\eta$. (The identities keep disjoint ranges throughout. \S\ref{sec:leafstack} appends three more entries per table that deliberately \emph{share} their powers, since only their total over tables has to be pinned; that lengthens the $\eta$-degree, and the error, by $3$.) +The ranges of $\eta$-powers are \emph{disjoint}, so no table's violation can be cancelled by another's, and proving $\sum_j S_j=0$ proves every $C_{j,i}$ vanishes on every row of its own table, up to Schwartz--Zippel error $(n+\sigma)/|\E|$ over $r$ and $\eta$. (The identities keep disjoint ranges throughout. \S\ref{sec:leafstack} appends three more entries per table that deliberately \emph{share} their powers, since only their total over tables has to be pinned; that lengthens the $\eta$-degree, and the error, by $3$.) \paragraph{One sumcheck.} The $S_j$ are sumcheck claims of different lengths. Lift each onto the common cube $\cube n$ by attaching the indicator of the all-ones assignment on the $n-\tau_j$ variables $T_j$ does not use: \[ @@ -209,20 +214,24 @@ \subsection{Balancing the bus: grand product}\label{sec:gp} The bus balances when its pushed and pulled tuples form the same multiset (\S\ref{sec:m3}). Two random challenges, $\alpha$ then $\gamma$, reduce this to a single product. -\emph{Fingerprint.} The verifier samples $\alpha\in\F_{2^{128}}$ and maps each width-$m$ tuple $\sigma=(\sigma_0,\dots,\sigma_{m-1})$ to a single field element, +\emph{Fingerprint.} The verifier samples $\alpha\in\E$ and maps each width-$m$ tuple $\sigma=(\sigma_0,\dots,\sigma_{m-1})$ to a single $\E$-element, \[ \pi_\alpha(\sigma)\;=\;\sum_{i=0}^{m-1}\alpha^{i}\,\sigma_i , \] -each coordinate $\sigma_i$ a committed column value, a virtual column value, or a public constant. +each coordinate $\sigma_i$ a committed column value, a virtual column value, or a public constant, all $\K$-elements, so each summand is a mixed $\K\times\E$ product. -\emph{Product check.} The verifier samples $\gamma\in\F_{2^{128}}$ and the prover proves +\emph{Product check.} The verifier samples $\gamma\in\E$ and the prover proves \begin{equation} \prod_{\sigma\ \text{pushed}}\bigl(\gamma-\pi_\alpha(\sigma)\bigr) \;=\; \prod_{\sigma\ \text{pulled}}\bigl(\gamma-\pi_\alpha(\sigma)\bigr). \label{eq:gp} \end{equation} -\emph{Completeness and soundness.} If the pushed and pulled multisets agree, the two products in \eqref{eq:gp} have the same factors and are equal for every $\alpha,\gamma$; completeness is immediate. For soundness, read each side as the polynomial $\prod_\sigma\bigl(X-\pi_\alpha(\sigma)\bigr)$ evaluated at $X=\gamma$: its roots are that side's fingerprints, which over the formal $\alpha$ are the tuples themselves, so the polynomial determines that side's multiset. The two sides therefore agree as polynomials in $(\alpha,\gamma)$ exactly when the multisets do. When the multisets differ the sides differ as polynomials, of total degree at most $K(m-1)$ with $K$ the number of tuples on a side, so by Schwartz--Zippel the prover passes at random $\alpha,\gamma$ with probability at most $K(m-1)/|\F_{2^{128}}|$. +\emph{Completeness and soundness.} If the pushed and pulled multisets agree, the two products in \eqref{eq:gp} have the same factors and are equal for every $\alpha,\gamma$; completeness is immediate. For soundness, read each side as the polynomial $\prod_\sigma\bigl(X-\pi_\alpha(\sigma)\bigr)$ evaluated at $X=\gamma$: its roots are that side's fingerprints, which over the formal $\alpha$ are the tuples themselves, so the polynomial determines that side's multiset. The two sides therefore agree as polynomials in $(\alpha,\gamma)$ exactly when the multisets do. When they differ, Schwartz--Zippel applies jointly to the tuple-fingerprinting challenge $\alpha$, the product challenge $\gamma$, the padding correction, and the GKR reductions. If each side has at most $2^\mu$ factors and tuples have width $m$, a conservative total degree bound is +\[ + 2m\,2^\mu+8(\mu+1)^2. +\] +The verifier checks that this bound over $|\E|=2^{192}$ leaves at least $128$ bits of soundness. The count product is verified by the same GKR argument, and its root is checked directly for nonzero; it introduces no separate root-at-random test. No Fiat--Shamir grinding is needed for the grand-product phase. Each side's product is proven by a GKR pass over a product tree (\S\ref{sec:gkr}). @@ -231,7 +240,7 @@ \subsection{Balancing the bus: grand product}\label{sec:gp} \subsection{Proving the grand product (GKR)}\label{sec:gkr} -Take one side of \eqref{eq:gp}. Its tuples give leaves $v_k=\gamma-\pi_\alpha(\sigma_k)$, padded with $1$s to a power of two, $2^\mu$ in all. Their product $P=\prod_kv_k$ is the root of a binary product tree: layer $0$ is the leaves, layer $i$ has $2^{\mu-i}$ nodes, and layer $\mu$ is $P$. Let $\widetilde V_i$ denote the MLE of layer $i$. +Take one side of \eqref{eq:gp}. Its tuples give leaves $v_k=\gamma-\pi_\alpha(\sigma_k)$, padded with $1$s to a power of two, $2^\mu$ in all. Their product $P=\prod_kv_k$ is the root of a binary product tree: layer $0$ is the leaves, layer $i$ has $2^{\mu-i}$ nodes, and layer $\mu$ is $P$. Let $\widetilde V_i:\E^{\mu-i}\to\E$ denote the multilinear extension of layer $i$. GKR contracts two binary levels at a time. Four grandchildren satisfy \[ @@ -271,7 +280,7 @@ \subsection{Stacking the leaves}\label{sec:leafstack} \[ \widetilde V_0(\zeta)\;=\;\sum_b \eq(\mathsf{sel}_b,\zeta^{b}_{\mathrm{hi}})\Bigl(\gamma-\sum_{i=0}^{m-1}\alpha^{i}\,\widetilde c_{b,i}(\zeta^{b}_{\mathrm{lo}})\Bigr)\;+\;\Bigl(1-\sum_b\eq(\mathsf{sel}_b,\zeta^{b}_{\mathrm{hi}})\Bigr). \] -The verifier forms the selectors $\eq(\mathsf{sel}_b,\cdot)$ and the constant coordinates itself. Each remaining $\widetilde c_{b,i}(\zeta^{b}_{\mathrm{lo}})$ is settled by its kind: a committed column by a direct opening (\S\ref{sec:prelim}), a virtual column off the committed columns it images (so $\gen\cdot\pc$ adds nothing beyond the $\pc$ opening), the index column by \S\ref{sec:idxcol}. A committed column feeding several equal-size blocks is opened once, at their shared point $\zeta_{\mathrm{lo}}$. +The verifier forms the selectors $\eq(\mathsf{sel}_b,\cdot)$ and the constant coordinates itself. Each remaining $\widetilde c_{b,i}(\zeta^{b}_{\mathrm{lo}})$ is settled by its kind: a committed column by a direct opening (\S\ref{sec:prelim}), a virtual column off the committed columns it images (so $\gen\cdot\pc$ adds nothing beyond the $\pc$ opening), a \texttt{BLAKE3} value coordinate by a point evaluation of $q_\pkd$ at its slot (\S\ref{sec:tab-blake3}), the index column by \S\ref{sec:idxcol}. A committed column feeding several equal-size blocks is opened once, at their shared point $\zeta_{\mathrm{lo}}$. \paragraph{Blocks of a table.} Split that sum by owner. Three blocks per side belong to no table, the state boundary and the memory and bytecode seeds (finalizations on the pull side); they settle exactly as above, and the counts side has none of them. Every other block belongs to a table, and a table owns many: one per flush on push and on pull, one per read count on the counts, so \texttt{BLAKE3} owns ten, ten and nine. @@ -304,7 +313,7 @@ \section{Memory and bytecode lookups}\label{sec:omc} \subsection{Memory interaction}\label{sec:memchan} -The interaction carries $\tup{\dsep{MEM},\addr,\cnt,\val}$: the address and count are $\gen$-powers, the value a field element. The prover commits the memory $\mem$ ($2^{h}$ cells) and the counts ("in the exponent", i.e. powers of $\gen$). With $A[i]$ the number of reads of address $i$, the flushes are: +The interaction carries $\tup{\dsep{MEM},\addr,\cnt,\val}$: the address and count are $\gen$-powers ($\K$-elements), the value a $192$-bit word. The word rides \emph{three} tuple coordinates, its $\K$-limbs $(\val_0,\val_1,\val_2)$, matching the three-limb committed memory image $M_0,M_1,M_2$; all three are seeded and pulled, so balance forces the whole word to match (below I write $\val$ for the triple). The tuple width is unchanged: the widest tuple is still the bytecode one (\S\ref{sec:bytecode}). The prover commits the memory $\mem$ ($2^{h}$ cells, three $\K$-limb columns) and the counts ("in the exponent", i.e. powers of $\gen$). With $A[i]$ the number of reads of address $i$, the flushes are: \begin{itemize} \item \textbf{Initialize:} for each address $i$, \push\ $\tup{\dsep{MEM},\gen^{i},1,\mem[i]}$ (seed count $\gen^{0}=1$). \item \textbf{Read} (value $v$ at address $a$, count $\cnt=\gen^{t}$): \pull\ $\tup{\dsep{MEM},a,\cnt,v}$ and \push\ $\tup{\dsep{MEM},a,\gen\cdot\cnt,v}$. @@ -312,7 +321,11 @@ \subsection{Memory interaction}\label{sec:memchan} \end{itemize} \paragraph{Counts must be nonzero.} A read multiplies its count by $\gen$, pulling $\tup{\dsep{MEM},a,\cnt,v}$ and pushing $\tup{\dsep{MEM},a,\gen\cdot\cnt,v}$. If $\cnt=0$ these coincide (as $\gen\cdot0=0$) and cancel, so the read disappears from the bus and its $v$ is never checked against $\mem[a]$. Every count must therefore be nonzero, i.e.\ a power of $\gen$. A third grand product, over all count columns (memory and bytecode), enforces this: its root is nonzero iff every count is (\S\ref{sec:gkr}). -\paragraph{Counts must not wrap.} A count is the power $\gen^{t}$ after $t$ reads, so the total accesses must stay below $2^{128}-1$. This is automatic at any real scale. +\paragraph{Counts must not wrap.} A count is the power $\gen^{t}$ after $t$ reads, so the total number of reads must stay below $\operatorname{ord}(\gen)=2^{64}-1$. At $64$ bits this is not waved through as automatic. The protocol fixes public \emph{instance caps}, and the verifier checks the announced instance against them before running any reduction (\S\ref{sec:e2e-unrolled}): memory size $16\le h\le 32$ (\S\ref{sec:vm}), each table has at most $2^{32}$ rows, and bytecode length is at most $2^{32}$. A row makes at most $9$ reads (the \texttt{BLAKE3} row: eight memory, one bytecode), so the seven instruction tables perform +\[ + R\;\le\;9\cdot 7\cdot 2^{32}\;<\;2^{38} +\] +read flushes in total, far below $2^{64}-1$: the powers $\gen^{0},\gen^{1},\dots$ occurring in one address's history are pairwise distinct, and the counting arguments of this section are exact, with no probabilistic term. The caps cost no completeness ($2^{32}$ rows per table is far beyond any provable trace). \paragraph{What balance certifies.} Fix an address $i$. Its only non-read flushes are the seed, a push of count $\gen^{0}$ with value $\mem[i]$, and the finalize, a pull of the committed final count with value $\mem[i]$. @@ -322,15 +335,15 @@ \subsection{Memory interaction}\label{sec:memchan} \[ \{\,\cnt : \text{reads returning } v\,\}\;=\;\{\,\gen\cdot\cnt : \text{reads returning } v\,\}. \] -This multiset of counts is thus invariant under multiplication by $\gen$. But the counts are powers of $\gen$, and $\times\gen$ cycles through all $2^{128}-1$ of them, so an invariant multiset weights every power equally: its size is a multiple of $2^{128}-1$. By the bound it is smaller, hence empty. So no read returns $v$, and every read of $i$ returns $\mem[i]$. +This multiset of counts is thus invariant under multiplication by $\gen$. But the counts are powers of $\gen$, and $\times\gen$ cycles through all $2^{64}-1$ of them, so an invariant multiset weights every power equally: its size is a multiple of $2^{64}-1$. Under the instance caps its size is at most $R<2^{39}$, hence it is empty. So no read returns $v$, and every read of $i$ returns $\mem[i]$. \subsection{Bytecode interaction}\label{sec:bytecode} -Instruction fetch is the same as the memory construction (\S\ref{sec:memchan}) with the \emph{public} program for the committed memory and the program counter for the address. The tuple is $\tup{\dsep{BC},\pc,\cnt,\mathit{instr}}$, where $\mathit{instr}$ is the encoded instruction: an opcode and seven operand/immediate slots. Shorter instructions zero their unused high slots. +Instruction fetch is the same as the memory construction (\S\ref{sec:memchan}) with the \emph{public} program for the committed memory and the program counter for the address. The tuple is $\tup{\dsep{BC},\pc,\cnt,\mathit{instr}}$, where $\mathit{instr}$ is the encoded instruction: an opcode and eight operand/immediate slots. Shorter instructions zero their unused high slots. \subsection{The index column}\label{sec:idxcol} -The memory seed and finalize tuples (\S\ref{sec:memchan}) run over every address $\gen^{i}$, $i=0,\dots,2^{h}-1$. Settling the grand product leaves the verifier evaluating, at a random $\zeta\in\F_{2^{128}}^{h}$, the MLE of this \emph{index column} $\gen^{0},\gen^{1},\dots,\gen^{2^{h}-1}$. It is not committed: as $\gen$-powers the MLE factors, costing $O(h)$. +The memory seed and finalize tuples (\S\ref{sec:memchan}) run over every address $\gen^{i}$, $i=0,\dots,2^{h}-1$. Settling the grand product leaves the verifier evaluating, at a random $\zeta\in\E^{h}$, the MLE of this \emph{index column} $\gen^{0},\gen^{1},\dots,\gen^{2^{h}-1}$. It is not committed: as $\gen$-powers the MLE factors, costing $O(h)$. Place address $\gen^{i}$ at the cube point $w\in\cube h$ of $i$'s bits, so $i=\sum_k 2^{k}w_k$ and \[ @@ -344,14 +357,14 @@ \subsection{The index column}\label{sec:idxcol} \section{The instruction tables}\label{sec:tables} -Each of the six instructions has its own table. +Each instruction has its own table. \subsection{\texttt{XOR}}\label{sec:tab-xor} -Asserts $\loc{o_C}=\loc{o_A}+\loc{o_B}$, the field sum. +Asserts $\loc{o_C}=\loc{o_A}+\loc{o_B}$, the $192$-bit field sum. \begin{itemize} -\item \textbf{Columns:} $\pc,\fp$; operands $o_A,o_B,o_C$; addresses $a_A,a_B,a_C$; values $v_A,v_B,v_C$; memory counts $r_A,r_B,r_C$; bytecode count $r_{\mathrm{bc}}$. -\item \textbf{Constraints:} the three addresses $a_X=\fp\cdot o_X$, and the sum $v_C=v_A+v_B$. +\item \textbf{Columns:} $\pc,\fp$; operands $o_A,o_B,o_C$; addresses $a_A,a_B,a_C$; values $v_A,v_B,v_C$ (each three $\K$-limbs); memory counts $r_A,r_B,r_C$; bytecode count $r_{\mathrm{bc}}$. +\item \textbf{Constraints:} the three addresses $a_X=\fp\cdot o_X$ ($\K$), and the sum $v_C=v_A+v_B$ (the $\E$-relation on the value limbs, reassembled $v=v_0+v_1 y+v_2y^2$; limb-wise it is three $\K$-equations). \item \textbf{Flushes:} \begin{itemize} \item \textbf{State:} \pull\ $\tup{\dsep{ST},\pc,\fp}$, \push\ $\tup{\dsep{ST},\gen\cdot\pc,\fp}$. @@ -365,7 +378,17 @@ \subsection{\texttt{MUL\_NATIVE}}\label{sec:tab-mul} Identical to \texttt{XOR} but for the product: \begin{itemize} \item \textbf{Columns:} as \texttt{XOR}. -\item \textbf{Constraints:} the three addresses, and the product $v_C=v_A\cdot v_B$ (the GHASH multiplication in $\F_{2^{128}}$). +\item \textbf{Constraints:} the three addresses and $v_C=v_Av_B$ in $\E$. With $y^3=y+1$, put +\[ +p_0=v_{A,0}v_{B,0},\quad +p_1=v_{A,0}v_{B,1}+v_{A,1}v_{B,0},\quad +p_2=v_{A,0}v_{B,2}+v_{A,1}v_{B,1}+v_{A,2}v_{B,0}, +\] +\[ +p_3=v_{A,1}v_{B,2}+v_{A,2}v_{B,1},\qquad +p_4=v_{A,2}v_{B,2}. +\] +Then $v_{C,0}=p_0+p_3$, $v_{C,1}=p_1+p_3+p_4$, and $v_{C,2}=p_2+p_4$. \item \textbf{Flushes:} as \texttt{XOR}, with opcode $\opc{MUL}$. \end{itemize} @@ -373,13 +396,13 @@ \subsection{\texttt{SET\_CONSTANT}}\label{sec:tab-set} Asserts $\loc{o}=k$, how a constant enters the read-only memory: the row ties the committed cell to the bytecode immediate $k$. \begin{itemize} -\item \textbf{Columns:} $\pc,\fp$; operand $o$; immediate $k$; address $a$; memory count $r$; bytecode count $r_{\mathrm{bc}}$. +\item \textbf{Columns:} $\pc,\fp$; operand $o$; immediate $k$ ($192$-bit, three $\K$-limbs $k_0,k_1,k_2$); address $a$; memory count $r$; bytecode count $r_{\mathrm{bc}}$. \item \textbf{Constraints:} the address $a=\fp\cdot o$. \item \textbf{Flushes:} \begin{itemize} \item \textbf{State:} \pull\ $\tup{\dsep{ST},\pc,\fp}$, \push\ $\tup{\dsep{ST},\gen\cdot\pc,\fp}$. - \item \textbf{Bytecode:} \pull\ $\tup{\dsep{BC},\pc,r_{\mathrm{bc}},\opc{SET},o,k,0,0,0}$, \push\ the same at $\gen\cdot r_{\mathrm{bc}}$. - \item \textbf{Memory:} \pull\ $\tup{\dsep{MEM},a,r,k}$, \push\ $\tup{\dsep{MEM},a,\gen\cdot r,k}$ (the value slot is the constant $k$). + \item \textbf{Bytecode:} \pull\ $\tup{\dsep{BC},\pc,r_{\mathrm{bc}},\opc{SET},o,k_0,k_1,k_2,0}$, \push\ the same at $\gen\cdot r_{\mathrm{bc}}$ (the immediate's three limbs ride the spare operand slots $5,6,7$). + \item \textbf{Memory:} \pull\ $\tup{\dsep{MEM},a,r,k}$, \push\ $\tup{\dsep{MEM},a,\gen\cdot r,k}$ (the value slot is the constant $k$, with all three limbs). \end{itemize} \end{itemize} @@ -393,12 +416,12 @@ \subsection{\texttt{DEREF}}\label{sec:tab-deref} \end{itemize} The store asserts that $v_2$ equals the source chosen by the mode $s$ (\S\ref{sec:vm}): the local cell $v_3$, the return address $\gen^{2}\cdot\pc$ (a virtual $\times\gen^{2}$ of $\pc$, \S\ref{sec:prelim}), or the frame pointer $\fp$. The mode is two boolean flags $(f_{\pc},f_{\fp})$: $(0,0)$ for $\texttt{deref\_cell}[\gamma]$, $(1,0)$ for $\texttt{deref\_pc}$, $(0,1)$ for $\texttt{deref\_fp}$. \begin{itemize} -\item \textbf{Columns:} $\pc,\fp$; operands $\alpha,\beta,\gamma$; flags $f_{\pc},f_{\fp}$; addresses $a_1,a_2,a_3$; pointer $p$; values $v_2,v_3$; memory counts $r_1,r_2,r_3$; bytecode count $r_{\mathrm{bc}}$. +\item \textbf{Columns:} $\pc,\fp$; operands $\alpha,\beta,\gamma$; flags $f_{\pc},f_{\fp}$; addresses $a_1,a_2,a_3$; pointer $p$ (a memory word, three $\K$-limbs); values $v_2,v_3$ (each three $\K$-limbs); memory counts $r_1,r_2,r_3$; bytecode count $r_{\mathrm{bc}}$. \item \textbf{Constraints:} the addresses $a_1=\fp\cdot\alpha$, $a_2=p\cdot\beta$, $a_3=\fp\cdot\gamma$, and the store \[ v_2 \;=\; (1+f_{\pc}+f_{\fp})\,v_3 \;+\; f_{\pc}\,(\gen^{2}\cdot\pc) \;+\; f_{\fp}\,\fp , \] -which gives $v_3$, $\gen^{2}\cdot\pc$, or $\fp$ at the three flag settings (in characteristic $2$ the $v_3$ term zeroes for the last two). The flags need no validity check: the verifier reads them from the public program (\S\ref{sec:e2e-bc}). +which gives $v_3$, $\gen^{2}\cdot\pc$, or $\fp$ at the three flag settings (in characteristic $2$ the $v_3$ term zeroes for the last two). The store is an $\E$-relation on the value limbs; $a_2=p\cdot\beta$, with $a_2$ a single $\K$ column and $\beta\in\K$, forces the pointer word $p$ into $\K$ (a genuine $\gen$-power). The flags need no validity check: the verifier reads them from the public program (\S\ref{sec:e2e-bc}). \item \textbf{Flushes:} \begin{itemize} \item \textbf{State:} \pull\ $\tup{\dsep{ST},\pc,\fp}$, \push\ $\tup{\dsep{ST},\gen\cdot\pc,\fp}$. @@ -409,7 +432,7 @@ \subsection{\texttt{DEREF}}\label{sec:tab-deref} \subsection{\texttt{JUMP}}\label{sec:tab-jump} -A conditional jump on whether the condition $c=\loc{o_c}$ is nonzero, with $c$ an arbitrary field element. It transfers to $\pc\gets d$, $\fp\gets f$ when $c\neq0$ (the destination $d=\loc{o_d}$, a code address, and the new frame $f=\loc{o_f}$), and falls through, $\pc\gets\gen\cdot\pc$, $\fp\gets\fp$, when $c=0$. The branch is taken according to a committed boolean indicator $b=[\,c\neq0\,]$, which a prover-supplied inverse $w$ certifies through the two degree-$2$ constraints +A conditional jump on whether the condition $c=\loc{o_c}$ is nonzero, with $c$ an arbitrary $192$-bit word. It transfers to $\pc\gets d$, $\fp\gets f$ when $c\neq0$ (the destination $d=\loc{o_d}$, a code address, and the new frame $f=\loc{o_f}$), and falls through, $\pc\gets\gen\cdot\pc$, $\fp\gets\fp$, when $c=0$. The branch is taken according to a committed boolean indicator $b=[\,c\neq0\,]$, which a prover-supplied inverse $w$ certifies through the two degree-$2$ constraints \[ b=c\cdot w,\qquad c\,(b+1)=0 . \] @@ -419,8 +442,8 @@ \subsection{\texttt{JUMP}}\label{sec:tab-jump} \] Alone among the tables, \texttt{JUMP} commits its successors. \begin{itemize} -\item \textbf{Columns:} $\pc,\fp,\mathit{next\_pc},\mathit{next\_fp}$; operands $o_c,o_d,o_f$; addresses $a_c,a_d,a_f$; values $c,d,f$; inverse $w$; indicator $b$; memory counts $r_c,r_d,r_f$; bytecode count $r_{\mathrm{bc}}$. ($w$ and $b$ are witness columns, neither read from memory nor in the bytecode; the fall-through $s=\gen\cdot\pc$ is virtual.) -\item \textbf{Constraints:} the three addresses $a_X=\fp\cdot o_X$; the indicator pair $b=c\cdot w$ and $c\,(b+1)=0$; and the two selections $\mathit{next\_pc},\mathit{next\_fp}$ above. +\item \textbf{Columns:} $\pc,\fp$ and $\mathit{next\_pc},\mathit{next\_fp}$ (single $\K$-columns, the next state); operands $o_c,o_d,o_f$; addresses $a_c,a_d,a_f$; the condition $c$ and the destination/frame words $d,f$ (each three $\K$-limbs); inverse $w$ (three $\K$-limbs, $c^{-1}$ in $\E$); indicator $b$ (one $\K$-limb, $0/1$); memory counts $r_c,r_d,r_f$; bytecode count $r_{\mathrm{bc}}$. ($w$ and $b$ are witness columns, neither read from memory nor in the bytecode; the fall-through $s=\gen\cdot\pc$ is virtual.) +\item \textbf{Constraints:} the three addresses $a_X=\fp\cdot o_X$; the indicator pair $b=c\cdot w$ and $c\,(b+1)=0$ ($\E$-relations on the limbs); and the two selections $\mathit{next\_pc},\mathit{next\_fp}$ above. Since $\mathit{next\_pc},\mathit{next\_fp}$ are single $\K$-columns, the selections force the chosen words $d,f$ into $\K$ on a taken branch. \item \textbf{Flushes:} \begin{itemize} \item \textbf{State:} \pull\ $\tup{\dsep{ST},\pc,\fp}$, \push\ $\tup{\dsep{ST},\mathit{next\_pc},\mathit{next\_fp}}$. @@ -429,19 +452,35 @@ \subsection{\texttt{JUMP}}\label{sec:tab-jump} \end{itemize} \end{itemize} +\subsection{\texttt{PACK64X2}}\label{sec:tab-pack64x2} + +Asserts that two source cells are $\K$-valued and packs them into one canonical $128$-bit word. +\begin{itemize} +\item \textbf{Columns:} $\pc,\fp$; operands $o_A,o_B,o_C$; addresses $a_A,a_B,a_C$; source limbs $v_A,v_B\in\K$; memory counts $r_A,r_B,r_C$; bytecode count $r_{\mathrm{bc}}$. +\item \textbf{Constraints:} the three addresses $a_X=\fp\cdot o_X$. Value validity is enforced by the memory interactions rather than separate AIR equations. +\item \textbf{Flushes:} + \begin{itemize} + \item \textbf{State:} \pull\ $\tup{\dsep{ST},\pc,\fp}$, \push\ $\tup{\dsep{ST},\gen\cdot\pc,\fp}$. + \item \textbf{Bytecode:} \pull\ $\tup{\dsep{BC},\pc,r_{\mathrm{bc}},\opc{PACK},o_A,o_B,o_C,0,0}$, \push\ the same at $\gen\cdot r_{\mathrm{bc}}$. + \item \textbf{Memory:} read $\tup{\dsep{MEM},a_A,r_A,v_A,0,0}$ and $\tup{\dsep{MEM},a_B,r_B,v_B,0,0}$, then read the destination as $\tup{\dsep{MEM},a_C,r_C,v_A,v_B,0}$; each pull is paired with the corresponding push at count $\gen\cdot r$. The literal zeros make the two source range assertions and the destination packing exact through bus balance. + \end{itemize} +\end{itemize} + \subsection{\texttt{BLAKE3}}\label{sec:tab-blake3} -Compresses a $64$-byte message block (four $128$-bit words) under a $32$-byte chaining value to the $32$-byte result at $c$. The message words are addressed independently at $a_i=\fp\cdot o_i$, while $a_V=\fp\cdot o_V$ is the base of the two consecutive chaining-value cells $a_V,\gen a_V$. The output similarly occupies $a_C=\fp\cdot o_C$ and $\gen a_C$. The immediate $\mu$ packs the $64$-bit counter, $32$-bit block length, and $32$-bit flags required by the BLAKE3 specification. +Compresses a $64$-byte message block (four $128$-bit chunks) under a $32$-byte chaining value to the $32$-byte result at $c$. Each chunk occupies one $192$-bit cell through the canonical embedding with top limb zero. The message chunks are addressed independently at $a_i=\fp\cdot o_i$, while $a_V=\fp\cdot o_V$ is the base of the two consecutive chaining-value cells $a_V,\gen a_V$. The output similarly occupies $a_C=\fp\cdot o_C$ and $\gen a_C$, so each row accesses eight cells. The $128$-bit immediate $\mu$ packs the $64$-bit counter, $32$-bit block length, and $32$-bit flags required by the BLAKE3 specification; its two $\K$-lanes $\mu_0,\mu_1$ ride the bytecode tuple. -The compression is \emph{proven} by flock~\cite{flock}, which proves a batched R1CS of the circuit against a committed Boolean witness $w$. We commit flock's ring-switched~\cite{cryptoeprint:2024/504} witness $\widehat w$ over $\F_{2^{128}}$ alongside our own columns, in one stacked commitment (\S\ref{sec:rs-dense}). The message, result, chaining value, and metadata already sit inside $\widehat w$ at fixed $128$-bit-aligned slots. Consequently the nine virtual value columns $v_{A0},v_{A1},v_{B0},v_{B1},v_{C0},v_{C1},v_{V0},v_{V1},v_\mu$ are not materialized in the AIR commitment: their opening claims are routed to the aligned packed coordinates of $q_{\mathrm{pkd}}$. +Flock~\cite{flock} proves the compression relation with a batched R1CS over a Boolean witness. The witness is packed into $q_\pkd$, with $64$ bits per $\K$-element. The concrete ring switch of \S\ref{sec:ringswitch} maps its $\F_2$ claims to one $\E$-weighted claim on $q_\pkd$, which joins the batched opening (\S\ref{sec:e2e-unrolled}). + +The message, result, chaining value, and metadata are all contained in $q_\pkd$. Each $64$-bit flock word occupies one packed slot, and each BLAKE3 cell contributes its two low limbs; the top limb is canonically zero. The eighteen limbs $v_{A0},\dots,v_{C3},v_{V0},\dots,v_{V3},\mu_0,\mu_1$ therefore need no separate commitment: each is a strided point evaluation of $q_\pkd$. Linking these embedded values to the full memory interaction implicitly forces the top limb of every BLAKE3-accessed cell to zero, and the zero limbs are never committed. The generalized R1CS matrices take the chaining value, counter, block length, and flags as free input rows; the bus and bytecode interactions bind them. \begin{itemize} -\item \textbf{Columns:} $\pc,\fp$; operands $o_0,o_1,o_2,o_3,o_V,o_C$; base addresses $a_0,a_1,a_2,a_3,a_V,a_C$; memory counts $r_{A0},r_{A1},r_{B0},r_{B1},r_{V0},r_{V1},r_{C0},r_{C1}$; bytecode count $r_{\mathrm{bc}}$. The nine values above are virtual columns backed by $q_{\mathrm{pkd}}$. The successor addresses $\gen a_V$ and $\gen a_C$ are virtual. -\item \textbf{Constraints:} the six address bindings $a_i=\fp\cdot o_i$ ($i\in\{0,1,2,3\}$), $a_V=\fp\cdot o_V$, and $a_C=\fp\cdot o_C$ — all degree $2$. The compression relation itself is handled by flock. +\item \textbf{Columns:} $\pc,\fp$; operands $o_0,o_1,o_2,o_3,o_V,o_C$; base addresses $a_0,a_1,a_2,a_3,a_V,a_C$; per-cell memory counts $r_0,\dots,r_3$ (the four message cells), $r_{V0},r_{V1}$ (the two chaining-value cells), and $r_{C0},r_{C1}$ (the two output cells); bytecode count $r_{\mathrm{bc}}$. The eighteen value limbs above are committed "inside" $q_\pkd$, see above; each cell's third limb is zero. The successor addresses $\gen\cdot a_V$ and $\gen\cdot a_C$ are virtual. +\item \textbf{Constraints:} the six degree-$2$ address bindings $a_i=\fp\cdot o_i$ for $i\in\{0,1,2,3\}$, $a_V=\fp\cdot o_V$, and $a_C=\fp\cdot o_C$. The compression relation itself is handled by flock. \item \textbf{Flushes:} \begin{itemize} \item \textbf{State:} \pull\ $\tup{\dsep{ST},\pc,\fp}$, \push\ $\tup{\dsep{ST},\gen\cdot\pc,\fp}$. - \item \textbf{Bytecode:} \pull\ $\tup{\dsep{BC},\pc,r_{\mathrm{bc}},\opc{B3},o_0,o_1,o_2,o_3,o_V,o_C,v_\mu}$, \push\ the same at $\gen\cdot r_{\mathrm{bc}}$. - \item \textbf{Memory:} eight reads, each \pull\ at count $r$ and \push\ at $\gen\cdot r$: four independently addressed message words, the chaining value at $a_V,\gen a_V$, and the result at $a_C,\gen a_C$. + \item \textbf{Bytecode:} \pull\ $\tup{\dsep{BC},\pc,r_{\mathrm{bc}},\opc{B3},o_0,o_1,o_2,o_3,o_V,o_C,\mu_0,\mu_1}$, \push\ the same at $\gen\cdot r_{\mathrm{bc}}$. + \item \textbf{Memory:} eight reads, each \pull\ at count $r$ and \push\ at $\gen\cdot r$, its embedded value carrying two limbs from $q_\pkd$ and a zero top limb (\S\ref{sec:memchan}): four independently addressed message cells, the chaining value at $a_V,\gen a_V$, and the result at $a_C,\gen a_C$. \end{itemize} \end{itemize} @@ -451,9 +490,15 @@ \section{End-to-end protocol}\label{sec:e2e} \subsection{Constants}\label{sec:e2e-const} -Fixed once, public, and shared by prover and verifier: the field, the bus shape, and the tags naming interactions and instructions on it. +Fixed once, public, and shared by prover and verifier: the fields, the bus shape, and the tags naming interactions and instructions on it. -\paragraph{Field, generator, and bus.} The field is $\F_{2^{128}}$ in the GHASH representation $\F_2[x]/(x^{128}+x^7+x^2+x+1)$. The generator is $\gen=x$ (has order $2^{128}-1$). Every bus tuple has width $m=11$ (the bytecode tuple); shorter tuples are zero-padded in their high coordinates (\S\ref{sec:m3}). +\paragraph{Fields, generator, and bus.} +\[ +\K=\F_2[x]/(x^{64}+x^4+x^3+x+1),\qquad +\E=\K[y]/(y^3+y+1),\qquad +\gen=x,\qquad \operatorname{ord}(\gen)=2^{64}-1. +\] +Committed columns are $\K$-valued; machine words and challenges are $\E$-valued (\S\ref{sec:prelim}). Every bus tuple has width $m=12$ (the bytecode tuple); shorter tuples are zero-padded in their high coordinates (\S\ref{sec:m3}). \paragraph{Domain separators.} Coordinate $0$ of every tuple is a domain separator, naming the interaction it belongs to (\S\ref{sec:m3}): \[ @@ -462,27 +507,32 @@ \subsection{Constants}\label{sec:e2e-const} \paragraph{Opcodes.} Coordinate $3$ of every bytecode tuple is an opcode, naming the instruction stored at that program counter (\S\ref{sec:bytecode}): \[ - \opc{XOR}=1,\quad \opc{MUL}=\gen,\quad \opc{SET}=\gen^{2},\quad \opc{DRF}=\gen^{3},\quad \opc{JMP}=\gen^{4},\quad \opc{B3}=\gen^{5}. + \opc{XOR}=1,\quad \opc{MUL}=\gen,\quad \opc{SET}=\gen^{2},\quad \opc{DRF}=\gen^{3},\quad \opc{JMP}=\gen^{4},\quad \opc{B3}=\gen^{5},\quad \opc{PACK}=\gen^{6}. \] \subsection{Bytecode encoding}\label{sec:e2e-bc} -The program is public. Instruction $\pc$ fills coordinates $3$ through $10$ of its bytecode tuple $\tup{\dsep{BC},\pc,\cnt,\cdot}$: an opcode in coordinate $3$ and seven operand/immediate slots in coordinates $4,\ldots,10$. Each reference operand is a $\gen$-power; constants and $\mu$ are field immediates. +The program is public. Instruction $\pc$ fills coordinates $3$ through $11$ of its bytecode tuple +\[ + \tup{\dsep{BC},\pc,\cnt,\cdot}. +\] +Coordinate $3$ holds the opcode and coordinates $4,\ldots,11$ hold eight operands or immediate lanes. Each reference operand is a $\gen$-power. Immediates occupy their $\K$-lanes: $k_0,k_1,k_2$ for \texttt{SET\_CONSTANT}, and $\mu_0,\mu_1$ for \texttt{BLAKE3}. \begin{center} -\begin{tabular}{lllllllll} +\begin{tabular}{llllllllll} \hline -Instruction & $3$ & $4$ & $5$ & $6$ & $7$ & $8$ & $9$ & $10$\\ +Instruction & $3$ & $4$ & $5$ & $6$ & $7$ & $8$ & $9$ & $10$ & $11$\\ \hline -\texttt{XOR} & $\opc{XOR}$ & $o_A$ & $o_B$ & $o_C$ & $0$ & $0$ & $0$ & $0$\\ -\texttt{MUL\_NATIVE} & $\opc{MUL}$ & $o_A$ & $o_B$ & $o_C$ & $0$ & $0$ & $0$ & $0$\\ -\texttt{SET\_CONSTANT} & $\opc{SET}$ & $o$ & $k$ & $0$ & $0$ & $0$ & $0$ & $0$\\ -\texttt{DEREF} & $\opc{DRF}$ & $\alpha$ & $\beta$ & $\gamma$ & $f_{\pc}$ & $f_{\fp}$ & $0$ & $0$\\ -\texttt{JUMP} & $\opc{JMP}$ & $o_c$ & $o_d$ & $o_f$ & $0$ & $0$ & $0$ & $0$\\ -\texttt{BLAKE3} & $\opc{B3}$ & $o_0$ & $o_1$ & $o_2$ & $o_3$ & $o_V$ & $o_C$ & $\mu$\\ +\texttt{XOR} & $\opc{XOR}$ & $o_A$ & $o_B$ & $o_C$ & $0$ & $0$ & $0$ & $0$ & $0$\\ +\texttt{MUL\_NATIVE} & $\opc{MUL}$ & $o_A$ & $o_B$ & $o_C$ & $0$ & $0$ & $0$ & $0$ & $0$\\ +\texttt{SET\_CONSTANT} & $\opc{SET}$ & $o$ & $k_0$ & $k_1$ & $k_2$ & $0$ & $0$ & $0$ & $0$\\ +\texttt{DEREF} & $\opc{DRF}$ & $\alpha$ & $\beta$ & $\gamma$ & $f_{\pc}$ & $f_{\fp}$ & $0$ & $0$ & $0$\\ +\texttt{JUMP} & $\opc{JMP}$ & $o_c$ & $o_d$ & $o_f$ & $0$ & $0$ & $0$ & $0$ & $0$\\ +\texttt{PACK64X2} & $\opc{PACK}$ & $o_A$ & $o_B$ & $o_C$ & $0$ & $0$ & $0$ & $0$ & $0$\\ +\texttt{BLAKE3} & $\opc{B3}$ & $o_0$ & $o_1$ & $o_2$ & $o_3$ & $o_V$ & $o_C$ & $\mu_0$ & $\mu_1$\\ \hline \end{tabular} \end{center} -So the program is eight public columns (opcode plus seven slots), whose MLEs the verifier forms itself. Writing $\mathit{instr}$ for the eight, the bytecode seed at $\pc$ pushes $\tup{\dsep{BC},\pc,1,\mathit{instr}}$ and an instruction table pulls it with its opcode hardcoded (\S\ref{sec:tables}); balance then forces every operand, flag, and immediate to match the program at its $\pc$. +So the program is nine public columns (opcode plus eight slots), whose MLEs the verifier forms itself. Writing $\mathit{instr}$ for the nine, the bytecode seed at $\pc$ pushes $\tup{\dsep{BC},\pc,1,\mathit{instr}}$ and an instruction table pulls it with its opcode hardcoded (\S\ref{sec:tables}); balance then forces every operand, flag, and immediate to match the program at its $\pc$. The program is padded to a power of two (\S\ref{sec:idxcol}). A padding entry is never executed: no table pulls it, so its seed push and finalize pull (both at count $1$) cancel. @@ -492,20 +542,25 @@ \subsection{Padding rows}\label{sec:e2e-pad} \subsection{Public input}\label{sec:e2e-pi} -The public input is the first two memory cells $\mem[0],\mem[1]$ ($256$ bits), fixed and agreed by both parties. Cells $0$ and $1$ span the subcube of the address cube $\cube h$ with the $h-1$ high coordinates $0$ and the low one free, so the committed memory MLE $\widetilde M$ (\S\ref{sec:memchan}) restricts there to the line $\widetilde{\mathrm{PI}}(r)=(1-r)\,\mem[0]+r\,\mem[1]$, which the verifier forms itself. It samples $r_m\in\F_{2^{128}}$ and checks +The public input is the first two memory cells $\mem[0],\mem[1]$, each a $192$-bit word, fixed and agreed by both parties. Cells $0$ and $1$ span the subcube of the address cube $\cube h$ with the $h-1$ high coordinates $0$ and the low one free, so on \emph{each} committed memory limb $M_\ell$ ($\ell\in\{0,1,2\}$, \S\ref{sec:memchan}) the MLE restricts there to the line $\widetilde{\mathrm{PI}}_\ell(r)=(1-r)\,\mem[0]_\ell+r\,\mem[1]_\ell$, which the verifier forms itself from the public words. It samples $r_m\in\E$ and checks all three limbs, \[ - \widetilde M(r_m,0,\dots,0)\;=\;\widetilde{\mathrm{PI}}(r_m). + \widetilde M_\ell(r_m,0,\dots,0)\;=\;\widetilde{\mathrm{PI}}_\ell(r_m),\qquad \ell\in\{0,1,2\}. \] +Writing $P=P_0+P_1y+P_2y^2$ for the public-input evaluation, the implementation transmits $P_0,P_1$ and computes +\[ +P_2=(P+P_0+P_1y)(y^2)^{-1}. +\] +The final stacked opening discharges all three limb claims. \subsection{The unrolled protocol}\label{sec:e2e-unrolled} We lay out the whole protocol as one ordered sequence. Each reduction below adds evaluation claims on committed columns to a shared \emph{claim pool}; the final \textsc{Opening} phase proves all of them with a single PCS opening. -\paragraph{Setup.} Fixed and shared in advance (\S\ref{sec:e2e-const} through \S\ref{sec:e2e-pi}): the field and constants; the program, which is public; the memory size $2^{h}$ and the program length; the initial state $(\pc_0,\fp_0)$ and final state $(\pc_{\mathrm{final}},\fp_{\mathrm{final}})$; and the public input $\mem[0],\mem[1]$. +\paragraph{Setup and statement binding.} Fixed and shared in advance (\S\ref{sec:e2e-const} through \S\ref{sec:e2e-pi}): the fields and constants; the instance caps (\S\ref{sec:memchan}); the public program and its length; the initial state $(\pc_0,\fp_0)$ and final state $(\pc_{\mathrm{final}},\fp_{\mathrm{final}})$; and the public input $\mem[0],\mem[1]$. Before any challenge, the transcript is seeded by the public input and one environment digest that binds both the exact bytecode and flock's BLAKE3 circuit family. The prover then announces $h$ and the seven real opcode-row counts; the verifier checks them against the caps and derives every table, stack, and leaf shape. Thus neither a different program nor an adaptively chosen shape can reuse the proof's later challenges. \paragraph{Commitment.} \begin{enumerate} -\item \textbf{Prover.} Builds all committed columns (field-valued): each table's state, operands, addresses, read values, and counts (plus $w,b,\mathit{next\_pc},\mathit{next\_fp}$ for \texttt{JUMP}); the memory column $M$; and the memory and bytecode finalize counts. Counts are always committed $\gen$-powers (\S\ref{sec:memchan}); the public program and the virtual increments $\gen\cdot\pc,\gen\cdot\cnt$ are not. It stacks these into $q$, sends the dense commitment (\S\ref{sec:stacking}, \S\ref{sec:rs-dense}), and announces each table's non-padded row count, which the verifier rounds up to a power of two to fix the stack and leaf layouts (\S\ref{sec:leafstack}). Padding rows are a fixed default (\S\ref{sec:e2e-pad}); the announced sizes fix the surplus the verifier later divides out (\S\ref{sec:gp}). +\item \textbf{Prover.} Builds the $\K$-valued committed columns: each table's state, operands, addresses, read-value limbs, and counts; the three memory limbs; the memory and bytecode finalize counts; and flock's packed witness $q_\pkd$ (\S\ref{sec:tab-blake3}). A program with no \texttt{BLAKE3} execution carries one padding instance so the proof shape remains uniform. A $192$-bit value occupies three $\K$-limb columns (\S\ref{sec:prelim}). The \texttt{JUMP} table additionally commits $w,b,\mathit{next\_pc},\mathit{next\_fp}$. Counts are committed $\gen$-powers (\S\ref{sec:memchan}); the public program and virtual increments $\gen\cdot\pc,\gen\cdot\cnt$ are not. The prover stacks these columns into $q$, sends the commitment (\S\ref{sec:stacking}, \S\ref{sec:rs-dense}), and announces each table's real row count. The verifier checks the instance caps and derives the power-of-two table, stack, and leaf shapes (\S\ref{sec:leafstack}). The sizes also fix the default-row surplus later divided out of the grand product (\S\ref{sec:e2e-pad}, \S\ref{sec:gp}). \end{enumerate} \paragraph{Bus.} @@ -513,44 +568,52 @@ \subsection{The unrolled protocol}\label{sec:e2e-unrolled} \item \textbf{Verifier.} Sends the fingerprint challenge $\alpha$ and the multiset challenge $\gamma$. \item \textbf{Prover.} Forms the push, pull, and \emph{count} leaf vectors (\S\ref{sec:leafstack}; the count leaves are the count columns themselves, \S\ref{sec:memchan}), stacked into blocks and padded with $1$s. Sends the count root $R_c$ and one bus root $R$, the push/pull product after the default surplus is removed (\S\ref{sec:gp}). \item \textbf{Prover \& Verifier.} Run the one batched GKR (\S\ref{sec:gkr}) over all three trees. For the bus, the verifier multiplies $R$ by each side's public surplus and checks the side's product against it; one $R$ for both sides is the balance check (\S\ref{sec:gp}). For the counts it checks $R_c\neq0$ (no read self-cancels, \S\ref{sec:memchan}). The pass yields three leaf claims $\widetilde V^{t}_0(\zeta)$ at one shared $\zeta$. -\item \textbf{Prover \& Verifier.} Decompose each leaf claim (\S\ref{sec:leafstack}). The verifier forms the public part: the block selectors; the constant coordinates (separators, the per-table read opcodes, the boundary state $(\pc_0,\fp_0)$ and $(\pc_{\mathrm{final}},\fp_{\mathrm{final}})$); the index column (\S\ref{sec:idxcol}); and the public program columns of the bytecode seed and finalize, whose MLEs it forms directly. For the table-less blocks the prover supplies the committed-column evaluations (one field element each, \S\ref{sec:prelim}); a virtual coordinate such as $\gen\cdot\pc$ is read off its committed source, and these evaluations enter the claim pool. It sends nothing for the other blocks: what they owe the side is $\widetilde V_0(\zeta)$ less the part just formed, so the verifier forms that too, and the batch below is what settles it. The count channel decomposes likewise; none of its blocks is table-less, so its whole share falls to the batch. +\item \textbf{Prover \& Verifier.} Decompose each leaf claim (\S\ref{sec:leafstack}). The verifier forms the public part: the block selectors; the constant coordinates (separators, the per-table read opcodes, the boundary state $(\pc_0,\fp_0)$ and $(\pc_{\mathrm{final}},\fp_{\mathrm{final}})$); the index column (\S\ref{sec:idxcol}); and the public program columns of the bytecode seed and finalize, whose MLEs it forms directly. For the table-less blocks the prover supplies the committed-column evaluations (one $\E$-element each, \S\ref{sec:prelim}); a virtual coordinate such as $\gen\cdot\pc$ is read off its committed source, and these evaluations enter the claim pool. It sends nothing for the other blocks: what they owe the side is $\widetilde V_0(\zeta)$ less the part just formed, so the verifier forms that too, and the batch below settles it. The count channel decomposes likewise; none of its blocks is table-less, so its whole share falls to the batch. \end{enumerate} -\paragraph{Local constraints.} One run for all six tables (\S\ref{sec:air}). +\paragraph{Local constraints.} One run for all seven tables (\S\ref{sec:air}). \begin{enumerate} \item \textbf{Verifier.} Sends the constraint-batch challenge $\eta$. Table $T_j$ takes a disjoint range of its powers, one per constraint; the bus forms absorbed above take three further powers, one per side and shared by every table. No point is drawn, the batch running at the bus point with $T_j$ tested at $\zeta_{<\tau_j}$. -\item \textbf{Prover \& Verifier.} Run the back-loaded batched zerocheck, an $n$-round sumcheck with $n=\max_j\tau_j$, a table of $2^{\tau}$ rows joining at round $n-\tau$. It starts from a target the verifier derives from the three leaf claims, and each round is four field elements, the round polynomial itself at four nodes. Its end needs every column of every table, \texttt{BLAKE3}'s nine value columns included, whose claims route to $q_{\mathrm{pkd}}$; each at that table's own reduced point $\rho_{<\tau_j}$, all of them prefixes of one $\rho$; these enter the pool, and they are also what the bus forms are read off. +\item \textbf{Prover \& Verifier.} Run the back-loaded batched zerocheck, an $n$-round sumcheck with $n=\max_j\tau_j$, a table of $2^{\tau}$ rows joining at round $n-\tau$. It starts from a target the verifier derives from the three leaf claims, and each round is four $\E$-elements, the round polynomial itself at four nodes. Its end needs every column of every table, \texttt{BLAKE3}'s eighteen value columns included, whose claims route to $q_{\mathrm{pkd}}$; each at that table's own reduced point $\rho_{<\tau_j}$, all of them prefixes of one $\rho$; these enter the pool, and they are also what the bus forms are read off. \end{enumerate} \paragraph{Public input.} \begin{enumerate} -\item \textbf{Verifier.} Sends $r_m$, forms $\widetilde{\mathrm{PI}}(r_m)$ from the public input, and adds to the pool the memory claim $\widetilde M(r_m,0,\dots,0)=\widetilde{\mathrm{PI}}(r_m)$ (\S\ref{sec:e2e-pi}). +\item \textbf{Verifier.} Sends $r_m$, forms $\widetilde{\mathrm{PI}}_\ell(r_m)$ from the public input, and adds to the pool the three memory-limb claims $\widetilde M_\ell(r_m,0,\dots,0)=\widetilde{\mathrm{PI}}_\ell(r_m)$, $\ell\in\{0,1,2\}$ (\S\ref{sec:e2e-pi}). +\end{enumerate} + +\paragraph{BLAKE3 validity.} +\begin{enumerate} +\item \textbf{Prover \& Verifier.} Reduce the executed BLAKE3 constraints to two evaluation claims on $q_\pkd$ (\S\ref{sec:tab-blake3}). +\item \textbf{Prover \& Verifier.} The ring switching of \S\ref{sec:ringswitch} reduces each bit-witness claim to a weighted-sum claim on the $q_\pkd$ region of the stack, with a transparent $\E$-valued weight the verifier can evaluate; both join the pool. \end{enumerate} \paragraph{Opening.} \begin{enumerate} -\item \textbf{Prover \& Verifier.} Each pooled claim is one evaluation of a committed column at a point, its value already supplied; via the stacking selectors (\S\ref{sec:stacking}) it becomes a weighted sum $\sum_w W_j(w)\,\widehat q(w)=c_j$ over the stacked witness. -\item \textbf{Verifier.} Sends the batching challenge $\lambda$. -\item \textbf{Prover \& Verifier.} Fold the $J$ claims into $W_\lambda=\sum_j\lambda^{j}W_j$ and $C_\lambda=\sum_j\lambda^{j}c_j$, and run the inner-product PCS opening on $\sum_w W_\lambda(w)\,\widehat q(w)=C_\lambda$ (\S\ref{sec:rs-dense}), the verifier evaluating $W_\lambda$ itself. This one opening discharges every pooled claim; there is no separate reduction sumcheck. -\item \textbf{Verifier.} Accepts iff the two roots agreed after the surplus division, the count root was nonzero and the bus grinding held, every sumcheck's rounds were consistent (the batch's against the target derived from the three leaf claims) and its final value matched the supplied evaluations, the public-input claim held, and the PCS opening verified. +\item \textbf{Prover \& Verifier.} Each pooled claim is one evaluation of a committed column at a point, its value already supplied; via the stacking selectors (\S\ref{sec:stacking}) it becomes a weighted sum $\sum_w W_j(w)\,\widehat q(w)=c_j$ over the stacked witness. (The ring-switched claims arrive already in this form, their weights supported on the $q_\pkd$ region.) +\item \textbf{Verifier.} Sends the batching coefficients $\lambda_1,\dots,\lambda_J\in\E$, one per claim. +\item \textbf{Prover \& Verifier.} Fold the $J$ claims into $W_\lambda=\sum_j\lambda_j W_j$ and $C_\lambda=\sum_j\lambda_j c_j$, and run the inner-product PCS opening on $\sum_w W_\lambda(w)\,\widehat q(w)=C_\lambda$ (\S\ref{sec:rs-dense}), the verifier evaluating $W_\lambda$ itself. This one opening discharges every pooled claim; there is no separate reduction sumcheck. +\item \textbf{Verifier.} Accepts iff the announced sizes passed the caps, the two roots agreed after the surplus division, the count root was nonzero, every sumcheck's rounds were consistent (the batch's against the target derived from the three leaf claims) and its final value matched the supplied evaluations, flock's reduction verified, the public-input claim held, and the PCS opening verified. \end{enumerate} \section{Recursive aggregation}\label{sec:recursion} -Proofs compose by running the verifier of \S\ref{sec:e2e-unrolled} \emph{inside} the VM: a guest program replays a sub-proof's transcript, sumchecks, and PCS opening, so many sub-proofs collapse into one. The VM is well suited to this, since the verifier's field arithmetic is the VM's native arithmetic and its hashing is the \texttt{BLAKE3} opcode. One step, however, does not fit. +The implemented N-to-1 aggregation runs the verifier of \S\ref{sec:e2e-unrolled} \emph{inside} the VM: one compiled guest replays the transcripts, sumchecks, ring switches, and stacked Ligerito openings of $n_{\mathrm{rec}}\ge1$ ordinary VM proofs for a fixed inner program. The proofs may have different announced table and commitment sizes; the guest dispatches among the certified shapes without recompilation. The VM is well suited to this because $\E$ is its machine-word field and its Fiat--Shamir hash is the \texttt{BLAKE3} opcode. One class of operations, however, does not fit. \subsection{Deferred evaluation claims}\label{sec:deferred-claims} Twice during verification, a large \emph{fixed} multilinear must be evaluated at a random point: \begin{itemize} -\item \textbf{the bytecode}: during leaf decomposition (\S\ref{sec:e2e-unrolled}, \emph{Bus}), the verifier evaluates the MLEs of the public program columns, a pass over the whole program; +\item \textbf{the bytecode}: during leaf decomposition (\S\ref{sec:e2e-unrolled}, \emph{Bus}), the verifier evaluates the nine public program columns, padded to sixteen selector slots under four selector variables, in one pass over the whole program; \item \textbf{the flock matrices}: the \texttt{BLAKE3} relation is proven by flock (\S\ref{sec:tab-blake3}), whose lincheck ends by evaluating $\widetilde A_0$ and $\widetilde B_0$, the per-block R1CS matrices: $28$-variable multilinears with $\approx 21\mathrm{M}$ nonzero Boolean entries between them. \end{itemize} Either evaluation is one pass over a fixed object. This takes milliseconds natively, but in-circuit it would dwarf the entire remaining verifier, in cycles and in bytecode size alike. -The aggregation program therefore does not check these claims. The polynomials are fixed and public, so each claim (an evaluation point and its claimed value) is forwarded to the public input, to be checked externally. +The aggregation guest therefore does not evaluate these fixed tables. It verifies every surrounding reduction and exports only the resulting points and claimed values. + +There are three fixed polynomials: the bytecode and the two matrices $A_0,B_0$. Verifying each ordinary sub-proof produces one claim on each, for $3n_{\mathrm{rec}}$ claims. A fresh Fiat--Shamir transcript absorbs all sub-statements and claim data, then batches them with two sumchecks: one for bytecode and one for the matrices, which share their $28$ variables. The aggregation and recursive-statement transcripts use distinct domain labels; each absorbs $n_{\mathrm{rec}}$ before any variable-length sequence, so concatenations cannot be confused. The guest reduces the batch to one bytecode evaluation and a common-point evaluation of each matrix. These three reduced claims, the inner proving-environment digest, and the sub-statements are hashed into the outer VM's two-word public input. -There are thus three fixed polynomials: the bytecode and the two matrices $A_0, B_0$. A sub-proof may itself be an aggregation carrying deferred claims, so verifying it yields \textbf{two} claims per polynomial: the one read from its public input, and the fresh one produced by the verification itself. Once all $n_{\mathrm{rec}}$ sub-proofs are verified, a fresh Fiat-Shamir transcript is fed with all the points and values, and the $6n_{\mathrm{rec}}$ claims are batched by two sumchecks: one for the bytecode, and one for the matrices, which share their $28$ variables. The three reduced claims are written to the public input, for the next level of recursion (zeros when $n_{\mathrm{rec}}=0$). Only the outermost verifier, which runs natively, ever evaluates the fixed polynomials. +The public \texttt{RecursiveProof::verify} path first verifies that outer VM proof and then evaluates the three fixed polynomials natively at the reduced points. It rejects an empty batch and is the only complete acceptance path. Feeding a \texttt{RecursiveProof} into another aggregation layer (carrying its reduced claims forward rather than checking them natively) is not yet exposed by the current harness. The matrix sumcheck runs over the $28$-variable polynomial \[ @@ -562,11 +625,17 @@ \subsection{Deferred evaluation claims}\label{sec:deferred-claims} \section{ISA programming}\label{sec:isa-programming} -How the zkDSL compiler (\texttt{crates/lean\_compiler}; surface language in \texttt{crates/lean\_compiler/zkDSL.md}) programs the six-instruction ISA. Two themes run through every pattern: \emph{write-once memory is an assertion mechanism} (a second write of a cell is an equality check, an unwritten cell is prover-chosen), and \emph{indices live in the exponent} (a logical index $i$ is the element $\gen^{i}$, so incrementing, offsetting, and address formation are single field multiplications). +How the zkDSL compiler (\path{crates/lean_compiler}; surface language in \path{crates/lean_compiler/zkDSL.md}) programs the seven-instruction ISA. Two themes run through every pattern: \emph{write-once memory is an assertion mechanism} (a second write of a cell is an equality check, an unwritten cell is prover-chosen), and \emph{indices live in the exponent} (a logical index $i$ is the element $\gen^{i}$, so incrementing, offsetting, and address formation are single field multiplications). \subsection{Hints}\label{sec:prog-hints} -The prover's nondeterminism enters only through write-once cells that no instruction defines — e.g.\ allocation pointers: a fresh region's base $\gen^{\mathrm{base}}$, written unconstrained. +The prover's nondeterminism enters through write-once cells that are still unset when execution needs them. Some are explicit advice (a fresh allocation pointer $\gen^{\mathrm{base}}$, a witness stream, a computed-advice builtin); others are back-solved by an instruction whose remaining operands are known. In every case the resulting word is unconstrained except by the instructions and memory equalities that consume it, so a program must check all security-relevant advice. + +\subsection{Division and inequality}\label{sec:prog-div-ne} + +The zkDSL's single-slash division $q=a/b$ uses write-once back-solving: it emits \texttt{MUL\_NATIVE} with $q$ as the one unset input and $a$ as the already-written output. Witness generation fills $q=a\,b^{-1}$, while the ordinary multiplication table proves $q b=a$. It costs one VM instruction. Division by zero is undefined. Double slash \texttt{//} remains compile-time integer floor division for sizes and indices; it is not a field operation. + +The assertion \texttt{assert a != b} first computes $a+b$ with \texttt{XOR}. A conditional \texttt{JUMP} skips a poison path exactly when this value is nonzero. Equality falls through to a jump to $\gen^{-1}$, outside the committed bytecode cube, so the bytecode bus cannot balance any purported continuation. This proves inequality without an inverse hint. \subsection{Functions}\label{sec:prog-functions} @@ -574,58 +643,108 @@ \subsection{Functions}\label{sec:prog-functions} \subsection{Loops}\label{sec:prog-loops} -A counted loop keeps its counter in the exponent: it starts at $\gen^{\mathrm{lo}}$, advances by one multiplication $i\gets\gen\cdot i$, and exits on reaching $\gen^{\mathrm{hi}}$. The body is lowered as a tail-recursive helper function whose exit test \emph{is} the recursive call's \texttt{JUMP} condition — the \texttt{XOR} $i+\gen^{\mathrm{hi}}$ is nonzero exactly while the loop must continue — so an iteration costs one call plus two instructions, with no is-zero gadget. Loop state is threaded through captured arguments or a heap buffer. +A counted loop keeps its counter in the exponent: it starts at $\gen^{\mathrm{lo}}$, advances by one multiplication $i\gets\gen\cdot i$, and exits on reaching $\gen^{\mathrm{hi}}$. The body is lowered as a tail-recursive helper function whose exit test \emph{is} the recursive call's \texttt{JUMP} condition (the \texttt{XOR} $i+\gen^{\mathrm{hi}}$ is nonzero exactly while the loop must continue), so an iteration costs one call plus two instructions, with no is-zero gadget. Loop state is threaded through captured arguments or a heap buffer. \subsection{Conditionals}\label{sec:prog-conditionals} \texttt{if}/\texttt{else} on a field equality is one \texttt{XOR} plus one conditional \texttt{JUMP}: the taken jump goes to whichever block the test should \emph{not} fall into ($=$ falls into \emph{then}, $\neq$ into \emph{else}), so no negation is ever computed. Intra-function jump targets are bytecode addresses $\gen^{\mathrm{entry}+i}$, backpatched at layout. -A taken \texttt{JUMP} reloads $\fp$ from a cell, and the ISA has no $\fp$-read; a branching function therefore materializes its own $\fp$ once: a \texttt{DEREF} in mode \texttt{fp} writes $\fp$ into a fresh one-cell heap buffer, a \texttt{DEREF} in mode \texttt{cell} copies it back into the frame (2 instructions, amortized; in \texttt{main}, $\fp=\gen^{0}=1$ is already a constant). Bindings made inside a branch are local to it; branches communicate through write-once cells — only one branch executes, so both may write the \emph{same} cell. +A taken \texttt{JUMP} reloads $\fp$ from a cell, and the ISA has no $\fp$-read; a branching function therefore materializes its own $\fp$ once: a \texttt{DEREF} in mode \texttt{fp} writes $\fp$ into a fresh one-cell heap buffer, a \texttt{DEREF} in mode \texttt{cell} copies it back into the frame (2 instructions, amortized; in \texttt{main}, $\fp=\gen^{0}=1$ is already a constant). Bindings made inside a branch are local to it; branches communicate through write-once cells: only one branch executes, so both may write the \emph{same} cell. \subsection{Range checks}\label{sec:prog-range-checks} The range check \emph{in the exponent} proves $x\in\{\gen^{0},\dots,\gen^{k-1}\}$, i.e.\ $\log_{\gen}x None: # Field arithmetic and BLAKE3 ------------------------------------------------- MASK32 = (1 << 32) - 1 -MASK128 = (1 << 128) - 1 -REDUCTION = 0x87 # x^128 = x^7 + x^2 + x + 1 +MASK64 = (1 << 64) - 1 +RING_SWITCH_SOUNDNESS_DEGREE = (1 << 31) + (1 << 15) + (1 << 7) + (1 << 3) + (1 << 1) + 1 + + +def _base_mul(left: int, right: int) -> int: + product = 0 + while right: + if right & 1: + product ^= left + right >>= 1 + left <<= 1 + low, high = product & MASK64, product >> 64 + folded = low ^ high ^ (high << 1) ^ (high << 3) ^ (high << 4) + overflow = folded >> 64 + return ( + (folded & MASK64) + ^ overflow + ^ (overflow << 1) + ^ (overflow << 3) + ^ (overflow << 4) + ) & MASK64 @dataclass(frozen=True, slots=True) -class F128: - """GF(2^128), in the GHASH polynomial basis.""" +class F192: + """The tower field GF(2^192) = GF(2^64)[y]/(y^3 + y + 1).""" - value: int = 0 + c0: int = 0 + c1: int = 0 + c2: int = 0 def __post_init__(self) -> None: - object.__setattr__(self, "value", self.value & MASK128) + object.__setattr__(self, "c0", self.c0 & MASK64) + object.__setattr__(self, "c1", self.c1 & MASK64) + object.__setattr__(self, "c2", self.c2 & MASK64) @classmethod - def new(cls, lo: int, hi: int) -> "F128": - return cls((lo & ((1 << 64) - 1)) | ((hi & ((1 << 64) - 1)) << 64)) + def new(cls, c0: int, c1: int, c2: int = 0) -> "F192": + return cls(c0, c1, c2) @classmethod - def from_bytes(cls, data: bytes) -> "F128": - require(len(data) == 16, "a field element must contain exactly 16 bytes") - return cls(int.from_bytes(data, "little")) - - @property - def lo(self) -> int: - return self.value & ((1 << 64) - 1) - - @property - def hi(self) -> int: - return self.value >> 64 + def from_bytes(cls, data: bytes) -> "F192": + require(len(data) == 24, "a field element must contain exactly 24 bytes") + return cls(*(int.from_bytes(data[offset : offset + 8], "little") for offset in (0, 8, 16))) def to_bytes(self) -> bytes: - return self.value.to_bytes(16, "little") + return b"".join(limb.to_bytes(8, "little") for limb in (self.c0, self.c1, self.c2)) @staticmethod - def _coerce(other: object) -> "F128": - if isinstance(other, F128): + def _coerce(other: object) -> "F192": + if isinstance(other, F192): return other if isinstance(other, int): - return F128(other) - return NotImplemented + return F192(other) + raise TypeError(f"cannot use {type(other).__name__} as a field element") def __int__(self) -> int: - return self.value + return self.c0 | self.c1 << 64 | self.c2 << 128 def __bool__(self) -> bool: - return self.value != 0 + return bool(self.c0 or self.c1 or self.c2) def __eq__(self, other: object) -> bool: + if not isinstance(other, (F192, int)): + return False rhs = self._coerce(other) - return False if rhs is NotImplemented else self.value == rhs.value + return self.c0 == rhs.c0 and self.c1 == rhs.c1 and self.c2 == rhs.c2 def __hash__(self) -> int: - return hash(self.value) + return hash((self.c0, self.c1, self.c2)) - def __add__(self, other: object) -> "F128": + def __add__(self, other: object) -> "F192": rhs = self._coerce(other) - if rhs is NotImplemented: - return NotImplemented - return F128(self.value ^ rhs.value) + return F192(self.c0 ^ rhs.c0, self.c1 ^ rhs.c1, self.c2 ^ rhs.c2) __radd__ = __add__ __sub__ = __add__ __rsub__ = __add__ - def __neg__(self) -> "F128": + def __neg__(self) -> "F192": return self - def __mul__(self, other: object) -> "F128": + def __mul__(self, other: object) -> "F192": rhs = self._coerce(other) - if rhs is NotImplemented: - return NotImplemented - # Four-bit carry-less windows cut the Python loop from 128 iterations - # to 32. Accumulate an unreduced polynomial, then fold its upper half - # with x^128 = x^7 + x^2 + x + 1 (and fold the at-most-seven overflow - # bits once more). - a, b, product = self.value, rhs.value, 0 - multiples = [0] * 16 - for nibble in range(1, 16): - multiples[nibble] = ( - (a if nibble & 1 else 0) - ^ (a << 1 if nibble & 2 else 0) - ^ (a << 2 if nibble & 4 else 0) - ^ (a << 3 if nibble & 8 else 0) - ) - shift = 0 - while b: - product ^= multiples[b & 15] << shift - b >>= 4 - shift += 4 - low, high = product & MASK128, product >> 128 - folded = low ^ high ^ (high << 1) ^ (high << 2) ^ (high << 7) - overflow = folded >> 128 - product = ( - (folded & MASK128) - ^ overflow - ^ (overflow << 1) - ^ (overflow << 2) - ^ (overflow << 7) - ) - return F128(product & MASK128) + p0 = _base_mul(self.c0, rhs.c0) + p1 = _base_mul(self.c0, rhs.c1) ^ _base_mul(self.c1, rhs.c0) + p2 = _base_mul(self.c0, rhs.c2) ^ _base_mul(self.c1, rhs.c1) ^ _base_mul(self.c2, rhs.c0) + p3 = _base_mul(self.c1, rhs.c2) ^ _base_mul(self.c2, rhs.c1) + p4 = _base_mul(self.c2, rhs.c2) + return F192(p0 ^ p3, p1 ^ p3 ^ p4, p2 ^ p4) __rmul__ = __mul__ - def __pow__(self, exponent: int) -> "F128": + def __pow__(self, exponent: int) -> "F192": if exponent < 0: return self.inv() ** -exponent base, out, n = self, ONE, exponent @@ -142,29 +132,25 @@ def __pow__(self, exponent: int) -> "F128": n >>= 1 return out - def inv(self) -> "F128": - require(self.value != 0, "division by zero in GF(2^128)") - return self ** ((1 << 128) - 2) + def inv(self) -> "F192": + require(bool(self), "division by zero in GF(2^192)") + return self ** ((1 << 192) - 2) - def __truediv__(self, other: object) -> "F128": + def __truediv__(self, other: object) -> "F192": rhs = self._coerce(other) - if rhs is NotImplemented: - return NotImplemented return self * rhs.inv() - def __rtruediv__(self, other: object) -> "F128": + def __rtruediv__(self, other: object) -> "F192": lhs = self._coerce(other) - if lhs is NotImplemented: - return NotImplemented return lhs * self.inv() def __repr__(self) -> str: - return f"F128(0x{self.value:032x})" + return f"F192(0x{self.c2:016x}{self.c1:016x}{self.c0:016x})" -ZERO = F128(0) -ONE = F128(1) -GEN = F128(2) +ZERO = F192(0) +ONE = F192(1) +GEN = F192(2) # BLAKE3 -------------------------------------------------------------------- @@ -291,14 +277,14 @@ def blake3_hash(data: bytes) -> bytes: return _output_root(output) -def build_eq(point: Sequence[F128]) -> list[F128]: +def build_eq(point: Sequence[F192]) -> list[F192]: out = [ONE] for r in point: out = [v * (ONE + r) for v in out] + [v * r for v in out] return out -def mle_eval(evals: Sequence[F128], point: Sequence[F128]) -> F128: +def mle_eval(evals: Sequence[F192], point: Sequence[F192]) -> F192: require(len(evals) == 1 << len(point), "multilinear table has the wrong size") cur = list(evals) for r in point: @@ -306,12 +292,12 @@ def mle_eval(evals: Sequence[F128], point: Sequence[F128]) -> F128: return cur[0] # Shared verification helpers ------------------------------------------------- -def interpolate(a: F128, b: F128, point: F128) -> F128: +def interpolate(a: F192, b: F192, point: F192) -> F192: """Evaluate the line through ``a`` and ``b`` at ``point``.""" return a + point * (a + b) -def eq_eval(left: Sequence[F128], right: Sequence[F128]) -> F128: +def eq_eval(left: Sequence[F192], right: Sequence[F192]) -> F192: require(len(left) == len(right), "eq: dimension mismatch") result = ONE for x, y in zip(left, right): @@ -324,8 +310,8 @@ def eq_eval(left: Sequence[F128], right: Sequence[F128]) -> F128: def lagrange_eval( - nodes: Sequence[F128], values: Sequence[F128], point: F128 -) -> F128: + nodes: Sequence[F192], values: Sequence[F192], point: F192 +) -> F192: require(len(nodes) == len(values), "Lagrange data length mismatch") result = ZERO for index, (node, value) in enumerate(zip(nodes, values)): @@ -358,14 +344,27 @@ def take(self, length: int) -> bytes: def u64(self) -> int: return int.from_bytes(self.take(8), "little") - def field(self) -> F128: - return F128.from_bytes(self.take(16)) + def field(self) -> F192: + return F192.from_bytes(self.take(24)) - def fields(self) -> list[F128]: + def base_field(self) -> int: + return self.u64() + + def fields(self) -> list[F192]: length = self.u64() - require(length <= self.remaining // 16, "invalid field-vector length") + require(length <= self.remaining // 24, "invalid field-vector length") return [self.field() for _ in range(length)] + def base_fields(self) -> list[int]: + length = self.u64() + require(length <= self.remaining // 8, "invalid base-field-vector length") + return [self.base_field() for _ in range(length)] + + def hashes(self) -> tuple[bytes, ...]: + length = self.u64() + require(length <= self.remaining // 32, "invalid hash-vector length") + return tuple(self.take(32) for _ in range(length)) + @property def remaining(self) -> int: return len(self.data) - self.offset @@ -375,38 +374,98 @@ def finish(self) -> None: @dataclass(frozen=True) -class LevelOpening: - opened_rows: tuple[tuple[F128, ...], ...] +class InitialOpening: + opened_rows: tuple[tuple[int, ...], ...] + merkle_proof: tuple[bytes, ...] + + @classmethod + def read(cls, reader: BinaryReader) -> "InitialOpening": + row_count = reader.u64() + require(row_count <= reader.remaining // 8, "invalid opened-row count") + rows = tuple(tuple(reader.base_fields()) for _ in range(row_count)) + return cls(rows, reader.hashes()) + + +@dataclass(frozen=True) +class RecursiveOpening: + opened_rows: tuple[tuple[F192, ...], ...] merkle_proof: tuple[bytes, ...] @classmethod - def read(cls, reader: BinaryReader) -> "LevelOpening": + def read(cls, reader: BinaryReader) -> "RecursiveOpening": row_count = reader.u64() require(row_count <= reader.remaining // 8, "invalid opened-row count") rows = tuple(tuple(reader.fields()) for _ in range(row_count)) - path_length = reader.u64() - require(path_length <= reader.remaining // 32, "invalid Merkle proof length") - return cls(rows, tuple(reader.take(32) for _ in range(path_length))) + return cls(rows, reader.hashes()) + + +@dataclass(frozen=True) +class FinalOpening: + residual: tuple[F192, ...] + opened_rows: tuple[tuple[F192, ...], ...] + merkle_proof: tuple[bytes, ...] + + @classmethod + def read(cls, reader: BinaryReader) -> "FinalOpening": + residual = tuple(reader.fields()) + opened = RecursiveOpening.read(reader) + return cls(residual, opened.opened_rows, opened.merkle_proof) + + +@dataclass(frozen=True) +class SumcheckMessage: + constant: F192 + quadratic: F192 + + +@dataclass(frozen=True) +class LigeritoProofData: + initial: InitialOpening + recursive_roots: tuple[bytes, ...] + recursive: tuple[RecursiveOpening, ...] + final: FinalOpening + sumcheck: tuple[SumcheckMessage, ...] + grinding_nonces: tuple[int, ...] + ood_values: tuple[F192, ...] + fold_grinding_nonces: tuple[int, ...] + + @classmethod + def read(cls, reader: BinaryReader) -> "LigeritoProofData": + initial = InitialOpening.read(reader) + roots = reader.hashes() + count = reader.u64() + require(count <= 32, "too many Ligerito levels") + recursive = tuple(RecursiveOpening.read(reader) for _ in range(count)) + final = FinalOpening.read(reader) + message_count = reader.u64() + require(message_count <= reader.remaining // 48, "invalid sumcheck length") + sumcheck = tuple(SumcheckMessage(reader.field(), reader.field()) for _ in range(message_count)) + nonce_count = reader.u64() + require(nonce_count <= reader.remaining // 8, "invalid nonce-vector length") + nonces = tuple(reader.u64() for _ in range(nonce_count)) + ood_values = tuple(reader.fields()) + fold_count = reader.u64() + require(fold_count <= reader.remaining // 8, "invalid fold-nonce length") + fold_nonces = tuple(reader.u64() for _ in range(fold_count)) + return cls(initial, roots, recursive, final, sumcheck, nonces, ood_values, fold_nonces) @dataclass(frozen=True) class LigeritoOpening: - initial: LevelOpening - levels: tuple[LevelOpening, ...] - final: LevelOpening + ring_switches: tuple[tuple[F192, ...], ...] + ligerito: LigeritoProofData @classmethod def read(cls, reader: BinaryReader) -> "LigeritoOpening": - initial = LevelOpening.read(reader) count = reader.u64() - require(count <= 32, "too many Ligerito levels") - levels = tuple(LevelOpening.read(reader) for _ in range(count)) - return cls(initial, levels, LevelOpening.read(reader)) + require(count <= 16, "too many ring-switch proofs") + ring_switches = tuple(tuple(reader.fields()) for _ in range(count)) + return cls(ring_switches, LigeritoProofData.read(reader)) @dataclass(frozen=True) class Proof: - stream: tuple[F128, ...] + stream: tuple[F192, ...] openings: tuple[LigeritoOpening, ...] @classmethod @@ -427,73 +486,78 @@ def load(cls, path: str | Path) -> "Proof": # Fiat--Shamir --------------------------------------------------------------- -DS_SCALAR = F128(1) -DS_BYTE = F128(2) -DS_LEN = F128(3) -DS_SQUEEZE = F128(4) -DS_POW = F128(5) +DS_SCALAR = 1 +DS_BYTE = 2 +DS_LEN = 3 +DS_SQUEEZE = 4 +DS_POW = 5 -def compress(left: Sequence[F128], right: Sequence[F128]) -> tuple[F128, F128]: - require(len(left) == len(right) == 2, "compression operands must contain two fields") - digest = blake3_hash(b"".join(x.to_bytes() for x in (*left, *right))) - return F128.from_bytes(digest[:16]), F128.from_bytes(digest[16:]) +def compress(left: Sequence[int], right: Sequence[int]) -> tuple[int, int, int, int]: + require(len(left) == len(right) == 4, "compression operands must contain four words") + digest = blake3_hash(b"".join(x.to_bytes(8, "little") for x in (*left, *right))) + return tuple(int.from_bytes(digest[offset : offset + 8], "little") for offset in (0, 8, 16, 24)) class Sponge: - def __init__(self, label: bytes, statement: Sequence[F128]): - self.state = (ZERO, ZERO) - self.absorb_bytes(b"leanvm-b/transcript/v1") + def __init__(self, label: bytes, statement: Sequence[F192]): + self.state = (0, 0, 0, 0) + self.absorb_bytes(b"leanvm-b/transcript/v2") self.absorb_bytes(label) for value in statement: self.observe(value) - def observe(self, value: F128) -> None: - self.state = compress(self.state, (value, DS_SCALAR)) + def observe(self, value: F192) -> None: + self.state = compress(self.state, (value.c0, value.c1, value.c2, DS_SCALAR)) def absorb_bytes(self, data: bytes) -> None: - self.state = compress(self.state, (F128(len(data)), DS_LEN)) + self.state = compress(self.state, (len(data), 0, DS_LEN, 0)) for offset in range(0, len(data), 16): - block = F128.from_bytes(data[offset : offset + 16].ljust(16, b"\0")) - self.state = compress(self.state, (block, DS_BYTE)) + block = data[offset : offset + 16].ljust(16, b"\0") + self.state = compress( + self.state, + (int.from_bytes(block[:8], "little"), int.from_bytes(block[8:], "little"), DS_BYTE, 0), + ) - def sample(self) -> F128: - self.state = compress(self.state, (ZERO, DS_SQUEEZE)) - return self.state[0] + def sample(self) -> F192: + self.state = compress(self.state, (0, 0, DS_SQUEEZE, 0)) + return F192(*self.state[:3]) - def check_pow(self, nonce: int, bits: int) -> None: + def check_pow(self, nonce: int | F192, bits: int) -> None: require(0 <= bits < 64, "invalid grinding width") - base = compress(self.state, (ZERO, DS_POW)) - digest = compress(base, (F128(nonce), DS_POW))[0] - valid = nonce == 0 if bits == 0 else digest.lo & ((1 << bits) - 1) == 0 - self.state = compress(self.state, (F128(nonce), DS_POW)) + encoded = nonce if isinstance(nonce, F192) else F192(nonce) + block = (encoded.c0, encoded.c1, encoded.c2, DS_POW) + base = compress(self.state, (0, 0, DS_POW, 0)) + digest = compress(base, block)[0] + valid = encoded == F192(0) if bits == 0 else digest & ((1 << bits) - 1) == 0 + self.state = compress(self.state, block) require(valid, "invalid grinding nonce") class Transcript: - def __init__(self, proof: Proof, label: bytes, statement: Sequence[F128]): + def __init__(self, proof: Proof, label: bytes, statement: Sequence[F192]): self.proof = proof self.sponge = Sponge(label, statement) self.stream_offset = 0 self.opening_offset = 0 - def scalar(self) -> F128: + def scalar(self) -> F192: require(self.stream_offset < len(self.proof.stream), "proof stream exhausted") value = self.proof.stream[self.stream_offset] self.stream_offset += 1 self.sponge.observe(value) return value - def scalars(self, count: int) -> list[F128]: + def scalars(self, count: int) -> list[F192]: return [self.scalar() for _ in range(count)] - def sample(self) -> F128: + def sample(self) -> F192: return self.sponge.sample() - def samples(self, count: int) -> list[F128]: + def samples(self, count: int) -> list[F192]: return [self.sample() for _ in range(count)] - def observe(self, value: F128) -> None: + def observe(self, value: F192) -> None: self.sponge.observe(value) def absorb_bytes(self, data: bytes) -> None: @@ -503,8 +567,7 @@ def grind(self, bits: int) -> None: require(self.stream_offset < len(self.proof.stream), "missing grinding nonce") encoded = self.proof.stream[self.stream_offset] self.stream_offset += 1 - require(encoded.hi == 0, "grinding nonce has a nonzero high limb") - self.sponge.check_pow(encoded.lo, bits) + self.sponge.check_pow(encoded, bits) def opening(self) -> LigeritoOpening: require(self.opening_offset < len(self.proof.openings), "PCS opening missing") @@ -522,20 +585,20 @@ def finish(self) -> None: @dataclass(frozen=True) class ProductTriple: - roots: tuple[F128, F128, F128] - point: tuple[F128, ...] - values: tuple[F128, F128, F128] + roots: tuple[F192, F192, F192] + point: tuple[F192, ...] + values: tuple[F192, F192, F192] def quartic_eval_from_eq( - claim: F128, - equality_point: F128, - difference: F128, - c2: F128, - c3: F128, - c4: F128, - challenge: F128, -) -> F128: + claim: F192, + equality_point: F192, + difference: F192, + c2: F192, + c3: F192, + c4: F192, + challenge: F192, +) -> F192: c0 = claim + equality_point * difference c1 = difference + c2 + c3 + c4 return c0 + challenge * (c1 + challenge * (c2 + challenge * (c3 + challenge * c4))) @@ -545,7 +608,7 @@ def verify_product_triple(depth: int, transcript: Transcript) -> ProductTriple: root_values = transcript.scalars(3) roots = (root_values[0], root_values[1], root_values[2]) combine = transcript.sample() - point: list[F128] = [] + point: list[F192] = [] values = list(roots) layer = depth @@ -565,7 +628,7 @@ def verify_product_triple(depth: int, transcript: Transcript) -> ProductTriple: layer -= 1 continue - round_point: list[F128] = [] + round_point: list[F192] = [] for prior in point[:round_count]: message = transcript.scalars(4) challenge = transcript.sample() @@ -604,11 +667,11 @@ class Coordinate: committed global columns; ``public`` is a dense public multilinear table. """ - constant: F128 | None = None + constant: F192 | None = None column: int | None = None generator_column: int | None = None index: bool = False - public: tuple[F128, ...] | None = None + public: tuple[F192, ...] | None = None def __post_init__(self) -> None: choices = ( @@ -649,7 +712,7 @@ def bus_layout(blocks: Sequence[BusBlock]) -> BusLayout: return BusLayout(depth, tuple(offsets)) -def index_mle(point: Sequence[F128]) -> F128: +def index_mle(point: Sequence[F192]) -> F192: """MLE of ``[1, g, g^2, ...]`` at an LSB-first point.""" result = ONE generator_power = GEN @@ -662,22 +725,22 @@ def index_mle(point: Sequence[F128]) -> F128: @dataclass(frozen=True) class ColumnClaim: column: int - point: tuple[F128, ...] - value: F128 + point: tuple[F192, ...] + value: F192 @dataclass(frozen=True) class BytecodeClaim: - point: tuple[F128, ...] - value: F128 + point: tuple[F192, ...] + value: F192 @dataclass class BusForm: - coefficients: list[F128] - constant: F128 = ZERO + coefficients: list[F192] + constant: F192 = ZERO - def evaluate(self, values: Sequence[F128]) -> F128: + def evaluate(self, values: Sequence[F192]) -> F192: require(len(values) == len(self.coefficients), "bus form width mismatch") return sum( (coefficient * value for coefficient, value in zip(self.coefficients, values)), @@ -688,16 +751,16 @@ def evaluate(self, values: Sequence[F128]) -> F128: def _decompose_bus_side( blocks: Sequence[BusBlock], layout: BusLayout, - point: Sequence[F128], - alpha: F128, - gamma: F128, + point: Sequence[F192], + alpha: F192, + gamma: F192, forms: Sequence[BusForm], claims: list[ColumnClaim], transcript: Transcript, -) -> F128: +) -> F192: require(len(point) == layout.depth, "bus point dimension mismatch") - def committed_value(column: int, low_point: tuple[F128, ...]) -> F128: + def committed_value(column: int, low_point: tuple[F192, ...]) -> F192: for prior in claims: if prior.column == column and prior.point == low_point: return prior.value @@ -712,7 +775,7 @@ def committed_value(column: int, low_point: tuple[F128, ...]) -> F128: high = point[block.log_rows :] selector = layout.offsets[block_index] >> block.log_rows selector_bits = tuple( - F128((selector >> bit) & 1) + F192((selector >> bit) & 1) for bit in range(layout.depth - block.log_rows) ) selector_weight = eq_eval(selector_bits, high) @@ -761,7 +824,7 @@ def committed_value(column: int, low_point: tuple[F128, ...]) -> F128: return result + ONE + selector_sum -def _padding_fingerprint(block: BusBlock, padding: Sequence[F128], alpha: F128) -> F128: +def _padding_fingerprint(block: BusBlock, padding: Sequence[F192], alpha: F192) -> F192: result = ZERO coefficient = ONE for coordinate in block.coordinates: @@ -780,10 +843,10 @@ def _padding_fingerprint(block: BusBlock, padding: Sequence[F128], alpha: F128) def _padding_surplus( blocks: Sequence[BusBlock], - padding: Sequence[F128], - alpha: F128, - gamma: F128, -) -> F128: + padding: Sequence[F192], + alpha: F192, + gamma: F192, +) -> F192: result = ONE for block in blocks: surplus_rows = (1 << block.log_rows) - block.real_rows @@ -793,10 +856,10 @@ def _padding_surplus( def _public_evaluations( - blocks: Sequence[BusBlock], point: Sequence[F128] -) -> tuple[int, list[F128]]: + blocks: Sequence[BusBlock], point: Sequence[F192] +) -> tuple[int, list[F192]]: log_rows = 0 - values: list[F128] = [] + values: list[F192] = [] for block in blocks: for coordinate in block.coordinates: if coordinate.public is not None: @@ -805,7 +868,7 @@ def _public_evaluations( return log_rows, values -def _stack_public_evaluations(values: Sequence[F128], selector_point: Sequence[F128]) -> F128: +def _stack_public_evaluations(values: Sequence[F192], selector_point: Sequence[F192]) -> F192: require(len(values) <= 1 << len(selector_point), "too many public columns") result = ZERO for column, value in enumerate(values): @@ -820,17 +883,17 @@ def _stack_public_evaluations(values: Sequence[F128], selector_point: Sequence[F class BusResult: claims: tuple[ColumnClaim, ...] bytecode_claim: BytecodeClaim - count_root: F128 - point: tuple[F128, ...] + count_root: F192 + point: tuple[F192, ...] forms: tuple[tuple[BusForm, ...], ...] - totals: tuple[F128, F128, F128] + totals: tuple[F192, F192, F192] def verify_bus_balance( push: Sequence[BusBlock], pull: Sequence[BusBlock], count: Sequence[BusBlock], - padding: Sequence[F128], + padding: Sequence[F192], transcript: Transcript, ) -> BusResult: push_layout = bus_layout(push) @@ -839,8 +902,6 @@ def verify_bus_balance( require(push_layout.depth == pull_layout.depth, "push/pull bus depths differ") require(count_layout.depth <= push_layout.depth, "count bus is deeper than push bus") - grinding_bits = max(0, 120 + push_layout.depth + 1 - 128) - transcript.grind(grinding_bits) alpha = transcript.sample() gamma = transcript.sample() padded_count_layout = BusLayout(push_layout.depth, count_layout.offsets) @@ -879,7 +940,7 @@ def verify_bus_balance( public_log_rows, public_values = _public_evaluations(push, product.point) for value in public_values: transcript.observe(value) - selector_point = transcript.samples(3) + selector_point = transcript.samples(4) bytecode_claim = BytecodeClaim( tuple(product.point[:public_log_rows]) + tuple(selector_point), _stack_public_evaluations(public_values, selector_point), @@ -902,16 +963,16 @@ class Air: log_height: int column_count: int constraint_count: int - evaluate: Callable[[Sequence[F128], Sequence[F128]], F128] + evaluate: Callable[[Sequence[F192], Sequence[F192]], F192] @dataclass(frozen=True) class AirClaim: - point: tuple[F128, ...] - evaluations: tuple[F128, ...] + point: tuple[F192, ...] + evaluations: tuple[F192, ...] -def powers(base: F128, count: int) -> list[F128]: +def powers(base: F192, count: int) -> list[F192]: result, current = [], ONE for _ in range(count): result.append(current) @@ -921,9 +982,9 @@ def powers(base: F128, count: int) -> list[F128]: def verify_constraints( airs: Sequence[Air], - eta: F128, - equality_point: Sequence[F128], - target: F128, + eta: F192, + equality_point: Sequence[F192], + target: F192, transcript: Transcript, ) -> list[AirClaim]: depth = max((air.log_height for air in airs), default=0) @@ -966,23 +1027,26 @@ def verify_constraints( # VM statement, layout, and AIR ----------------------------------------------- FAMILY_DIGEST = bytes.fromhex("afed7472c6f771a857599272ff33a4da86b21f2600f057fa0da797d15863eb58") -BASES = (4, 19, 34, 41, 58, 77) -WIDTHS = (15, 15, 7, 17, 19, 32) -CONSTRAINT_COUNTS = (4, 4, 1, 4, 7, 6) -COUNT_COLUMNS = ((11, 12, 13, 14), (11, 12, 13, 14), (5, 6), - (13, 14, 15, 16), (13, 14, 15, 16), - (23, 24, 25, 26, 27, 28, 29, 30, 31)) -BLAKE3_VALUES = (14, 15, 16, 17, 18, 19, 20, 21, 22) -BLAKE3_SLOTS = (5, 6, 7, 8, 2, 3, 0, 1, 9) -BLAKE3_SLOT_BY_VALUE: dict[int, int] = dict(zip(BLAKE3_VALUES, BLAKE3_SLOTS)) -VM_IV = ( - F128.new(0xBB67AE856A09E667, 0xA54FF53A3C6EF372), - F128.new(0x9B05688C510E527F, 0x5BE0CD191F83D9AB), +BASES = (6, 27, 48, 57, 78, 105, 146) +WIDTHS = (21, 21, 9, 21, 27, 41, 14) +CONSTRAINT_COUNTS = (4, 4, 1, 4, 7, 6, 3) +COUNT_COLUMNS = ( + (17, 18, 19, 20), + (17, 18, 19, 20), + (7, 8), + (17, 18, 19, 20), + (19, 20, 21, 22), + (32, 33, 34, 35, 36, 37, 38, 39, 40), + (10, 11, 12, 13), ) +BLAKE3_VALUES = tuple(range(14, 32)) +BLAKE3_SLOTS = (10, 11, 12, 13, 14, 15, 16, 17, 4, 5, 6, 7, 0, 1, 2, 3, 18, 19) +BLAKE3_SLOT_BY_VALUE: dict[int, int] = dict(zip(BLAKE3_VALUES, BLAKE3_SLOTS)) +VM_IV = (0xBB67AE856A09E667, 0xA54FF53A3C6EF372, 0x9B05688C510E527F, 0x5BE0CD191F83D9AB) -def _field(value: Any) -> F128: - if isinstance(value, F128): +def _field(value: Any) -> F192: + if isinstance(value, F192): return value if isinstance(value, int) and not isinstance(value, bool): integer = value @@ -994,22 +1058,22 @@ def _field(value: Any) -> F128: else: integer = None if integer is not None: - require(0 <= integer < 1 << 128, f"field element is out of range: {value!r}") - return F128(integer) - if isinstance(value, (list, tuple)) and len(value) == 2: + require(0 <= integer < 1 << 192, f"field element is out of range: {value!r}") + return F192(integer & MASK64, integer >> 64 & MASK64, integer >> 128) + if isinstance(value, (list, tuple)) and len(value) == 3: try: - low, high = (int(limb) for limb in value) + limbs = tuple(int(limb) for limb in value) except (TypeError, ValueError) as exc: raise VerificationError(f"invalid field limbs: {value!r}") from exc require( - 0 <= low < 1 << 64 and 0 <= high < 1 << 64, + all(0 <= limb < 1 << 64 for limb in limbs), f"field limb is out of range: {value!r}", ) - return F128.new(low, high) + return F192(*limbs) raise VerificationError(f"invalid field element: {value!r}") -def parse_field(value: Any) -> F128: +def parse_field(value: Any) -> F192: """Parse the field-element forms accepted by the statement JSON schema.""" return _field(value) @@ -1031,9 +1095,9 @@ class Operation: def parse(cls, data: dict[str, Any]) -> "Operation": require(isinstance(data, dict), "each program operation must be an object") name = str(data.get("op", "")).lower() - require(name in {"xor", "mul", "set", "deref", "jump", "blake3"}, + require(name in {"xor", "mul", "set", "deref", "jump", "pack64x2", "blake3"}, f"unknown operation {name!r}") - if name in {"xor", "mul"}: + if name in {"xor", "mul", "pack64x2"}: for key in ("a", "b", "c"): _u32(data[key], f"{name}.{key}") elif name == "set": @@ -1074,14 +1138,14 @@ def parse(cls, data: dict[str, Any]) -> "Program": "program length must be a nonzero power of two") return cls(operations) - def digest(self) -> tuple[F128, F128]: - words = [F128.new(len(self.operations), 1)] - tags = {"xor": 0, "mul": 1, "set": 2, "jump": 6, "blake3": 7} + def digest(self) -> tuple[int, int, int, int]: + words = [len(self.operations), 3] + tags = {"xor": 0, "mul": 1, "set": 2, "jump": 6, "blake3": 7, "pack64x2": 9} for operation in self.operations: d = operation.values name = operation.name k = x = y = ZERO - if name in {"xor", "mul"}: + if name in {"xor", "mul", "pack64x2"}: a, b, c = int(d["a"]), int(d["b"]), int(d["c"]) tag = tags[name] elif name == "set": @@ -1096,17 +1160,21 @@ def digest(self) -> tuple[F128, F128]: inputs = [int(v) for v in d["ins"]] a, b, c, tag = inputs[0], inputs[1], inputs[2], 7 k = _field(d["metadata"]) - x = F128.new(inputs[3], int(d["cv"])) - y = F128.new(int(d["out"]), 0) - words.extend((F128.new(a | b << 32, c | tag << 32), k, x, y)) - digest = blake3_hash(b"".join(word.to_bytes() for word in words)) - return F128.from_bytes(digest[:16]), F128.from_bytes(digest[16:]) + x = inputs[3] | int(d["cv"]) << 32 + y = int(d["out"]) + words.extend((a | b << 32, c | tag << 32, k.c0, k.c1, k.c2, int(x), int(y))) + digest = blake3_hash(b"".join(word.to_bytes(8, "little") for word in words)) + return tuple(int.from_bytes(digest[offset : offset + 8], "little") for offset in (0, 8, 16, 24)) - def transcript_statement(self, public_input: Sequence[F128]) -> tuple[F128, ...]: + def transcript_statement(self, public_input: Sequence[F192]) -> tuple[F192, ...]: program_digest = self.digest() - seed = blake3_hash(b"leanvm-b-fs-seed-v1" + FAMILY_DIGEST - + program_digest[0].to_bytes() + program_digest[1].to_bytes()) - return (F128.from_bytes(seed[:16]), F128.from_bytes(seed[16:]), *public_input) + seed = blake3_hash( + b"leanvm-b-fs-seed-v1" + + FAMILY_DIGEST + + b"".join(word.to_bytes(8, "little") for word in program_digest) + ) + words = tuple(int.from_bytes(seed[offset : offset + 8], "little") for offset in (0, 8, 16, 24)) + return (F192(words[0], words[1]), F192(words[2], words[3]), *public_input) @dataclass(frozen=True) @@ -1124,7 +1192,7 @@ class Layout: push: tuple[BusBlock, ...] pull: tuple[BusBlock, ...] count: tuple[BusBlock, ...] - padding: tuple[F128, ...] + padding: tuple[F192, ...] placements: tuple[Placement, ...] stack_log: int table_logs: tuple[int, ...] @@ -1134,11 +1202,11 @@ def _ceil_log(value: int) -> int: return max(0, (value - 1).bit_length()) -def _gpow(index: int) -> F128: +def _gpow(index: int) -> F192: return GEN ** index -def _const(value: F128 | int) -> Coordinate: +def _const(value: F192 | int) -> Coordinate: return Coordinate(constant=_field(value)) @@ -1150,7 +1218,7 @@ def _gcol(index: int) -> Coordinate: return Coordinate(generator_column=index) -def _public(values: Sequence[F128]) -> Coordinate: +def _public(values: Sequence[F192]) -> Coordinate: return Coordinate(public=tuple(values)) @@ -1174,44 +1242,69 @@ def bytecode(self, pc: int, count: int, opcode: int, operands: Sequence[Coordina prefix_pull = (_const(GEN * GEN), _col(pc), _col(count), _const(_gpow(opcode))) self.pair((*prefix_push, *operands), (*prefix_pull, *operands)) - def memory(self, address: int, count: int, value: int, successor: bool = False) -> None: + def memory( + self, address: int, count: int, values: Sequence[Coordinate], successor: bool = False + ) -> None: addr = _gcol(address) if successor else _col(address) - self.pair((_const(GEN), addr, _gcol(count), _col(value)), - (_const(GEN), addr, _col(count), _col(value))) + self.pair( + (_const(GEN), addr, _gcol(count), *values), + (_const(GEN), addr, _col(count), *values), + ) + + def memory_word(self, address: int, count: int, lo: int, hi: int, top: int) -> None: + self.memory(address, count, (_col(lo), _col(hi), _col(top))) + + def memory_base(self, address: int, count: int, value: int) -> None: + self.memory(address, count, (_col(value), _const(ZERO), _const(ZERO))) + + def memory_128(self, address: int, count: int, lo: int, hi: int, successor: bool = False) -> None: + self.memory(address, count, (_col(lo), _col(hi), _const(ZERO)), successor) def _table_flushes(table: int) -> Flushes: f = Flushes() if table in (0, 1): f.state_step(0, 1) - f.bytecode(0, 14, table, (_col(2), _col(3), _col(4), _const(ZERO), _const(ZERO))) - f.memory(5, 11, 8) - f.memory(6, 12, 9) - f.memory(7, 13, 10) + f.bytecode(0, 20, table, (_col(2), _col(3), _col(4), _const(ZERO), _const(ZERO))) + f.memory_word(5, 17, 8, 9, 10) + f.memory_word(6, 18, 11, 12, 13) + f.memory_word(7, 19, 14, 15, 16) elif table == 2: f.state_step(0, 1) - f.bytecode(0, 6, 2, (_col(2), _col(3), _const(ZERO), _const(ZERO), _const(ZERO))) - f.memory(4, 5, 3) + f.bytecode(0, 8, 2, (_col(2), _col(3), _col(4), _col(5), _const(ZERO))) + f.memory_word(6, 7, 3, 4, 5) elif table == 3: f.state_step(0, 1) - f.bytecode(0, 16, 3, (_col(2), _col(3), _col(4), _col(5), _col(6))) - f.memory(7, 13, 10) - f.memory(8, 14, 11) - f.memory(9, 15, 12) + f.bytecode(0, 20, 3, (_col(2), _col(3), _col(4), _col(5), _col(6))) + f.memory_base(7, 17, 10) + f.memory_word(8, 18, 11, 12, 13) + f.memory_word(9, 19, 14, 15, 16) elif table == 4: f.state_jump(0, 1, 2, 3) - f.bytecode(0, 16, 4, (_col(4), _col(5), _col(6), _const(ZERO), _const(ZERO))) - f.memory(7, 13, 10) - f.memory(8, 14, 11) - f.memory(9, 15, 12) - else: + f.bytecode(0, 22, 4, (_col(4), _col(5), _col(6), _const(ZERO), _const(ZERO))) + f.memory_word(7, 19, 10, 11, 12) + f.memory_word(8, 20, 13, 14, 15) + f.memory_word(9, 21, 16, 17, 18) + elif table == 5: f.state_step(0, 1) - f.bytecode(0, 31, 5, tuple(_col(i) for i in (2, 3, 4, 5, 6, 7, 22))) - for address, count, value, successor in ( - (8,23,14,False), (9,24,15,False), (10,25,16,False), (11,26,17,False), - (12,27,20,False), (12,28,21,True), (13,29,18,False), (13,30,19,True), + f.bytecode(0, 40, 5, tuple(_col(i) for i in (2, 3, 4, 5, 6, 7, 30, 31))) + for address, count, lo, hi, successor in ( + (8, 32, 14, 15, False), + (9, 33, 16, 17, False), + (10, 34, 18, 19, False), + (11, 35, 20, 21, False), + (12, 36, 26, 27, False), + (12, 37, 28, 29, True), + (13, 38, 22, 23, False), + (13, 39, 24, 25, True), ): - f.memory(address, count, value, successor) + f.memory_128(address, count, lo, hi, successor) + else: + f.state_step(0, 1) + f.bytecode(0, 13, 6, (_col(2), _col(3), _col(4), _const(ZERO), _const(ZERO))) + f.memory_base(5, 10, 8) + f.memory_base(6, 11, 9) + f.memory_128(7, 12, 8, 9) return f @@ -1223,16 +1316,17 @@ def _offset_coordinate(coordinate: Coordinate, base: int) -> Coordinate: return coordinate -def _program_columns(program: Program) -> tuple[tuple[F128, ...], ...]: - columns = [[] for _ in range(8)] - opcodes = {"xor": 0, "mul": 1, "set": 2, "deref": 3, "jump": 4, "blake3": 5} +def _program_columns(program: Program) -> tuple[tuple[F192, ...], ...]: + columns = [[] for _ in range(9)] + opcodes = {"xor": 0, "mul": 1, "set": 2, "deref": 3, "jump": 4, "blake3": 5, "pack64x2": 6} for operation in program.operations: d, name = operation.values, operation.name - operands = [ZERO] * 7 - if name in {"xor", "mul"}: + operands = [ZERO] * 8 + if name in {"xor", "mul", "pack64x2"}: operands[:3] = [_gpow(int(d[k])) for k in ("a", "b", "c")] elif name == "set": - operands[:2] = [_gpow(int(d["o"])), _field(d["k"])] + immediate = _field(d["k"]) + operands[:4] = [_gpow(int(d["o"])), F192(immediate.c0), F192(immediate.c1), F192(immediate.c2)] elif name == "deref": operands[:3] = [_gpow(int(d[k])) for k in ("alpha", "beta", "gamma")] mode = str(d["mode"]).lower() @@ -1241,8 +1335,14 @@ def _program_columns(program: Program) -> tuple[tuple[F128, ...], ...]: operands[:3] = [_gpow(int(d[k])) for k in ("oc", "od", "of")] else: inputs = [int(v) for v in d["ins"]] - operands = [*(_gpow(v) for v in inputs), _gpow(int(d["cv"])), - _gpow(int(d["out"])), _field(d["metadata"])] + metadata = _field(d["metadata"]) + operands = [ + *(_gpow(v) for v in inputs), + _gpow(int(d["cv"])), + _gpow(int(d["out"])), + F192(metadata.c0), + F192(metadata.c1), + ] row = [_gpow(opcodes[name]), *operands] for column, value in zip(columns, row): column.append(value) @@ -1252,7 +1352,7 @@ def _program_columns(program: Program) -> tuple[tuple[F128, ...], ...]: def build_layout(program: Program, log_memory: int, row_counts: Sequence[int]) -> Layout: require( 16 <= log_memory <= 32 - and len(row_counts) == 6 + and len(row_counts) == 7 and all(0 <= count < 1 << 32 for count in row_counts), "invalid announced table sizes", ) @@ -1265,7 +1365,7 @@ def build_layout(program: Program, log_memory: int, row_counts: Sequence[int]) - BusBlock(0, (_const(ONE), _const(ONE), _const(ONE)), 1), BusBlock( log_memory, - (_const(GEN), Coordinate(index=True), _const(ONE), _col(0)), + (_const(GEN), Coordinate(index=True), _const(ONE), _col(0), _col(1), _col(2)), 1 << log_memory, ), BusBlock( @@ -1283,7 +1383,7 @@ def build_layout(program: Program, log_memory: int, row_counts: Sequence[int]) - BusBlock(0, (_const(ONE), _const(_gpow(len(program.operations) - 1)), _const(ONE)), 1), BusBlock( log_memory, - (_const(GEN), Coordinate(index=True), _col(1), _col(0)), + (_const(GEN), Coordinate(index=True), _col(3), _col(0), _col(1), _col(2)), 1 << log_memory, ), BusBlock( @@ -1291,14 +1391,14 @@ def build_layout(program: Program, log_memory: int, row_counts: Sequence[int]) - ( _const(GEN * GEN), Coordinate(index=True), - _col(2), + _col(4), *(_public(column) for column in public_columns), ), len(program.operations), ), ] count: list[BusBlock] = [] - padding = [ZERO] * (4 + sum(WIDTHS)) + padding = [ZERO] * (6 + sum(WIDTHS)) for table, (base, height, real) in enumerate(zip(BASES, table_logs, row_counts)): flushes = _table_flushes(table) for coordinates in flushes.push: @@ -1313,15 +1413,17 @@ def build_layout(program: Program, log_memory: int, row_counts: Sequence[int]) - zero_digest = blake3_hash(bytes(64)) b3 = BASES[5] - padding[b3 + 18] = F128.from_bytes(zero_digest[:16]) - padding[b3 + 19] = F128.from_bytes(zero_digest[16:]) - padding[b3 + 20], padding[b3 + 21] = VM_IV - padding[b3 + 22] = F128.new(0, 64 | 11 << 32) - - kappas: list[int | None] = [0] * (4 + sum(WIDTHS)) - kappas[0] = kappas[1] = log_memory - kappas[2] = bytecode_log - kappas[3] = table_logs[5] + 7 + digest_words = [int.from_bytes(zero_digest[offset : offset + 8], "little") for offset in (0, 8, 16, 24)] + for index, value in enumerate(digest_words): + padding[b3 + 22 + index] = F192(value) + padding[b3 + 26 + index] = F192(VM_IV[index]) + padding[b3 + 30] = ZERO + padding[b3 + 31] = F192(64 | 11 << 32) + + kappas: list[int | None] = [0] * (6 + sum(WIDTHS)) + kappas[0] = kappas[1] = kappas[2] = kappas[3] = log_memory + kappas[4] = bytecode_log + kappas[5] = table_logs[5] + 8 for table, (base, width) in enumerate(zip(BASES, WIDTHS)): kappas[base : base + width] = [table_logs[table]] * width for local in BLAKE3_VALUES: @@ -1347,52 +1449,59 @@ def build_layout(program: Program, log_memory: int, row_counts: Sequence[int]) - def _air_evaluator( table: int, forms: Sequence[BusForm], - form_powers: Sequence[F128], -) -> Callable[[Sequence[F128], Sequence[F128]], F128]: - def evaluate(weights: Sequence[F128], columns: Sequence[F128]) -> F128: - def value(column: int) -> F128: + form_powers: Sequence[F192], +) -> Callable[[Sequence[F192], Sequence[F192]], F192]: + def evaluate(weights: Sequence[F192], columns: Sequence[F192]) -> F192: + def value(column: int) -> F192: return columns[column] + def word(lo: int, hi: int, top: int) -> F192: + return value(lo) + F192(0, 1) * (value(hi) + F192(0, 1) * value(top)) + if table in (0, 1): - operation = ( - value(8) + value(9) if table == 0 else value(8) * value(9) - ) + va, vb, vc = word(8, 9, 10), word(11, 12, 13), word(14, 15, 16) + operation = va + vb if table == 0 else va * vb terms = ( value(5) + value(1) * value(2), value(6) + value(1) * value(3), value(7) + value(1) * value(4), - value(10) + operation, + vc + operation, ) elif table == 2: - terms = (value(4) + value(1) * value(2),) + terms = (value(6) + value(1) * value(2),) elif table == 3: - source = ( - (ONE + value(5) + value(6)) * value(12) - + value(5) * GEN * GEN * value(0) - + value(6) * value(1) - ) + v2, v3 = word(11, 12, 13), word(14, 15, 16) + source = (ONE + value(5) + value(6)) * v3 + value(5) * GEN * GEN * value(0) + value(6) * value(1) terms = ( value(7) + value(1) * value(2), value(8) + value(10) * value(3), value(9) + value(1) * value(4), - value(11) + source, + v2 + source, ) elif table == 4: - flag, condition = value(18), value(10) + condition = word(10, 11, 12) + destination = word(13, 14, 15) + frame = word(16, 17, 18) + inverse, flag = word(23, 24, 25), value(26) terms = ( value(7) + value(1) * value(4), value(8) + value(1) * value(5), value(9) + value(1) * value(6), - flag + condition * value(17), + flag + condition * inverse, condition * (flag + ONE), - value(2) + flag * value(11) + (flag + ONE) * GEN * value(0), - value(3) + flag * value(12) + (flag + ONE) * value(1), + value(2) + flag * destination + (flag + ONE) * GEN * value(0), + value(3) + flag * frame + (flag + ONE) * value(1), ) - else: + elif table == 5: terms = tuple( value(address) + value(1) * value(operand) for address, operand in zip((8, 9, 10, 11, 12, 13), (2, 3, 4, 5, 6, 7)) ) + else: + terms = tuple( + value(address) + value(1) * value(operand) + for address, operand in zip((5, 6, 7), (2, 3, 4)) + ) require(len(weights) == len(terms), "AIR constraint weight mismatch") identities = sum((weight * term for weight, term in zip(weights, terms)), ZERO) @@ -1408,7 +1517,7 @@ def value(column: int) -> F128: def build_airs( layout: Layout, bus_forms: Sequence[Sequence[BusForm]], - form_powers: Sequence[F128], + form_powers: Sequence[F192], ) -> list[Air]: return [ Air( @@ -1436,6 +1545,17 @@ def virtual_slot(column: int) -> int | None: # Ligerito opening ------------------------------------------------------------ +# Ligerito ladder geometry. These mirror the Rust source of truth in +# crates/pcs/src/ligerito_config.rs and must stay in sync with it: the prover +# derives its opening shape from those constants, so a mismatch here rejects a +# valid proof. Change a factor there, change it here. +INITIAL_FOLDING_FACTOR = 6 +SUBSEQUENT_FOLDING_FACTOR = 3 +RS_DOMAIN_INITIAL_REDUCTION_FACTOR = 3 +RS_DOMAIN_SUBSEQUENT_REDUCTION_FACTOR = 1 +RESIDUAL_MAX_LOG = 5 + + @dataclass(frozen=True) class LigeritoConfig: rates: tuple[int, ...] @@ -1443,74 +1563,111 @@ class LigeritoConfig: queries: tuple[int, ...] query_grinding: tuple[int, ...] fold_grinding: tuple[int, ...] + ood_samples: tuple[int, ...] -def _per_query_bits(rate: int, message_log: int) -> float: - rho = 2.0 ** -rate - delta = 1.0 - rho - codeword_length = 2.0 ** (message_log + rate) - radius = delta / 2.0 - 3.0 / (delta * codeword_length) - return log2(1.0 / (1.0 - radius)) +def _reduced_rate(rate: int, message_log: int) -> float: + return ((2.0 ** message_log) - 1.0) / (2.0 ** (message_log + rate)) -def derive_config(log_n: int) -> LigeritoConfig: - """Reproduce ``LigeritoSecurityConfig::derive_config(log_n + 7)``.""" - require(log_n > 6, "Ligerito input is too small") - message_logs = [log_n - 6] - folds = [6] - rates = [1] - - # The feasibility pass in Rust uses the asymptotic UDR query bound. - def feasible_queries(rate: int) -> int: - rho = 2.0 ** -rate - bits = log2(1.0 / (1.0 - (1.0 - rho) / 2.0)) - return ceil(107.0 / bits) - - require(1 << (message_logs[0] + rates[0]) >= feasible_queries(1), - "Ligerito level zero cannot hold its query set") +def _johnson_parameters(rate: int, message_log: int, interleaved_log: int) -> tuple[int, int]: + rho = _reduced_rate(rate, message_log) + root_rho = sqrt(rho) + block_length = 1 << (message_log + rate) + variables = message_log + interleaved_log + best: tuple[int, int] | None = None + for theorem_m in range(3, 4097): + eta = root_rho / theorem_m + while ceil(root_rho / eta) > theorem_m: + eta = nextafter(eta, float("inf")) + if eta >= 1.0 - root_rho: + continue + gamma = 1.0 - root_rho - eta + half = theorem_m + 0.5 + a = ((2.0 * half ** 5 + 3.0 * half * gamma * rho) + / (3.0 * rho ** 1.5) * block_length + half / root_rho) + proximity_bits = 192.0 - log2(a) - max(0, interleaved_log - 1) + if proximity_bits + 1e-12 < 128.0: + break + per_query = log2(1.0 / (1.0 - gamma)) + if not isfinite(per_query) or per_query <= 0.0: + continue + queries = ceil(111.0 / per_query) + if queries > block_length: + continue + list_log = log2(1.0 / (2.0 * eta * root_rho)) + ood = 0 if interleaved_log == INITIAL_FOLDING_FACTOR else next( + (count for count in range(1, 9) + if count * (192.0 - log2(variables)) - (2.0 * list_log - 1.0) + 1e-12 >= 128.0), + 0, + ) + ood_bits = (192.0 - list_log - log2(variables) if ood == 0 + else ood * (192.0 - log2(variables)) - (2.0 * list_log - 1.0)) + algebraic_bits = 192.0 - log2(max(RING_SWITCH_SOUNDNESS_DEGREE, ceil(log2(queries)), 2)) - list_log + if ood_bits + 1e-12 < 128.0 or algebraic_bits + 1e-12 < 128.0: + continue + candidate = (queries, ood) + if best is None or queries < best[0]: + best = candidate + require(best is not None, "no secure Ligerito configuration") + return best + + +def derive_config(log_n: int, initial_rate: int) -> LigeritoConfig: + """Derive the production Johnson/OOD ladder used by the Rust PCS.""" + require(log_n > INITIAL_FOLDING_FACTOR and 1 <= initial_rate <= 4, "invalid Ligerito shape") + folds = [INITIAL_FOLDING_FACTOR] + message_logs = [log_n - INITIAL_FOLDING_FACTOR] + rates = [initial_rate] remaining = message_logs[0] - while remaining > 5: - fold = min(3, remaining) - next_message_log = remaining - fold - rate = rates[-1] + 1 - while 1 << (next_message_log + rate) < feasible_queries(rate): - rate += 1 - require(rate <= 20, "no feasible Ligerito rate") + prior_fold = INITIAL_FOLDING_FACTOR + reduction = RS_DOMAIN_INITIAL_REDUCTION_FACTOR + while remaining > RESIDUAL_MAX_LOG: + fold = min(SUBSEQUENT_FOLDING_FACTOR, remaining) + rates.append(rates[-1] + prior_fold - reduction) + remaining -= fold folds.append(fold) - message_logs.append(next_message_log) - rates.append(rate) - remaining = next_message_log - + message_logs.append(remaining) + prior_fold = fold + reduction = RS_DOMAIN_SUBSEQUENT_REDUCTION_FACTOR require(len(folds) >= 2, "Ligerito requires at least two levels") - queries = tuple(ceil(102.0 / _per_query_bits(r, n)) for r, n in zip(rates, message_logs)) - fold_grinding = [] - for rate, n in zip(rates, message_logs): - rho = 2.0 ** -rate - delta = 1.0 - rho - codeword_length = 2.0 ** (n + rate) - radius = delta / 2.0 - 3.0 / (delta * codeword_length) - exceptional_log = log2(radius * codeword_length + 1.0) - fold_grinding.append(max(0, ceil(120.0 - (128.0 - exceptional_log)))) - return LigeritoConfig(tuple(rates), tuple(folds), queries, - (18,) * len(folds), tuple(fold_grinding)) + parameters = tuple( + _johnson_parameters(rate, columns, fold) + for rate, columns, fold in zip(rates, message_logs, folds) + ) + return LigeritoConfig( + tuple(rates), + tuple(folds), + tuple(value[0] for value in parameters), + (17,) * len(folds), + (0,) * len(folds), + tuple(value[1] for value in parameters), + ) def _hash_pair(left: bytes, right: bytes) -> bytes: return blake3_hash(left + right) -def _row_hash(row: Sequence[F128]) -> bytes: - return blake3_hash(b"".join(value.to_bytes() for value in row)) +FieldValue = F192 | int + + +def _row_hash(row: Sequence[FieldValue], base_field: bool) -> bytes: + if base_field: + return blake3_hash(b"".join(int(value).to_bytes(8, "little") for value in row)) + require(all(isinstance(value, F192) for value in row), "non-field value in extension row") + return blake3_hash(b"".join(value.to_bytes() for value in row if isinstance(value, F192))) def authenticate_rows( root: bytes, leaf_count: int, queries: Sequence[int], - rows: Sequence[Sequence[F128]], + rows: Sequence[Sequence[FieldValue]], row_width: int, octopus: Sequence[bytes], -) -> list[Sequence[F128]]: + base_field: bool = False, +) -> list[Sequence[FieldValue]]: """Authenticate a compressed multiproof and restore transcript row order.""" require(leaf_count > 0 and leaf_count & (leaf_count - 1) == 0, "invalid Merkle leaf count") @@ -1520,7 +1677,7 @@ def authenticate_rows( require(all(0 <= q < leaf_count for q in unique), "Merkle query is out of range") require(all(len(row) == row_width for row in rows), "opened row has the wrong width") - nodes = [(index, _row_hash(row)) for index, row in zip(unique, rows)] + nodes = [(index, _row_hash(row, base_field)) for index, row in zip(unique, rows)] supplied = iter(octopus) for _ in range(leaf_count.bit_length() - 1): parents: list[tuple[int, bytes]] = [] @@ -1553,7 +1710,7 @@ def authenticate_rows( def sample_queries(sponge: Sponge, block_length: int, count: int) -> list[int]: depth = block_length.bit_length() - 1 require(block_length == 1 << depth and depth > 0, "invalid query domain") - per_word = 128 // depth + per_word = 192 // depth result: list[int] = [] while len(result) < count: bits = int(sponge.sample()) @@ -1564,19 +1721,14 @@ def sample_queries(sponge: Sponge, block_length: int, count: int) -> list[int]: @dataclass(frozen=True) class QuadraticMessage: - constant: F128 - linear: F128 - quadratic: F128 - - @classmethod - def read(cls, transcript: Transcript, target: F128) -> "QuadraticMessage": - constant, quadratic = transcript.scalars(2) - return cls(constant, target + quadratic, quadratic) + constant: F192 + linear: F192 + quadratic: F192 - def evaluate(self, point: F128) -> F128: + def evaluate(self, point: F192) -> F192: return self.constant + point * self.linear + point * point * self.quadratic - def add_scaled(self, other: "QuadraticMessage", scale: F128) -> "QuadraticMessage": + def add_scaled(self, other: "QuadraticMessage", scale: F192) -> "QuadraticMessage": return QuadraticMessage( self.constant + scale * other.constant, self.linear + scale * other.linear, @@ -1585,23 +1737,26 @@ def add_scaled(self, other: "QuadraticMessage", scale: F128) -> "QuadraticMessag def _enforced_sum( - rows: Sequence[Sequence[F128]], - folds: Sequence[F128], - alpha: Sequence[F128], -) -> F128: + rows: Sequence[Sequence[FieldValue]], + folds: Sequence[F192], + alpha: Sequence[F192], +) -> F192: lane_weights = build_eq(folds) query_weights = build_eq(alpha)[: len(rows)] total = ZERO for query_weight, row in zip(query_weights, rows): require(len(row) == len(lane_weights), "Ligerito row/fold width mismatch") - total += query_weight * sum((x * y for x, y in zip(row, lane_weights)), ZERO) + total += query_weight * sum( + ((F192(x) if isinstance(x, int) else x) * y for x, y in zip(row, lane_weights)), + ZERO, + ) return total -def _subspace_roots(log_n: int) -> list[F128]: +def _subspace_roots(log_n: int) -> list[F192]: roots = [ZERO] * (log_n + 1) roots[0] = ONE - layer = [F128(1 << i) for i in range(1, log_n + 1)] + layer = [F192(1 << i) for i in range(1, log_n + 1)] for level in range(log_n): for index in range(log_n - level): value = layer[index] * layer[index] + roots[level] * layer[index] @@ -1615,18 +1770,18 @@ def _subspace_roots(log_n: int) -> list[F128]: def _induced_residual( message_log: int, queries: Sequence[int], - alpha: Sequence[F128], - prefix: Sequence[F128], + alpha: Sequence[F192], + prefix: Sequence[F192], residual_log: int, -) -> list[F128]: +) -> list[F192]: require(len(prefix) + residual_log == message_log, "bad induced-basis dimensions") roots = _subspace_roots(message_log) inverses = [value.inv() if value else ZERO for value in roots] query_weights = build_eq(alpha)[: len(queries)] - prepared: list[tuple[F128, tuple[F128, ...]]] = [] + prepared: list[tuple[F192, tuple[F192, ...]]] = [] for query in queries: - normalized: list[F128] = [] - current = F128(query) + normalized: list[F192] = [] + current = F192(query) for coordinate in range(message_log): normalized.append(current * inverses[coordinate]) current = current * current + roots[coordinate] * current @@ -1635,7 +1790,7 @@ def _induced_residual( fixed *= ONE + challenge * (ONE + basis_value) prepared.append((fixed, tuple(normalized[len(prefix) :]))) - result: list[F128] = [] + result: list[F192] = [] for vertex in range(1 << residual_log): value = ZERO for query_weight, (fixed, tail) in zip(query_weights, prepared): @@ -1650,104 +1805,136 @@ def _induced_residual( def verify_ligerito( transcript: Transcript, - opening: LigeritoOpening, + proof: LigeritoProofData, log_n: int, - target: F128, + initial_rate: int, + target: F192, root: bytes, - evaluate_basis: Callable[[Sequence[F128], int], Sequence[F128]], + evaluate_basis: Callable[[Sequence[F192], int], Sequence[F192]], ) -> None: - """Verify one succinct multilevel opening and consume its transcript data.""" - config = derive_config(log_n) - level_count = len(config.folds) - require(len(opening.levels) == max(0, level_count - 2), "wrong Ligerito level count") - transcript.observe(target) - transcript.absorb_bytes(root) + """Verify the base-field multilevel opening with a one-point terminal check.""" + config = derive_config(log_n, initial_rate) + levels = len(config.folds) + require(len(proof.recursive_roots) == levels - 1, "wrong Ligerito root count") + require(len(proof.recursive) == levels - 2, "wrong Ligerito recursive-proof count") + require(len(proof.grinding_nonces) == levels, "wrong Ligerito nonce count") + + def observe_root(value: bytes) -> None: + require(len(value) == 32, "invalid Merkle root") + transcript.observe(F192.from_bytes(value[:24])) + transcript.observe(F192(int.from_bytes(value[24:], "little"))) + + message_index = 0 + + def next_quad(claim: F192) -> QuadraticMessage: + nonlocal message_index + require(message_index < len(proof.sumcheck), "truncated Ligerito sumcheck") + message = proof.sumcheck[message_index] + message_index += 1 + transcript.observe(message.constant) + transcript.observe(message.quadratic) + return QuadraticMessage(message.constant, claim + message.quadratic, message.quadratic) + transcript.observe(target) + observe_root(root) running_target = target - running_quad = QuadraticMessage.read(transcript, running_target) - all_folds: list[F128] = [] - level_contexts: list[tuple[int, list[int], list[F128], int, F128]] = [] - previous_root: bytes | None = None - previous_message_log = 0 - previous_rate = 0 - previous_fold = 0 - middle_index = 0 - - for level in range(level_count): - fold_values: list[F128] = [] - for fold_index in range(config.folds[level]): + running_quad = next_quad(target) + folds: list[F192] = [] + contexts: list[tuple[int, list[int], list[F192], int, F192]] = [] + ood_contexts: list[tuple[tuple[F192, ...], int, F192]] = [] + fold_nonce_index = 0 + ood_index = 0 + current_root = root + + for level, (fold_count, rate) in enumerate(zip(config.folds, config.rates)): + level_folds: list[F192] = [] + for fold_index in range(fold_count): bits = max(0, config.fold_grinding[level] - fold_index) if bits: - transcript.grind(bits) + require(fold_nonce_index < len(proof.fold_grinding_nonces), + "missing Ligerito fold nonce") + transcript.sponge.check_pow(proof.fold_grinding_nonces[fold_nonce_index], bits) + fold_nonce_index += 1 challenge = transcript.sample() - all_folds.append(challenge) - fold_values.append(challenge) + folds.append(challenge) + level_folds.append(challenge) running_target = running_quad.evaluate(challenge) - running_quad = QuadraticMessage.read(transcript, running_target) - - message_log = log_n - len(all_folds) - if level == 0: - opened = opening.initial - authentication_root = root - opened_message_log = message_log - opened_rate = config.rates[0] - opened_width = 1 << config.folds[0] - else: - opened = opening.final if level == level_count - 1 else opening.levels[middle_index] - if level < level_count - 1: - middle_index += 1 - if previous_root is None: - raise VerificationError("missing preceding Ligerito commitment") - authentication_root = previous_root - opened_message_log = previous_message_log - opened_rate = previous_rate - opened_width = 1 << previous_fold - - if level == level_count - 1: - residual = transcript.scalars(1 << message_log) + running_quad = next_quad(running_target) + + message_log = log_n - len(folds) + final_level = level == levels - 1 + if final_level: + residual = proof.final.residual + require(len(residual) == 1 << message_log, "wrong Ligerito residual length") + for value in residual: + transcript.observe(value) else: - root_words = transcript.scalars(2) - next_root = root_words[0].to_bytes() + root_words[1].to_bytes() - - transcript.grind(config.query_grinding[level]) - block_length = 1 << (opened_message_log + opened_rate) + next_root = proof.recursive_roots[level] + observe_root(next_root) + for _ in range(config.ood_samples[level + 1]): + point = tuple(transcript.samples(message_log)) + require(ood_index < len(proof.ood_values), "missing Ligerito OOD value") + value = proof.ood_values[ood_index] + ood_index += 1 + transcript.observe(value) + intro = next_quad(value) + beta = transcript.sample() + running_quad = running_quad.add_scaled(intro, beta) + running_target += beta * value + ood_contexts.append((point, len(folds), beta)) + + transcript.sponge.check_pow(proof.grinding_nonces[level], config.query_grinding[level]) + block_length = 1 << (message_log + rate) queries = sample_queries(transcript.sponge, block_length, config.queries[level]) alpha = transcript.samples(max(0, (len(queries) - 1).bit_length())) + if level == 0: + opened = proof.initial + elif final_level: + opened = proof.final + else: + opened = proof.recursive[level - 1] try: rows = authenticate_rows( - authentication_root, + current_root, block_length, queries, opened.opened_rows, - opened_width, + 1 << fold_count, opened.merkle_proof, + level == 0, ) except VerificationError as exc: raise VerificationError(f"Ligerito level {level}: {exc}") from exc - enforced = _enforced_sum(rows, fold_values, alpha) - - if level == level_count - 1: - # Same intro/glue shape as every other level, then finish the - # sumcheck over the tail so the opening closes on ONE evaluation of - # the basis rather than 2^message_log of them. - intro = QuadraticMessage.read(transcript, enforced) - beta = transcript.sample() - running_quad = running_quad.add_scaled(intro, beta) - running_target += beta * enforced - level_contexts.append((message_log, queries, alpha, len(all_folds), beta)) - tail_folds: list[F128] = [] + enforced = _enforced_sum(rows, level_folds, alpha) + + # Every commitment, including the last one, enters through an intro + # message before its separation challenge. + intro = next_quad(enforced) + beta = transcript.sample() + running_quad = running_quad.add_scaled(intro, beta) + running_target += beta * enforced + contexts.append((message_log, queries, alpha, len(folds), beta)) + + if final_level: + # Finish the remaining sumcheck rounds and close on one evaluation + # of every basis at the resulting point. + tail_folds: list[F192] = [] for round_index in range(message_log): challenge = transcript.sample() running_target = running_quad.evaluate(challenge) tail_folds.append(challenge) if round_index + 1 < message_log: - running_quad = QuadraticMessage.read(transcript, running_target) - # A residual width of zero turns each residual evaluator into a - # point evaluator, so no new closed form is needed. - weight = evaluate_basis(list(all_folds) + tail_folds, 0)[0] - for context_log, context_queries, context_alpha, start, context_beta in level_contexts: - prefix_length = context_log - message_log - point = list(all_folds[start : start + prefix_length]) + tail_folds + running_quad = next_quad(running_target) + require(message_index == len(proof.sumcheck), "trailing Ligerito sumcheck messages") + require(ood_index == len(proof.ood_values), "trailing Ligerito OOD values") + require(fold_nonce_index == len(proof.fold_grinding_nonces), + "trailing Ligerito fold nonces") + weight_values = list(evaluate_basis(list(folds) + tail_folds, 0)) + require(len(weight_values) == 1, "basis point evaluation has the wrong length") + weight = weight_values[0] + for context_log, context_queries, context_alpha, start, separation in contexts: + fixed = context_log - message_log + point = list(folds[start : start + fixed]) + tail_folds induced = _induced_residual( context_log, context_queries, @@ -1755,45 +1942,56 @@ def verify_ligerito( point, 0, ) - weight += context_beta * induced[0] + require(len(induced) == 1, "induced point evaluation has the wrong length") + weight += separation * induced[0] + for point, start, separation in ood_contexts: + fixed = len(point) - message_log + scale = separation + for expected, actual in zip(point[:fixed], folds[start : start + fixed]): + scale *= ONE + expected + actual + for expected, actual in zip(point[fixed:], tail_folds): + scale *= ONE + expected + actual + weight += scale terminal = weight * mle_eval(residual, tail_folds) require(terminal == running_target, "Ligerito terminal check failed") return - - intro = QuadraticMessage.read(transcript, enforced) - beta = transcript.sample() - running_quad = running_quad.add_scaled(intro, beta) - running_target += beta * enforced - level_contexts.append((message_log, queries, alpha, len(all_folds), beta)) - previous_root = next_root - previous_message_log = message_log - config.folds[level + 1] - previous_rate = config.rates[level + 1] - previous_fold = config.folds[level + 1] + current_root = next_root raise VerificationError("Ligerito verification ended without a terminal level") # Flock reduction ------------------------------------------------------------- PHI_BASIS = ( - F128.new(0x0000000000000001, 0x0000000000000000), - F128.new(0x6B8330483C2E9849, 0x0DCB364640A222FE), - F128.new(0x7573DA4A5F7710ED, 0x3D5BD35C94646A24), - F128.new(0x41A12DB1F974F3AC, 0x6D58C4E181F9199F), - F128.new(0x5E2F716F4EDE412F, 0xA72EC17764D7CED5), - F128.new(0x5CB10FBABCF00118, 0x4D52354A3A3D8C86), - F128.new(0x95ED1F57F3632D4D, 0x553E92E8BC0AE9A7), - F128.new(0x512625B1F09FA87E, 0x93252331BF042B11), + F192(0x0000000000000001), + F192(0x033CE8BEDDC8A656), + F192(0x512620375ED2A108), + F192(0x0C9E636090AAFC01), + F192(0xBA4F3CD82801769C), + F192(0xBA26E7904ADB4A47), + F192(0x467698598926DC01), + F192(0x4418AE808B28BDD0), ) PHI = tuple(sum((PHI_BASIS[bit] for bit in range(8) if value >> bit & 1), ZERO) for value in range(256)) +_MEDIUM_GENERATOR = F192.new( + 0x243F6A8885A308D3, + 0x13198A2E03707344, + 0xA4093822299F31D0, +) +_MEDIUM_POWERS = ( + _MEDIUM_GENERATOR, + _MEDIUM_GENERATOR ** 2, + _MEDIUM_GENERATOR ** 4, + _MEDIUM_GENERATOR ** 8, +) FIXED_CHALLENGES = ( PHI[0xF7], PHI[0x53], PHI[0xB5], - F128(2) / F128(3), F128(4) / F128(5), F128(16) / F128(17), F128(256) / F128(257), + *tuple(value / (ONE + value) for value in _MEDIUM_POWERS), ) @lru_cache(maxsize=None) -def _denominators(nodes: tuple[F128, ...]) -> tuple[F128, ...]: +def _denominators(nodes: tuple[F192, ...]) -> tuple[F192, ...]: result = [] for index, node in enumerate(nodes): denominator = ONE @@ -1804,7 +2002,7 @@ def _denominators(nodes: tuple[F128, ...]) -> tuple[F128, ...]: return tuple(result) -def lagrange_weights(nodes: Sequence[F128], point: F128) -> list[F128]: +def lagrange_weights(nodes: Sequence[F192], point: F192) -> list[F192]: fixed_nodes = tuple(nodes) differences = [point + node for node in fixed_nodes] prefix = [ONE] @@ -1819,12 +2017,12 @@ def lagrange_weights(nodes: Sequence[F128], point: F128) -> list[F128]: return result -def lagrange_interpolate(nodes: Sequence[F128], values: Sequence[F128], point: F128) -> F128: +def lagrange_interpolate(nodes: Sequence[F192], values: Sequence[F192], point: F192) -> F192: weights = lagrange_weights(nodes, point) return sum((weight * value for weight, value in zip(weights, values)), ZERO) -def quirky_weights(skip_point: F128, rest: Sequence[F128]) -> list[F128]: +def quirky_weights(skip_point: F192, rest: Sequence[F192]) -> list[F192]: skip = lagrange_weights(PHI[:64], skip_point) tail = build_eq(rest) return [a * b for b in tail for a in skip] @@ -1832,29 +2030,29 @@ def quirky_weights(skip_point: F128, rest: Sequence[F128]) -> list[F128]: @dataclass(frozen=True) class QuirkyPoint: - skip: F128 - inner: tuple[F128, ...] - outer: tuple[F128, ...] + skip: F192 + inner: tuple[F192, ...] + outer: tuple[F192, ...] @property - def ring_tail(self) -> tuple[F128, ...]: + def ring_tail(self) -> tuple[F192, ...]: return self.inner + self.outer @dataclass(frozen=True) class ZClaim: point: QuirkyPoint - value: F128 + value: F192 @dataclass(frozen=True) class ZerocheckResult: - skip: F128 - rounds: tuple[F128, ...] - equality_tail: tuple[F128, ...] - a: F128 - b: F128 - c: F128 + skip: F192 + rounds: tuple[F192, ...] + equality_tail: tuple[F192, ...] + a: F192 + b: F192 + c: F192 def verify_zerocheck(log_n: int, transcript: Transcript) -> ZerocheckResult: @@ -1887,7 +2085,7 @@ def verify_zerocheck(log_n: int, transcript: Transcript) -> ZerocheckResult: @dataclass(frozen=True) class LincheckResult: point: QuirkyPoint - value: F128 + value: F192 @dataclass(frozen=True) @@ -1896,35 +2094,70 @@ class Reduction: c: ZClaim -def _claim_weights(point: QuirkyPoint) -> list[F128]: - low = lagrange_weights(PHI[:64], point.skip) - low_scale, high_scale = ONE + point.ring_tail[0], point.ring_tail[0] - return [value * low_scale for value in low] + [value * high_scale for value in low] +def _claim_weights(point: QuirkyPoint) -> list[F192]: + return lagrange_weights(PHI[:64], point.skip) + +RING_MAP_SHIFTS = (32, 16, 8, 4, 2, 1) -def _transpose(values: Sequence[F128]) -> list[F128]: - require(len(values) == 128, "ring-switch slice has the wrong length") - output = [0] * 128 + +def _coordinate_weights(challenges: Sequence[F192]) -> list[F192]: + """`ring_switch::build_coordinate_weights`: the images `Phi(basis_w)` of the + F2-coordinate basis under the six composed two-term linearized maps. The + verifier weights the transposed columns with these; the guest applies the + same composition directly.""" + require(len(challenges) == len(RING_MAP_SHIFTS), "wrong ring-map challenge count") + weights = [] + for w in range(192): + # b_w has only bit w set: limb w // 64, bit w % 64. + limbs = [0, 0, 0] + limbs[w // 64] = 1 << (w % 64) + element = F192(*limbs) + for challenge, shift in zip(challenges, RING_MAP_SHIFTS): + frobenius = element + for _ in range(shift): + frobenius *= frobenius + element += challenge * frobenius + weights.append(element) + return weights + + +def _transpose(values: Sequence[F192]) -> list[F192]: + require(len(values) == 64, "ring-switch slice has the wrong length") + output = [0] * 192 for row, value in enumerate(values): bits = int(value) while bits: bit = (bits & -bits).bit_length() - 1 output[bit] ^= 1 << row bits &= bits - 1 - return [F128(value) for value in output] + return [F192(value) for value in output] -def _fold_binary_elements(elements: Sequence[F128], weights: Sequence[F128]) -> list[F128]: - output = [] - for element in elements: - bits = int(element) - value = ZERO - while bits: - bit = (bits & -bits).bit_length() - 1 - value += weights[bit] - bits &= bits - 1 - output.append(value) - return output +def _linear_map(value: F192, weights: Sequence[F192]) -> F192: + result = ZERO + bits = int(value) + while bits: + bit = (bits & -bits).bit_length() - 1 + result += weights[bit] + bits &= bits - 1 + return result + + +def _ring_weight( + suffix_point: Sequence[F192], + query: Sequence[F192], + coordinate_weights: Sequence[F192], +) -> F192: + """Evaluate the transparent ring-switch weight at one query point.""" + require(len(suffix_point) == len(query), "ring-switch query dimension mismatch") + suffix_tensor = build_eq(suffix_point) + query_tensor = build_eq(query) + return sum( + (query_weight * _linear_map(suffix_weight, coordinate_weights) + for query_weight, suffix_weight in zip(query_tensor, suffix_tensor)), + ZERO, + ) def verify_stacked_opening( @@ -1932,23 +2165,27 @@ def verify_stacked_opening( opening: LigeritoOpening, root: bytes, stack_log: int, + initial_rate: int, qpkd_offset: int, qpkd_variables: int, reduction: Reduction, - point_claims: Sequence[tuple[Sequence[F128], F128]], + point_claims: Sequence[tuple[Sequence[F192], F192]], ) -> None: """Bind both ring-switched claims and all ordinary stack point claims.""" ring_claims = (reduction.ab, reduction.c) - slices = [] - for claim in ring_claims: - values = transcript.scalars(128) + require(len(opening.ring_switches) == len(ring_claims), "wrong ring-switch proof count") + slices: list[Sequence[F192]] = [] + for claim, values in zip(ring_claims, opening.ring_switches): + require(len(values) == 64, "ring-switch proof has the wrong width") + for value in values: + transcript.observe(value) expected = sum((a * b for a, b in zip(_claim_weights(claim.point), values)), ZERO) require(expected == claim.value, "ring-switch claim mismatch") slices.append(values) - ring_challenge = transcript.samples(7) - ring_weights = build_eq(ring_challenge) - ring_values = [sum((a * b for a, b in zip(_transpose(values), ring_weights)), ZERO) + map_challenges = [transcript.sample() for _ in RING_MAP_SHIFTS] + coordinate_weights = _coordinate_weights(map_challenges) + ring_values = [sum((a * b for a, b in zip(_transpose(values), coordinate_weights)), ZERO) for values in slices] ring_scales = transcript.samples(2) target = sum((scale * value for scale, value in zip(ring_scales, ring_values)), ZERO) @@ -1958,23 +2195,30 @@ def verify_stacked_opening( point_scales = transcript.samples(len(point_claims)) target += sum((scale * value for scale, (_, value) in zip(point_scales, point_claims)), ZERO) - ring_polynomials = [] - for claim in ring_claims: - suffix = build_eq(claim.point.ring_tail[1:]) - ring_polynomials.append(_fold_binary_elements(suffix, ring_weights)) selector = qpkd_offset >> qpkd_variables - def evaluate_basis(prefix: Sequence[F128], residual_log: int) -> list[F128]: + def evaluate_basis(prefix: Sequence[F192], residual_log: int) -> list[F192]: + shared_ring = None + if len(prefix) >= qpkd_variables: + shared_ring = sum( + (scale * _ring_weight( + claim.point.ring_tail, + prefix[:qpkd_variables], + coordinate_weights, + ) for scale, claim in zip(ring_scales, ring_claims)), + ZERO, + ) result = [] for vertex in range(1 << residual_log): - point = list(prefix) + [F128(vertex >> bit & 1) for bit in range(residual_log)] + point = list(prefix) + [F192(vertex >> bit & 1) for bit in range(residual_log)] low, high = point[:qpkd_variables], point[qpkd_variables:] selector_weight = ONE for bit, challenge in enumerate(high): selector_weight *= challenge if selector >> bit & 1 else ONE + challenge - value = selector_weight * sum( - (scale * mle_eval(poly, low) - for scale, poly in zip(ring_scales, ring_polynomials)), ZERO) + ring_value = shared_ring if shared_ring is not None else sum( + (scale * _ring_weight(claim.point.ring_tail, low, coordinate_weights) + for scale, claim in zip(ring_scales, ring_claims)), ZERO) + value = selector_weight * ring_value for scale, (claim_point, _) in zip(point_scales, point_claims): require(len(claim_point) == len(point), "stack point has the wrong dimension") factor = ONE @@ -1984,7 +2228,15 @@ def evaluate_basis(prefix: Sequence[F128], residual_log: int) -> list[F128]: result.append(value) return result - verify_ligerito(transcript, opening, stack_log, target, root, evaluate_basis) + verify_ligerito( + transcript, + opening.ligerito, + stack_log, + initial_rate, + target, + root, + evaluate_basis, + ) def verify_reduction(log_n: int, transcript: Transcript) -> Reduction: @@ -2001,16 +2253,14 @@ def verify_reduction(log_n: int, transcript: Transcript) -> Reduction: def verify_lincheck( log_n: int, point: QuirkyPoint, - a: F128, - b: F128, + a: F192, + b: F192, transcript: Transcript, ) -> LincheckResult: """Replay the fixed BLAKE3 matrix reduction.""" alpha = transcript.sample() inner_weights = quirky_weights(point.skip, point.inner) - coefficients = blake3_matrix_fold(alpha, inner_weights) beta = transcript.sample() - coefficients[512] += beta running = alpha * a + b + beta challenges = [] for _ in range(8): @@ -2019,22 +2269,28 @@ def verify_lincheck( linear = at_zero + at_one + at_infinity challenge = transcript.sample() running = at_infinity * challenge * challenge + linear * challenge + at_zero - half = len(coefficients) // 2 - coefficients = [coefficients[i] * (ONE + challenge) + coefficients[i + half] * challenge - for i in range(half)] challenges.append(challenge) partial = transcript.scalars(64) - require(sum((x * y for x, y in zip(coefficients, partial)), ZERO) == running, + rest_weights = build_eq(tuple(reversed(challenges))) + column_weights = [value * weight for weight in rest_weights for value in partial] + terminal = blake3_bilinear(alpha, inner_weights, column_weights) + terminal += beta * column_weights[512] + require(terminal == running, "Flock lincheck terminal mismatch") skip = transcript.sample() value = sum((x * y for x, y in zip(lagrange_weights(PHI[:64], skip), partial)), ZERO) return LincheckResult(QuirkyPoint(skip, tuple(reversed(challenges)), point.outer), value) -def blake3_matrix_fold(alpha: F128, row_weights: Sequence[F128]) -> list[F128]: - """Compute ``alpha * eq^T A + eq^T B`` by walking the fixed circuit.""" - require(len(row_weights) == 1 << 14, "bad BLAKE3 row-weight vector") +def blake3_bilinear( + alpha: F192, + row_weights: Sequence[F192], + column_weights: Sequence[F192], +) -> F192: + """Evaluate the two BLAKE3 R1CS matrix forms by walking the circuit.""" size = 1 << 14 + require(len(row_weights) == size, "bad BLAKE3 row-weight vector") + require(len(column_weights) == size, "bad BLAKE3 column-weight vector") constant = 512 message_base = 640 counter_low = 1152 @@ -2049,48 +2305,50 @@ def blake3_matrix_fold(alpha: F128, row_weights: Sequence[F128]) -> list[F128]: (0,5,10,15),(1,6,11,12),(2,7,8,13),(3,4,9,14)) message_pairs = ((0,1),(2,3),(4,5),(6,7),(8,9),(10,11),(12,13),(14,15)) permutation = (2,6,3,10,7,0,4,13,1,11,12,5,9,14,15,8) - accumulated = [0] * size + left_total = ZERO + right_total = ZERO + constant_rows = ZERO - def emit(row: int, left: frozenset[int], right: frozenset[int]) -> None: - weight = row_weights[row] - scaled = int(alpha * weight) - plain = int(weight) - for column in left: - accumulated[column] ^= scaled - for column in right: - accumulated[column] ^= plain + def slots(base: int) -> tuple[F192, ...]: + return tuple(column_weights[base + bit] for bit in range(32)) - def slots(base: int) -> tuple[frozenset[int], ...]: - return tuple(frozenset((base + bit,)) for bit in range(32)) + empty_word = (ZERO,) * 32 - empty_word = tuple(frozenset() for _ in range(32)) - - def literal(value: int) -> tuple[frozenset[int], ...]: - return tuple(frozenset((constant,)) if value >> bit & 1 else frozenset() + def literal(value: int) -> tuple[F192, ...]: + return tuple(column_weights[constant] if value >> bit & 1 else ZERO for bit in range(32)) - def xor(x: Sequence[frozenset[int]], y: Sequence[frozenset[int]]) -> tuple[frozenset[int], ...]: - return tuple(a ^ b for a, b in zip(x, y)) + def xor(x: Sequence[F192], y: Sequence[F192]) -> tuple[F192, ...]: + return tuple(a + b for a, b in zip(x, y)) - def rotate_right(word: Sequence[frozenset[int]], amount: int) -> tuple[frozenset[int], ...]: + def rotate_right(word: Sequence[F192], amount: int) -> tuple[F192, ...]: return tuple(word[(bit + amount) & 31] for bit in range(32)) - def add(x: Sequence[frozenset[int]], y: Sequence[frozenset[int]], carry_base: int): - carry: set[int] = set() + def add(x: Sequence[F192], y: Sequence[F192], carry_base: int) -> tuple[F192, ...]: + nonlocal left_total, right_total + carry = ZERO output = [] for bit in range(32): if bit < 31: - emit(carry_base + bit, x[bit] ^ carry, y[bit] ^ carry) - output.append(x[bit] ^ y[bit] ^ carry) + weight = row_weights[carry_base + bit] + left_total += weight * (x[bit] + carry) + right_total += weight * (y[bit] + carry) + output.append(x[bit] + y[bit] + carry) if bit < 31: - carry.add(carry_base + bit) + carry += column_weights[carry_base + bit] return tuple(output) - # Booleanity/constant and unconstrained public input rows. - emit(constant, frozenset((constant,)), frozenset((constant,))) - for base, length in ((0,256),(message_base,512),(counter_low,64),(block_length,32),(flags,32)): + def linear_rows(values: Sequence[F192], base: int) -> None: + nonlocal left_total, constant_rows + for bit in range(32): + left_total += row_weights[base + bit] * values[bit] + constant_rows += row_weights[base + bit] + + for base, length in ((0, 256), (message_base, 512), (counter_low, 64), + (block_length, 32), (flags, 32)): for row in range(base, base + length): - emit(row, frozenset((row,)), frozenset((constant,))) + left_total += row_weights[row] * column_weights[row] + constant_rows += row_weights[row] state = [empty_word for _ in range(16)] for word in range(8): @@ -2121,9 +2379,8 @@ def add(x: Sequence[frozenset[int]], y: Sequence[frozenset[int]], carry_base: in b_new = rotate_right(xor(b1, c2), 7) b_base = gate_base + 186 d_base = b_base + 32 - for bit in range(32): - emit(b_base + bit, b_new[bit], frozenset((constant,))) - emit(d_base + bit, d2[bit], frozenset((constant,))) + linear_rows(b_new, b_base) + linear_rows(d2, d_base) state[lane_a] = a2 state[lane_b] = slots(b_base) state[lane_c] = c2 @@ -2133,16 +2390,19 @@ def add(x: Sequence[frozenset[int]], y: Sequence[frozenset[int]], carry_base: in for word in range(8): low = xor(state[word], state[word + 8]) high = xor(state[word + 8], slots(32 * word)) - for bit in range(32): - emit(256 + 32 * word + bit, low[bit], frozenset((constant,))) - emit(output_high + 32 * word + bit, high[bit], frozenset((constant,))) - return [F128(value) for value in accumulated] + linear_rows(low, 256 + 32 * word) + linear_rows(high, output_high + 32 * word) + + constant_weight = column_weights[constant] + left_total += constant_weight * row_weights[constant] + right_total += constant_weight * (constant_rows + row_weights[constant]) + return alpha * left_total + right_total # Complete VM verification and CLI ------------------------------------------- -def _selector_point(selector: int, length: int) -> tuple[F128, ...]: - return tuple(F128(selector >> bit & 1) for bit in range(length)) +def _selector_point(selector: int, length: int) -> tuple[F192, ...]: + return tuple(F192(selector >> bit & 1) for bit in range(length)) def verify_execution(statement: dict[str, Any], proof: Proof) -> None: @@ -2154,14 +2414,20 @@ def verify_execution(statement: dict[str, Any], proof: Proof) -> None: public_input = tuple(parse_field(value) for value in encoded_input) transcript = Transcript(proof, b"leanvm-b", program.transcript_statement(public_input)) - announced = transcript.scalars(7) - require(all(value.hi == 0 for value in announced), "announced size has a nonzero high limb") - log_memory = announced[0].lo - row_counts = tuple(value.lo for value in announced[1:]) + announced = transcript.scalars(9) + require(all(value.c1 == value.c2 == 0 for value in announced), "announced size has a nonzero high limb") + log_memory = announced[0].c0 + row_counts = tuple(value.c0 for value in announced[1:8]) + log_inverse_rate = announced[8].c0 + require(1 <= log_inverse_rate <= 4, "invalid PCS inverse rate") layout = build_layout(program, log_memory, row_counts) root_words = transcript.scalars(2) - root = root_words[0].to_bytes() + root_words[1].to_bytes() + require(all(word.c2 == 0 for word in root_words), "commitment root has a nonzero top limb") + root = b"".join( + limb.to_bytes(8, "little") + for limb in (root_words[0].c0, root_words[0].c1, root_words[1].c0, root_words[1].c1) + ) bus = verify_bus_balance(layout.push, layout.pull, layout.count, layout.padding, transcript) eta = transcript.sample() identity_count = sum(CONSTRAINT_COUNTS) @@ -2181,13 +2447,19 @@ def verify_execution(statement: dict[str, Any], proof: Proof) -> None: claims = list(bus.claims) claims.extend(constraint_claims(layout, air_claims)) public_challenge = transcript.sample() + public_low, public_high = transcript.scalars(2) public_point = [ZERO] * layout.placements[0].variables public_point[0] = public_challenge public_value = interpolate(public_input[0], public_input[1], public_challenge) - claims.append(ColumnClaim(0, tuple(public_point), public_value)) + y = F192(0, 1) + public_top = (public_value + public_low + y * public_high) / (y * y) + claims.extend( + ColumnClaim(column, tuple(public_point), value) + for column, value in enumerate((public_low, public_high, public_top)) + ) - point_claims: list[tuple[tuple[F128, ...], F128]] = [] - qpkd = layout.placements[3] + point_claims: list[tuple[tuple[F192, ...], F192]] = [] + qpkd = layout.placements[5] for claim in claims: slot = virtual_slot(claim.column) if slot is None: @@ -2199,10 +2471,10 @@ def verify_execution(statement: dict[str, Any], proof: Proof) -> None: selector, layout.stack_log - placement.variables ) else: - require(len(claim.point) + 7 == qpkd.variables, "BLAKE3 slot claim dimension mismatch") + require(len(claim.point) + 8 == qpkd.variables, "BLAKE3 slot claim dimension mismatch") selector = qpkd.offset >> qpkd.variables full_point = ( - _selector_point(slot, 7) + _selector_point(slot, 8) + claim.point + _selector_point(selector, layout.stack_log - qpkd.variables) ) @@ -2215,6 +2487,7 @@ def verify_execution(statement: dict[str, Any], proof: Proof) -> None: opening, root, layout.stack_log, + log_inverse_rate, qpkd.offset, qpkd.variables, reduction, diff --git a/snark_lib.py b/snark_lib.py index ad996502..3aac4f60 100644 --- a/snark_lib.py +++ b/snark_lib.py @@ -10,10 +10,10 @@ class _Elt: - """A GF(2^128) element (GHASH form). Indices and addresses are carried as - powers of GEN — "in the exponent": `GEN ** k` is the k-th index, `x * GEN` - its successor. A heap pointer is an element too, its cells addressed by - g-power offsets (`buf[i]` is the cell at `buf * i`, write-once).""" + """A 192-bit machine word in E = GF(2^192), represented as a cubic tower + over K = GF(2^64). Indices and addresses are K-valued powers of GEN — + "in the exponent": `GEN ** k` is the k-th index and `x * GEN` its successor. + A heap pointer is K-valued too; `buf[i]` is the write-once cell at `buf * i`.""" def __add__(self, other): # field addition = XOR _ = other @@ -21,7 +21,7 @@ def __add__(self, other): # field addition = XOR __radd__ = __add__ - def __mul__(self, other): # field (GHASH) product + def __mul__(self, other): # tower-field product _ = other return _Elt() @@ -45,8 +45,14 @@ def __setitem__(self, idx, value): # heap store m[self · idx] (write-once) _ = idx, value +def f192(c0: int, c1: int, c2: int) -> _Elt: + """Construct a field constant from its three little-endian GF(2^64) limbs.""" + _ = c0, c1, c2 + return _Elt() + + GEN = _Elt() -"""The fixed generator g = x of GF(2^128)^× (order 2^128 - 1).""" +"""The fixed generator g = x of K^× = GF(2^64)^× (order 2^64 - 1).""" def hint_decompose_bits(bits, value, nbits: int) -> None: @@ -144,6 +150,27 @@ def hint_witness(dest, name: str) -> None: _ = dest, name +def pack64x2(a, b) -> _Elt: + """Prove that `a` and `b` are GF(2^64)-valued machine words and return + their canonical 128-bit packing `(a.c0, b.c0, 0)` as one GF(2^192) word. + This is one VM instruction.""" + _ = a, b + return _Elt() + + +def pack64x2_into(a, b, out) -> None: + """The destination-target form of `pack64x2`: assert that `out` is the + canonical packing `(a.c0, b.c0, 0)`. All three arguments are scalar cells.""" + _ = a, b, out + + +def hint_f192_limbs(dest, value) -> None: + """Computed advice: write the first `len(dest)` GF(2^64) coordinate limbs + of `value` into a 1-to-3-cell StackBuf. UNCONSTRAINED; callers bind the + result with `PACK64X2` and/or field reconstruction.""" + _ = dest, value + + def blake3( a, b, diff --git a/src/main.rs b/src/main.rs index 1ac90350..ed786b60 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,6 +2,7 @@ //! //! ```text //! cargo run --release -- xmss --n-signatures 820 +//! cargo run --release -- xmss --n-signatures 820 --log-inv-rate 2 //! cargo run --release -- recursion --n 2 //! cargo run --release -- fibonacci --n 2000000 //! cargo run --release -- --tracing fibonacci --n 2000000 @@ -11,6 +12,16 @@ use clap::{Parser, Subcommand}; #[derive(Parser)] struct Cli { + /// Ligerito inverse-rate logarithm: 1, 2, 3, or 4 selects rate 1/2, + /// 1/4, 1/8, or 1/16 respectively. + #[arg( + long, + global = true, + default_value_t = 1, + value_parser = parse_log_inv_rate + )] + log_inv_rate: usize, + /// Enable hierarchical timing traces. Use RUST_LOG to adjust verbosity. #[arg(long, global = true)] tracing: bool, @@ -35,8 +46,11 @@ enum Command { /// BLAKE3 compressions per inner proof (inner program shape). #[arg(long, default_value = "8")] hashes: usize, - /// MUL iterations per inner proof (inner program shape). - #[arg(long, default_value = "32768")] + /// MUL iterations per inner proof (inner program shape). Chosen so the + /// inner committed witness fills most of a 2^26 PCS, which is the size + /// the recursion cost should be quoted at. The inner program's DEREF + /// range check gives out just above 66000, so this is near the ceiling. + #[arg(long, default_value = "64000")] iters: usize, }, /// Prove and verify Fibonacci in the exponent (demo). @@ -47,6 +61,17 @@ enum Command { }, } +fn parse_log_inv_rate(value: &str) -> Result { + let rate = value + .parse::() + .map_err(|_| "log_inv_rate must be one of 1, 2, 3, or 4".to_string())?; + if (1..=4).contains(&rate) { + Ok(rate) + } else { + Err("log_inv_rate must be one of 1, 2, 3, or 4".to_string()) + } +} + fn main() { let cli = Cli::parse(); match cli.command { @@ -54,17 +79,17 @@ fn main() { if cli.tracing { primitives::init_tracing(); } - rec_aggregation::run_xmss_aggregation(n_signatures); + rec_aggregation::run_xmss_aggregation(n_signatures, cli.log_inv_rate); } Command::Recursion { n, hashes, iters } => { let inner: Vec<(usize, usize)> = (0..n).map(|_| (hashes, iters)).collect(); - rec_aggregation::run_recursion(&inner, cli.tracing); + rec_aggregation::run_recursion(&inner, cli.log_inv_rate, cli.tracing); } Command::Fibonacci { n } => { if cli.tracing { primitives::init_tracing(); } - rec_aggregation::run_fibonacci(n); + rec_aggregation::run_fibonacci(n, cli.log_inv_rate); } } }