feat(wasm)!: support large-file downloads and disk streaming

README.md

@@ -35,6 +35,46 @@ transports use `record.rs` for BLAKE3 verification. Native
 `Client::data_download_range` and the browser media reader call the same range
 implementation.
 
+Browser downloads and range reads have no 1 GB file cap. File sizes and offsets
+use 64-bit arithmetic internally; the JavaScript API accepts exact, nonnegative
+`number` positions through `Number.MAX_SAFE_INTEGER` (just under 8 PiB). The
+resolved DataMap and seek index still occupy memory proportional to the number
+of chunks. Individual chunk buffers remain bounded, including for seeks past
+4 GiB. Uploads retain their existing size limits.
+
+`downloadPublicFile(file, concurrency?, onProgress?, maxMemoryBytes?)` and its
+private-file counterpart allocate the output as a JavaScript `Uint8Array`, then
+fill it using bounded WASM reads. They no longer retain a complete plaintext
+or ciphertext file in WASM memory. The optional budget limits the output buffer,
+not total process memory. Without a budget, allocation is attempted up to the
+JavaScript engine's buffer limit. Allocation errors reject with guidance to use
+streaming; browsers do not expose a reliable portable amount of free memory.
+
+For large downloads, pass a `WritableStream` to `BrowserFileReader.pipeTo`.
+For example, after the application obtains a file handle from a user gesture:
+
+```js
+const reader = await client.openPublicFile(address);
+try {
+  const writable = await fileHandle.createWritable();
+  const { bytesWritten, hash } = await reader.pipeTo(writable);
+} finally {
+  reader.close();
+}
+```
+
+The same API works with `openPrivateFile({ data_map })`. `pipeTo(writable,
+{ start?, end? }, onProgress?)` optionally streams a half-open byte range and
+reports `(bytesWritten, totalBytes)` after each write. It fetches and verifies
+at most 4 MiB of plaintext per write, waits for the destination before continuing,
+closes the destination on success, aborts on failure, and releases the writer
+lock. Its result contains the byte count and BLAKE3 of the written range.
+`reader.close()` cancels at a read/write boundary. `readRange(start, length)`
+remains available for media and custom streaming sinks, with a 4 MiB per-call
+limit. A sink must persist or consume the bytes instead of accumulating them to
+keep memory bounded. The application chooses a memory budget and supplies its
+disk destination; the core does not open file pickers or silently select a path.
+
 The adapters supply QUIC or WebRTC discovery/GET requests, runtime timers,
 caches, progress callbacks, and output handling. Browser descriptors and wallet
 callbacks remain browser API concerns. Native filesystem streaming still uses

ant-core/examples/generate-browser-large-file.rs

@@ -0,0 +1,47 @@
+//! Regenerate the compact native fixture used by the WASM large-file tests:
+//! cargo run -p ant-core --release --example generate-browser-large-file
+//!
+//! Encrypts >4 GiB with bounded memory. Repeated plaintext compresses well;
+//! all encrypted records fit in a small fixture without allocating the file.
+use bytes::Bytes;
+use std::collections::HashMap;
+
+fn main() {
+    let size = (1u64 << 32) + 2 * self_encryption::MAX_CHUNK_SIZE as u64 + 123;
+    let block = Bytes::from(vec![0x5a; self_encryption::MAX_CHUNK_SIZE]);
+    let mut remaining = size as usize;
+    let mut hasher = blake3::Hasher::new();
+    let input = std::iter::from_fn(|| {
+        if remaining == 0 {
+            return None;
+        }
+        let length = remaining.min(block.len());
+        remaining -= length;
+        hasher.update(&block[..length]);
+        Some(block.slice(..length))
+    });
+    let mut encryptor = self_encryption::stream_encrypt(size as usize, input).unwrap();
+    let records: HashMap<_, _> = encryptor.chunks().map(|item| item.unwrap()).collect();
+    let map = encryptor.into_datamap().unwrap();
+    let root =
+        self_encryption::get_root_data_map(map.clone(), &mut |hash| Ok(records[&hash].clone()))
+            .unwrap();
+    assert_eq!(root.original_file_size() as u64, size);
+    let mut retained = records;
+    let map_bytes = rmp_serde::to_vec(&map).unwrap();
+    let address = *blake3::hash(&map_bytes).as_bytes();
+    retained.insert(self_encryption::XorName(address), Bytes::from(map_bytes));
+    let mut records: Vec<_> = retained.into_iter().map(|(hash, content)| {
+        serde_json::json!({"address": hex::encode(hash.0), "content": hex::encode(content)})
+    }).collect();
+    records.sort_by(|a, b| a["address"].as_str().cmp(&b["address"].as_str()));
+    let fixture = serde_json::json!({
+        "size": size, "byte": 0x5a, "hash": hasher.finalize().to_hex().to_string(),
+        "address": hex::encode(address), "records": records,
+    });
+    let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("wasm-tests/fixtures/large-file.json");
+    std::fs::create_dir_all(path.parent().unwrap()).unwrap();
+    std::fs::write(&path, serde_json::to_vec_pretty(&fixture).unwrap()).unwrap();
+    eprintln!("Wrote {}-byte file fixture to {}", size, path.display());
+}

ant-core/src/browser.rs

@@ -31,10 +31,10 @@ use self_encryption::{DataMap, EncryptedChunk};
 use serde::{Deserialize, Serialize};
 use std::collections::HashSet;
 
-/// Maximum file size accepted by the browser API (1 GB decimal).
+/// Maximum file size accepted by the browser upload API (1 GB decimal).
 ///
 /// The page upload path streams through a worker and browser storage. Complete
-/// downloads and the legacy whole-buffer encryption binding remain memory-bound.
+/// downloads use bounded ranges; the legacy encryption binding remains memory-bound.
 pub const MAX_BROWSER_FILE_BYTES: usize = 1_000_000_000;
 
 /// One native self-encryption chunk descriptor exposed to the browser.

ant-core/src/browser/manifest.rs

@@ -8,7 +8,8 @@ use serde::{Deserialize, Serialize};
 /// Current browser testnet manifest version.
 pub const BROWSER_MANIFEST_VERSION: u16 = 6;
 const MAX_DATA_MAP_BYTES: usize = 4 * 1024 * 1024;
-const MAX_FILE_CHUNKS: usize = 1024;
+/// Largest exact byte position exposed through the JavaScript number API.
+pub const MAX_BROWSER_FILE_POSITION: u64 = 9_007_199_254_740_991;
 
 /// A validated WebRTC Direct bootstrap endpoint.
 pub type BrowserManifestEndpoint = BrowserEndpoint;
@@ -21,7 +22,7 @@ pub struct PublicFileDescriptor {
     /// Public DataMap content address.
     pub address: String,
     /// Plaintext file size.
-    pub size: usize,
+    pub size: u64,
     /// Browser MIME type.
     pub content_type: String,
     /// Whole-file plaintext BLAKE3 hash.
@@ -150,7 +151,7 @@ fn normalize_file(file: &mut PublicFileDescriptor) -> Result<(), BrowserManifest
         ));
     }
     file.address = normalize_hex(&file.address, 32).map_err(BrowserManifestError)?;
-    if !(self_encryption::MIN_ENCRYPTABLE_BYTES..=super::MAX_BROWSER_FILE_BYTES)
+    if !(self_encryption::MIN_ENCRYPTABLE_BYTES as u64..=MAX_BROWSER_FILE_POSITION)
         .contains(&file.size)
     {
         return Err(BrowserManifestError(format!(
@@ -165,13 +166,13 @@ fn normalize_file(file: &mut PublicFileDescriptor) -> Result<(), BrowserManifest
             file.data_map_size
         )));
     }
-    if !(3..=MAX_FILE_CHUNKS).contains(&file.chunks.len()) {
+    if file.chunks.len() < 3 {
         return Err(BrowserManifestError(
             "public file has an invalid self-encryption chunk list".to_string(),
         ));
     }
     file.chunks.sort_by_key(|chunk| chunk.index);
-    let mut reconstructed_size = 0usize;
+    let mut reconstructed_size = 0u64;
     for (expected_index, chunk) in file.chunks.iter_mut().enumerate() {
         if chunk.index != expected_index {
             return Err(BrowserManifestError(
@@ -180,14 +181,14 @@ fn normalize_file(file: &mut PublicFileDescriptor) -> Result<(), BrowserManifest
         }
         chunk.dst_hash = normalize_hex(&chunk.dst_hash, 32).map_err(BrowserManifestError)?;
         chunk.src_hash = normalize_hex(&chunk.src_hash, 32).map_err(BrowserManifestError)?;
-        if chunk.src_size == 0 {
+        if chunk.src_size == 0 || chunk.src_size > self_encryption::MAX_CHUNK_SIZE {
             return Err(BrowserManifestError(format!(
                 "invalid plaintext chunk size {}",
                 chunk.src_size
             )));
         }
         reconstructed_size = reconstructed_size
-            .checked_add(chunk.src_size)
+            .checked_add(chunk.src_size as u64)
             .ok_or_else(|| BrowserManifestError("file size overflow".to_string()))?;
     }
     if reconstructed_size != file.size {
@@ -286,4 +287,31 @@ mod tests {
         assert_eq!(manifest.files[0].chunks[0].index, 0);
         assert_eq!(manifest.payment.chain_id, 31337);
     }
+
+    #[test]
+    fn file_descriptors_allow_more_than_four_gib_and_1024_chunks() {
+        let mut file = PublicFileDescriptor {
+            name: "large.bin".into(),
+            address: "11".repeat(32),
+            size: 1030 * self_encryption::MAX_CHUNK_SIZE as u64,
+            content_type: "application/octet-stream".into(),
+            blake3: "22".repeat(32),
+            data_map_size: 256,
+            replicas: 4,
+            chunks: (0..1030)
+                .map(|index| BrowserChunkInfo {
+                    index,
+                    src_size: self_encryption::MAX_CHUNK_SIZE,
+                    src_hash: "33".repeat(32),
+                    dst_hash: "44".repeat(32),
+                })
+                .collect(),
+        };
+        assert!(file.size > u32::MAX as u64);
+        normalize_file(&mut file).unwrap();
+        file.size += 1;
+        assert!(normalize_file(&mut file).is_err());
+        file.size = MAX_BROWSER_FILE_POSITION + 1;
+        assert!(normalize_file(&mut file).is_err());
+    }
 }