diff --git a/bin/node/src/commands/lifecycle.rs b/bin/node/src/commands/lifecycle.rs index cc5660ecb7..ce3a306c96 100644 --- a/bin/node/src/commands/lifecycle.rs +++ b/bin/node/src/commands/lifecycle.rs @@ -57,7 +57,7 @@ impl BootstrapCommand { let genesis_block = read_bootstrap_genesis_block(self.genesis_block_file.as_deref(), self.network).await?; let genesis_commitment = genesis_block.inner().header().commitment(); - State::bootstrap(genesis_block, &self.data_directory)?; + State::bootstrap(genesis_block, &self.data_directory).await?; tracing::info!( target: crate::LOG_TARGET, { diff --git a/bin/stress-test/src/seeding/mod.rs b/bin/stress-test/src/seeding/mod.rs index 073a96f85c..6761eaa79a 100644 --- a/bin/stress-test/src/seeding/mod.rs +++ b/bin/stress-test/src/seeding/mod.rs @@ -234,7 +234,9 @@ pub async fn seed_store_with_readers( ); let genesis_block = genesis_state.into_block().expect("genesis block should be created"); let genesis_header = genesis_block.inner().header().clone(); - State::bootstrap(genesis_block, &data_directory).expect("store should bootstrap"); + State::bootstrap(genesis_block, &data_directory) + .await + .expect("store should bootstrap"); let (state, mut block_writer, writer_task) = load_state(data_directory.clone()).await; diff --git a/crates/block-producer/src/server/tests.rs b/crates/block-producer/src/server/tests.rs index db9efcad5b..23b249e6d5 100644 --- a/crates/block-producer/src/server/tests.rs +++ b/crates/block-producer/src/server/tests.rs @@ -72,7 +72,7 @@ fn mempool_stats_track_uncommitted_work_and_the_canonical_tip() { #[tokio::test] async fn block_producer_starts_with_store_state() { let data_directory = tempfile::tempdir().expect("tempdir should be created"); - bootstrap_store(data_directory.path()); + bootstrap_store(data_directory.path()).await; let (state, block_writer, proof_writer) = State::for_tests(data_directory.path()).await; let block_producer = Sequencer { @@ -99,7 +99,7 @@ async fn block_producer_starts_with_store_state() { assert_eq!(status.chain_tip, BlockNumber::GENESIS); } -fn bootstrap_store(path: &std::path::Path) { +async fn bootstrap_store(path: &std::path::Path) { let signer = random_secret_key(); let genesis_state = GenesisState::new( vec![], @@ -110,5 +110,5 @@ fn bootstrap_store(path: &std::path::Path) { ); let genesis_block = genesis_state.into_block().expect("genesis block should be created"); - State::bootstrap(genesis_block, path).expect("store should bootstrap"); + State::bootstrap(genesis_block, path).await.expect("store should bootstrap"); } diff --git a/crates/rpc/src/tests.rs b/crates/rpc/src/tests.rs index b2028a1316..39148d0204 100644 --- a/crates/rpc/src/tests.rs +++ b/crates/rpc/src/tests.rs @@ -93,7 +93,7 @@ impl TestStore { async fn start() -> Self { let data_directory = new_tempdir(); - let genesis_commitment = Self::bootstrap(&data_directory); + let genesis_commitment = Self::bootstrap(&data_directory).await; let (state, ..) = State::for_tests(&data_directory).await; Self { state, @@ -102,7 +102,7 @@ impl TestStore { } } - fn bootstrap(path: &std::path::Path) -> Word { + async fn bootstrap(path: &std::path::Path) -> Word { let config = GenesisConfig::default(); let validator_key = miden_protocol::crypto::dsa::ecdsa_k256_keccak::SigningKey::read_from_bytes(&[7; 32]) @@ -115,7 +115,7 @@ impl TestStore { genesis_state.clone().into_block().expect("genesis block should be created"); let genesis_commitment = genesis_block.inner().header().commitment(); - State::bootstrap(genesis_block, path).expect("store should bootstrap"); + State::bootstrap(genesis_block, path).await.expect("store should bootstrap"); genesis_commitment } @@ -434,7 +434,8 @@ async fn rpc_rejects_post_deployment_network_account_tx() { miden_node_store::test_support::seed_network_account( &store.data_directory_path().join("miden-store.sqlite3"), network_account_id, - ); + ) + .await; // Build a non-deployment tx for that account. let (account, _) = build_test_account([0; 32]); @@ -556,7 +557,7 @@ async fn start_source_rpc( ) -> (RpcClient, TestStore) { let store = TestStore::start().await; let block_producer_dir = new_tempdir(); - TestStore::bootstrap(&block_producer_dir); + TestStore::bootstrap(&block_producer_dir).await; let (block_producer_state, ..) = State::for_tests(&block_producer_dir).await; let state = Arc::clone(&store.state); @@ -1024,7 +1025,7 @@ async fn start_rpc() -> (RpcClient, std::net::SocketAddr, TestStore) { let grpc_options = GrpcOptions::test(); let store = TestStore::start().await; let block_producer_dir = new_tempdir(); - TestStore::bootstrap(&block_producer_dir); + TestStore::bootstrap(&block_producer_dir).await; let (block_producer_state, ..) = State::for_tests(&block_producer_dir).await; let state = Arc::clone(&store.state); diff --git a/crates/store/src/account_state_forest/mod.rs b/crates/store/src/account_state_forest/mod.rs index 34d3477c08..89ccf941a5 100644 --- a/crates/store/src/account_state_forest/mod.rs +++ b/crates/store/src/account_state_forest/mod.rs @@ -42,8 +42,8 @@ use miden_protocol::{EMPTY_WORD, Word}; use thiserror::Error; use crate::COMPONENT; -pub use crate::db::models::queries::HISTORICAL_BLOCK_RETENTION; -use crate::db::models::queries::{PrecomputedPublicAccountState, PrecomputedPublicAccountStates}; +pub use crate::db::HISTORICAL_BLOCK_RETENTION; +use crate::db::{PrecomputedPublicAccountState, PrecomputedPublicAccountStates}; use crate::errors::AccountStateForestUpdateError; #[cfg(test)] diff --git a/crates/store/src/db/migrations.rs b/crates/store/src/db/migrations.rs index 213b6df53c..2ca8a0e1de 100644 --- a/crates/store/src/db/migrations.rs +++ b/crates/store/src/db/migrations.rs @@ -63,21 +63,5 @@ pub fn verify_latest_schema(database_filepath: &Path) -> std::result::Result<(), Ok(()) } -#[cfg(test)] -pub(crate) fn test_connection() -> diesel::SqliteConnection { - use diesel::{Connection, SqliteConnection}; - - let temp_dir = tempfile::tempdir().expect("failed to create temp directory"); - let database_filepath = temp_dir.path().join("test.sqlite3"); - bootstrap_database(&database_filepath).expect("database should bootstrap"); - - let conn = SqliteConnection::establish( - database_filepath.to_str().expect("temp database path should be valid UTF-8"), - ) - .expect("temp file sqlite should always work"); - let _kept_dir = temp_dir.keep(); - conn -} - #[cfg(test)] mod tests; diff --git a/crates/store/src/db/migrations/tests/mod.rs b/crates/store/src/db/migrations/tests/mod.rs index 9584a32f70..2096789f76 100644 --- a/crates/store/src/db/migrations/tests/mod.rs +++ b/crates/store/src/db/migrations/tests/mod.rs @@ -2,13 +2,12 @@ use std::process::Command; use anyhow::{Context, Result, ensure}; use diesel::connection::SimpleConnection; -use diesel::query_dsl::methods::{OrderDsl, SelectDsl}; -use diesel::{Connection, ExpressionMethods, RunQueryDsl, SqliteConnection}; +use diesel::{Connection, SqliteConnection}; use miden_node_db::migration::{SchemaHash, SchemaHashes}; use super::*; -use crate::db::models::queries::VALID_FOREVER; -use crate::db::schema; +use crate::db::TestDb; +use crate::db::queries::VALID_FOREVER; const EXPECTED_SCHEMA_HASHES: [SchemaHash; 5] = [ SchemaHash::from_hex("cc92cb332410e6f63036b52cf953acb446c142d5c0fbbdbd6d3b4f466510b210"), @@ -61,33 +60,25 @@ fn migration_004_validity_intervals_backfills_valid_until() -> Result<()> { migrate_database(&database_filepath)?; - let mut conn = SqliteConnection::establish(database_path_str)?; + let db = TestDb::open(&database_filepath); - let accounts: Vec<(i64, i64)> = OrderDsl::order( - SelectDsl::select( - schema::accounts::table, - (schema::accounts::block_num, schema::accounts::valid_until), - ), - schema::accounts::block_num.asc(), - ) - .load(&mut conn)?; + let accounts = db.read::<_, DatabaseError, _>(|tx| { + tx.query( + "SELECT block_num, valid_until FROM accounts ORDER BY block_num ASC", + &[], + |row| Ok((row.get::(0)?, row.get::(1)?)), + ) + })?; pretty_assertions::assert_eq!(accounts, vec![(1, 5), (5, VALID_FOREVER)]); - let vault: Vec<(Vec, i64, i64)> = OrderDsl::order( - SelectDsl::select( - schema::account_vault_assets::table, - ( - schema::account_vault_assets::vault_key, - schema::account_vault_assets::block_num, - schema::account_vault_assets::valid_until, - ), - ), - ( - schema::account_vault_assets::vault_key.asc(), - schema::account_vault_assets::block_num.asc(), - ), - ) - .load(&mut conn)?; + let vault = db.read::<_, DatabaseError, _>(|tx| { + tx.query( + "SELECT vault_key, block_num, valid_until FROM account_vault_assets \ + ORDER BY vault_key ASC, block_num ASC", + &[], + |row| Ok((row.get::>(0)?, row.get::(1)?, row.get::(2)?)), + ) + })?; pretty_assertions::assert_eq!( vault, vec![ @@ -97,17 +88,13 @@ fn migration_004_validity_intervals_backfills_valid_until() -> Result<()> { ] ); - let storage: Vec<(i64, i64)> = OrderDsl::order( - SelectDsl::select( - schema::account_storage_map_values::table, - ( - schema::account_storage_map_values::block_num, - schema::account_storage_map_values::valid_until, - ), - ), - schema::account_storage_map_values::block_num.asc(), - ) - .load(&mut conn)?; + let storage = db.read::<_, DatabaseError, _>(|tx| { + tx.query( + "SELECT block_num, valid_until FROM account_storage_map_values ORDER BY block_num ASC", + &[], + |row| Ok((row.get::(0)?, row.get::(1)?)), + ) + })?; pretty_assertions::assert_eq!(storage, vec![(1, 5), (5, VALID_FOREVER)]); Ok(()) diff --git a/crates/store/src/db/mod.rs b/crates/store/src/db/mod.rs index 2155eeaf9c..8cce01e44a 100644 --- a/crates/store/src/db/mod.rs +++ b/crates/store/src/db/mod.rs @@ -1,12 +1,11 @@ use std::collections::{BTreeMap, BTreeSet, HashSet}; use std::mem::size_of; use std::num::NonZeroUsize; -use std::ops::{Deref, DerefMut}; use std::path::{Path, PathBuf}; use std::sync::Arc; use anyhow::Context; -use diesel::{Connection, SqliteConnection}; +use miden_node_db::sqlite::{DbReader, DbWriter}; use miden_node_proto::domain::account::AccountInfo; use miden_node_utils::limiter::{ MAX_RESPONSE_PAYLOAD_BYTES, @@ -35,21 +34,17 @@ use miden_protocol::note::{ Nullifier, }; use miden_protocol::transaction::TransactionHeader; -use miden_protocol::utils::serde::Deserializable; use tracing::info; use crate::db::migrations::{migrate_database, verify_latest_schema}; -use crate::db::models::conv::SqlTypeConvert; -use crate::db::models::queries; -pub use crate::db::models::queries::{ +pub use crate::db::queries::{ AccountCommitmentsPage, + HISTORICAL_BLOCK_RETENTION, NullifiersPage, + PrecomputedPublicAccountState, + PrecomputedPublicAccountStates, PublicAccountIdsPage, PublicAccountStateRootsPage, -}; -use crate::db::models::queries::{ - BlockHeaderCommitment, - PrecomputedPublicAccountStates, StorageMapValuesPage, }; use crate::errors::{DatabaseError, NoteSyncError}; @@ -68,10 +63,18 @@ mod migrations; #[cfg(test)] pub(crate) use migrations::bootstrap_database; +#[cfg(test)] +mod test_db; +#[cfg(test)] +pub(crate) use test_db::TestDb; + #[cfg(test)] mod tests; -pub(crate) mod models; +/// Query functions on the `miden-node-db` SQLite framework. +pub(crate) mod queries; + +mod utils; /// [diesel](https://diesel.rs) generated schema /// @@ -98,22 +101,28 @@ impl Default for DatabaseOptions { /// The Store's database. /// -/// Extends the underlying [`miden_node_db::Db`] type with functionality specific to the Store. +/// Owns the writer and reader handles of the `miden-node-db` SQLite framework: every write +/// serializes on the single writer connection, while reads run concurrently on the reader pool. pub struct Db { - db: miden_node_db::Db, + writer: DbWriter, + reader: DbReader, } -impl Deref for Db { - type Target = miden_node_db::Db; +/// The commitment of a [`BlockHeader`], stored alongside the header it belongs to. +/// +/// Keeping it in its own column lets the chain MMR be rebuilt at startup without deserializing +/// every header. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(transparent)] +pub struct BlockHeaderCommitment(pub(crate) Word); - fn deref(&self) -> &Self::Target { - &self.db +impl BlockHeaderCommitment { + pub fn new(header: &BlockHeader) -> Self { + Self(header.commitment()) } -} -impl DerefMut for Db { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.db + pub fn word(self) -> Word { + self.0 } } @@ -128,18 +137,6 @@ pub struct AccountVaultValue { pub asset: Option, } -impl AccountVaultValue { - pub fn from_raw_row(row: (i64, Vec, Option>)) -> Result { - let (block_num, vault_key, asset) = row; - let vault_key = Word::read_from_bytes(&vault_key)?; - Ok(Self { - block_num: BlockNumber::from_raw_sql(block_num)?, - vault_key: AssetId::try_from(vault_key)?, - asset: asset.map(|b| Asset::read_from_bytes(&b)).transpose()?, - }) - } -} - #[derive(Debug, PartialEq)] pub struct NullifierInfo { pub nullifier: Nullifier, @@ -212,28 +209,29 @@ impl Db { fields(path=%database_filepath.display()) err, )] - pub fn bootstrap(database_filepath: PathBuf, genesis: GenesisBlock) -> anyhow::Result<()> { + pub async fn bootstrap( + database_filepath: PathBuf, + genesis: GenesisBlock, + ) -> anyhow::Result<()> { migrations::bootstrap_database(&database_filepath) .context("failed to bootstrap database schema")?; - let mut conn: SqliteConnection = diesel::sqlite::SqliteConnection::establish( - database_filepath.to_str().context("database filepath is invalid")?, - ) - .context("failed to open a database connection")?; - - miden_node_db::configure_connection_on_creation(&mut conn)?; + let (writer, _reader) = miden_node_db::sqlite::open(&database_filepath) + .context("failed to open a database connection")?; // Insert genesis block data. let genesis_block = genesis.into_inner(); - conn.transaction(move |conn| { - models::queries::apply_block( - conn, - &genesis_block, - &[], - &PrecomputedPublicAccountStates::new(), - ) - }) - .context("failed to insert genesis block")?; + writer + .write::<_, DatabaseError, _>("insert genesis block", move |tx| { + queries::apply_block( + tx, + &genesis_block, + &[], + &PrecomputedPublicAccountStates::new(), + ) + }) + .await + .context("failed to insert genesis block")?; Ok(()) } @@ -257,7 +255,8 @@ impl Db { ) -> Result { verify_latest_schema(&database_filepath)?; - let db = miden_node_db::Db::new_with_pool_size(&database_filepath, connection_pool_size)?; + let (writer, reader) = + miden_node_db::sqlite::open_with_pool_size(&database_filepath, connection_pool_size)?; info!( target: LOG_TARGET, sqlite= %database_filepath.display(), @@ -265,7 +264,13 @@ impl Db { "Connected to the database" ); - Ok(Self { db }) + Ok(Self { writer, reader }) + } + + /// The write handle, for tests that need to seed or corrupt rows no production method writes. + #[cfg(test)] + pub(crate) fn writer(&self) -> &DbWriter { + &self.writer } /// Applies all pending migrations to an existing DB. @@ -288,10 +293,11 @@ impl Db { page_size: std::num::NonZeroUsize, after_nullifier: Option, ) -> Result { - self.transact("read nullifiers paged", move |conn| { - queries::select_nullifiers_paged(conn, page_size, after_nullifier) - }) - .await + self.reader + .read("read nullifiers paged", move |tx| { + queries::select_nullifiers_paged(tx, page_size, after_nullifier) + }) + .await } /// Loads the nullifiers that match the prefixes from the DB. @@ -313,17 +319,18 @@ impl Db { let block_range = block_range.into_inner(); assert_eq!(prefix_len, 16, "Only 16-bit prefixes are supported"); - self.transact("nullifieres by prefix", move |conn| { - let nullifier_prefixes = - Vec::from_iter(nullifier_prefixes.into_iter().map(|prefix| prefix as u16)); - queries::select_nullifiers_by_prefix( - conn, - prefix_len as u8, - &nullifier_prefixes[..], - block_range, - ) - }) - .await + self.reader + .read("nullifieres by prefix", move |tx| { + let nullifier_prefixes = + Vec::from_iter(nullifier_prefixes.into_iter().map(|prefix| prefix as u16)); + queries::select_nullifiers_by_prefix( + tx, + prefix_len as u8, + &nullifier_prefixes[..], + block_range, + ) + }) + .await } /// Search for a [`BlockHeader`] from the database by its `block_num`. @@ -338,14 +345,14 @@ impl Db { &self, maybe_block_number: Option, ) -> Result> { - self.transact("block headers by block number", move |conn| { - let val = queries::select_block_header_by_block_num( - conn, - maybe_block_number.map(|block_number| *block_number), - )?; - Ok(val) - }) - .await + self.reader + .read("block headers by block number", move |tx| { + queries::select_block_header_by_block_num( + tx, + maybe_block_number.map(|block_number| *block_number), + ) + }) + .await } /// Search for a [`BlockHeader`] and its [`BlockSignatures`] from the database by its @@ -359,12 +366,11 @@ impl Db { &self, block_number: ScopedBlockNum, ) -> Result> { - self.transact("block headers and signatures by block number", move |conn| { - let val = - queries::select_block_header_and_signatures_by_block_num(conn, *block_number)?; - Ok(val) - }) - .await + self.reader + .read("block headers and signatures by block number", move |tx| { + queries::select_block_header_and_signatures_by_block_num(tx, *block_number) + }) + .await } /// Loads multiple block headers from the DB. @@ -377,11 +383,11 @@ impl Db { &self, blocks: impl Iterator + Send + 'static, ) -> Result> { - self.transact("block headers from given block numbers", move |conn| { - let raw = queries::select_block_headers(conn, blocks.map(|block| *block))?; - Ok(raw) - }) - .await + self.reader + .read("block headers from given block numbers", move |tx| { + queries::select_block_headers(tx, blocks.map(|block| *block)) + }) + .await } /// Loads all the block headers from the DB. @@ -391,11 +397,9 @@ impl Db { err, )] pub async fn select_all_block_header_commitments(&self) -> Result> { - self.transact("all block headers", |conn| { - let raw = queries::select_all_block_header_commitments(conn)?; - Ok(raw) - }) - .await + self.reader + .read("all block headers", queries::select_all_block_header_commitments) + .await } /// Returns a page of account commitments for tree rebuilding. @@ -409,10 +413,11 @@ impl Db { page_size: std::num::NonZeroUsize, after_account_id: Option, ) -> Result { - self.transact("read account commitments paged", move |conn| { - queries::select_account_commitments_paged(conn, page_size, after_account_id) - }) - .await + self.reader + .read("read account commitments paged", move |tx| { + queries::select_account_commitments_paged(tx, page_size, after_account_id) + }) + .await } /// Returns a page of public account IDs for forest rebuilding. @@ -426,10 +431,11 @@ impl Db { page_size: std::num::NonZeroUsize, after_account_id: Option, ) -> Result { - self.transact("read public account IDs paged", move |conn| { - queries::select_public_account_ids_paged(conn, page_size, after_account_id) - }) - .await + self.reader + .read("read public account IDs paged", move |tx| { + queries::select_public_account_ids_paged(tx, page_size, after_account_id) + }) + .await } /// Returns a page of public account state roots for forest consistency verification. @@ -443,10 +449,11 @@ impl Db { page_size: std::num::NonZeroUsize, after_account_id: Option, ) -> Result { - self.transact("read public account state roots paged", move |conn| { - queries::select_public_account_state_roots_paged(conn, page_size, after_account_id) - }) - .await + self.reader + .read("read public account state roots paged", move |tx| { + queries::select_public_account_state_roots_paged(tx, page_size, after_account_id) + }) + .await } /// Loads public account details from the DB. @@ -456,7 +463,8 @@ impl Db { err, )] pub async fn select_account(&self, id: AccountId) -> Result { - self.transact("Get account details", move |conn| queries::select_account(conn, id)) + self.reader + .read("Get account details", move |tx| queries::select_account(tx, id)) .await } @@ -470,10 +478,11 @@ impl Db { &self, account_ids: Vec, ) -> Result> { - self.transact("Filter network accounts subset", move |conn| { - queries::select_network_accounts_subset(conn, &account_ids) - }) - .await + self.reader + .read("Filter network accounts subset", move |tx| { + queries::select_network_accounts_subset(tx, &account_ids) + }) + .await } /// Queries the account code by its commitment hash. @@ -486,10 +495,11 @@ impl Db { &self, code_commitment: Word, ) -> Result>> { - self.transact("Get account code by commitment", move |conn| { - queries::select_account_code_by_commitment(conn, code_commitment) - }) - .await + self.reader + .read("Get account code by commitment", move |tx| { + queries::select_account_code_by_commitment(tx, code_commitment) + }) + .await } /// Queries the account header and storage header for a specific account at a block. @@ -504,12 +514,13 @@ impl Db { account_id: AccountId, block_num: ScopedBlockNum, ) -> Result> { - self.transact("Get account header with storage header at block", move |conn| { - queries::select_account_header_with_storage_header_at_block( - conn, account_id, *block_num, - ) - }) - .await + self.reader + .read("Get account header with storage header at block", move |tx| { + queries::select_account_header_with_storage_header_at_block( + tx, account_id, *block_num, + ) + }) + .await } #[miden_instrument( @@ -523,10 +534,16 @@ impl Db { note_tags: Arc<[u32]>, ) -> Result, NoteSyncError> { let block_range = block_range.into_inner(); - self.transact("notes sync task", move |conn| { - queries::get_note_sync_multi(conn, ¬e_tags, block_range, MAX_RESPONSE_PAYLOAD_BYTES) - }) - .await + self.reader + .read("notes sync task", move |tx| { + queries::get_note_sync_multi( + tx, + ¬e_tags, + block_range, + MAX_RESPONSE_PAYLOAD_BYTES, + ) + }) + .await } /// Loads all the [`miden_protocol::note::Note`]s matching a certain [`NoteId`] from the @@ -537,10 +554,9 @@ impl Db { err, )] pub async fn select_notes_by_id(&self, note_ids: Vec) -> Result> { - self.transact("note by id", move |conn| { - queries::select_notes_by_id(conn, note_ids.as_slice()) - }) - .await + self.reader + .read("note by id", move |tx| queries::select_notes_by_id(tx, note_ids.as_slice())) + .await } /// Returns all note commitments from the DB that match the provided ones and were committed at @@ -555,14 +571,15 @@ impl Db { note_commitments: Vec, up_to_block: ScopedBlockNum, ) -> Result> { - self.transact("note by commitment", move |conn| { - queries::select_existing_note_commitments( - conn, - note_commitments.as_slice(), - *up_to_block, - ) - }) - .await + self.reader + .read("note by commitment", move |tx| { + queries::select_existing_note_commitments( + tx, + note_commitments.as_slice(), + *up_to_block, + ) + }) + .await } /// Loads inclusion proofs for notes matching the given note commitments that were committed at @@ -577,10 +594,11 @@ impl Db { note_commitments: BTreeSet, up_to_block: ScopedBlockNum, ) -> Result> { - self.transact("block note inclusion proofs by commitment", move |conn| { - models::queries::select_note_inclusion_proofs(conn, ¬e_commitments, *up_to_block) - }) - .await + self.reader + .read("block note inclusion proofs by commitment", move |tx| { + queries::select_note_inclusion_proofs(tx, ¬e_commitments, *up_to_block) + }) + .await } /// Inserts the data of a new block into the DB. @@ -610,29 +628,50 @@ impl Db { unresolved_note_nullifiers: Vec, prune_tip: BlockNumber, ) -> Result> { - self.transact("apply block", move |conn| { - models::queries::apply_block(conn, &signed_block, ¬es, &precomputed_public_states)?; - models::queries::prune_history(conn, prune_tip)?; - - let mut resolved_note_ids = BTreeMap::new(); - for chunk in unresolved_note_nullifiers.chunks(QueryParamNoteCommitmentLimit::LIMIT) { - match queries::select_note_ids_by_nullifier(conn, chunk) { - Ok(note_ids) => resolved_note_ids.extend(note_ids), - Err(err) => { - tracing::warn!( - target: COMPONENT, - %err, - nullifiers.count = chunk.len(), - "Failed to resolve consumed note IDs for lifecycle events", - ); - break; - }, - } + self.writer + .write::<_, DatabaseError, _>("apply block", move |tx| { + queries::apply_block(tx, &signed_block, ¬es, &precomputed_public_states)?; + queries::prune_history(tx, prune_tip)?; + Ok(()) + }) + .await?; + + Ok(self.resolve_consumed_note_ids(unresolved_note_nullifiers).await) + } + + /// Maps consumed nullifiers back to their note IDs for lifecycle events, on a best-effort basis. + /// + /// A failed lookup is logged and abandoned: the caller uses this only for reporting. + async fn resolve_consumed_note_ids( + &self, + nullifiers: Vec, + ) -> BTreeMap { + let mut resolved_note_ids = BTreeMap::new(); + for chunk in nullifiers.chunks(QueryParamNoteCommitmentLimit::LIMIT) { + let chunk = chunk.to_vec(); + let count = chunk.len(); + let result = self + .reader + .read("resolve consumed note ids", move |tx| { + queries::select_note_ids_by_nullifier(tx, &chunk) + }) + .await; + + match result { + Ok(note_ids) => resolved_note_ids.extend(note_ids), + Err(err) => { + tracing::warn!( + target: COMPONENT, + %err, + nullifiers.count = count, + "Failed to resolve consumed note IDs for lifecycle events", + ); + break; + }, } + } - Ok(resolved_note_ids) - }) - .await + resolved_note_ids } /// Selects storage map values for syncing storage maps for a specific account ID. @@ -648,15 +687,16 @@ impl Db { let block_range = block_range.into_inner(); let entries_limit = entries_limit.unwrap_or_else(default_storage_map_entries_limit); - self.transact("select storage map sync values", move |conn| { - models::queries::select_account_storage_map_values_paged( - conn, - account_id, - block_range, - entries_limit, - ) - }) - .await + self.reader + .read("select storage map sync values", move |tx| { + queries::select_account_storage_map_values_paged( + tx, + account_id, + block_range, + entries_limit, + ) + }) + .await } /// Reconstructs storage map details from the database for a specific slot at a block. @@ -766,10 +806,11 @@ impl Db { account_id: AccountId, block_num: ScopedBlockNum, ) -> Result, DatabaseError> { - self.transact("select account vault at block", move |conn| { - queries::select_account_vault_at_block(conn, account_id, *block_num) - }) - .await + self.reader + .read("select account vault at block", move |tx| { + queries::select_account_vault_at_block(tx, account_id, *block_num) + }) + .await } pub async fn get_account_vault_sync( @@ -778,18 +819,18 @@ impl Db { block_range: ScopedBlockRange, ) -> Result<(BlockNumber, Vec)> { let block_range = block_range.into_inner(); - self.transact("account vault sync", move |conn| { - queries::select_account_vault_assets(conn, account_id, block_range) - }) - .await + self.reader + .read("account vault sync", move |tx| { + queries::select_account_vault_assets(tx, account_id, block_range) + }) + .await } /// Returns the script for a note by its root. pub async fn select_note_script_by_root(&self, root: Word) -> Result> { - self.transact("note script by root", move |conn| { - queries::select_note_script_by_root(conn, root) - }) - .await + self.reader + .read("note script by root", move |tx| queries::select_note_script_by_root(tx, root)) + .await } /// Returns the complete transaction records for the specified accounts within the specified @@ -804,9 +845,10 @@ impl Db { block_range: ScopedBlockRange, ) -> Result<(BlockNumber, Vec)> { let block_range = block_range.into_inner(); - self.transact("full transactions records", move |conn| { - queries::select_transactions_records(conn, &account_ids, block_range) - }) - .await + self.reader + .read("full transactions records", move |tx| { + queries::select_transactions_records(tx, &account_ids, block_range) + }) + .await } } diff --git a/crates/store/src/db/models/conv.rs b/crates/store/src/db/models/conv.rs deleted file mode 100644 index abd26debca..0000000000 --- a/crates/store/src/db/models/conv.rs +++ /dev/null @@ -1,240 +0,0 @@ -//! Central place to define conversion from and to database primitive types -//! -//! Eventually, all of them should have types and we can implement a trait for them -//! rather than function pairs. -//! -//! Notice: All of them are infallible. The invariant is a sane content of the database -//! and humans ensure the sanity of casts. -//! -//! Notice: Keep in mind if you _need_ to expand the datatype, only if you require sorting this is -//! mandatory! -//! -//! Notice: Ensure you understand what casting does at the bit-level before changing any. -//! -//! Notice: Changing any of these are _backwards-incompatible_ changes that are not caught/covered -//! by migrations! - -#![expect( - clippy::inline_always, - reason = "Just unification helpers of 1-2 lines of casting types" -)] -#![expect( - dead_code, - reason = "Not all converters are used bidirectionally, however, keeping them is a good thing" -)] -#![expect( - clippy::cast_sign_loss, - reason = "This is the one file where we map the signed database types to the working types" -)] -#![expect( - clippy::cast_possible_wrap, - reason = "We will not approach the item count where i64 and usize casting will cause issues - on relevant platforms" -)] - -use miden_crypto::Word; -use miden_crypto::utils::Deserializable; -use miden_protocol::Felt; -use miden_protocol::account::{StorageSlotName, StorageSlotType}; -use miden_protocol::block::{BlockHeader, BlockNumber}; -use miden_protocol::note::NoteTag; - -use crate::db::models::queries::{BlockHeaderCommitment, NetworkAccountType}; - -#[derive(Debug, thiserror::Error)] -#[error("failed to convert from database type {from_type} into {into_type}")] -pub struct DatabaseTypeConversionError { - source: Box, - from_type: &'static str, - into_type: &'static str, -} - -/// Convert from and to it's database representation and back -/// -/// We do not assume sanity of DB types. -pub trait SqlTypeConvert: Sized { - type Raw: Sized; - - fn to_raw_sql(self) -> Self::Raw; - fn from_raw_sql(_raw: Self::Raw) -> Result; - - fn map_err( - source: E, - ) -> DatabaseTypeConversionError { - DatabaseTypeConversionError { - source: Box::new(source), - from_type: std::any::type_name::(), - into_type: std::any::type_name::(), - } - } -} - -impl SqlTypeConvert for BlockHeaderCommitment { - type Raw = Vec; - fn from_raw_sql( - raw: Self::Raw, - ) -> Result { - let inner = - ::read_from_bytes(raw.as_slice()).map_err(Self::map_err)?; - Ok(BlockHeaderCommitment(inner)) - } - fn to_raw_sql(self) -> Self::Raw { - self.0.as_bytes().to_vec() - } -} - -impl SqlTypeConvert for BlockHeader { - type Raw = Vec; - - fn from_raw_sql(raw: Self::Raw) -> Result { - ::read_from_bytes(raw.as_slice()).map_err(Self::map_err) - } - - fn to_raw_sql(self) -> Self::Raw { - miden_crypto::utils::Serializable::to_bytes(&self) - } -} - -impl SqlTypeConvert for NetworkAccountType { - type Raw = i32; - - fn to_raw_sql(self) -> Self::Raw { - match self { - NetworkAccountType::None => 0, - NetworkAccountType::Network => 1, - } - } - - fn from_raw_sql(raw: Self::Raw) -> Result { - #[derive(Debug, thiserror::Error)] - #[error("invalid network account type value {0}")] - struct ValueError(i32); - - match raw { - 0 => Ok(Self::None), - 1 => Ok(Self::Network), - other => Err(Self::map_err(ValueError(other))), - } - } -} - -impl SqlTypeConvert for BlockNumber { - type Raw = i64; - - fn from_raw_sql(raw: Self::Raw) -> Result { - u32::try_from(raw).map(BlockNumber::from).map_err(Self::map_err) - } - - fn to_raw_sql(self) -> Self::Raw { - i64::from(self.as_u32()) - } -} - -impl SqlTypeConvert for NoteTag { - type Raw = i32; - - #[inline(always)] - fn from_raw_sql(raw: Self::Raw) -> Result { - #[expect(clippy::cast_sign_loss)] - Ok(NoteTag::new(raw as u32)) - } - - #[inline(always)] - fn to_raw_sql(self) -> Self::Raw { - self.as_u32() as i32 - } -} - -impl SqlTypeConvert for StorageSlotType { - type Raw = i32; - - #[inline(always)] - fn from_raw_sql(raw: Self::Raw) -> Result { - #[derive(Debug, thiserror::Error)] - #[error("invalid storage slot type value {0}")] - struct ValueError(i32); - - Ok(match raw { - 0 => StorageSlotType::Value, - 1 => StorageSlotType::Map, - invalid => { - return Err(Self::map_err(ValueError(invalid))); - }, - }) - } - - #[inline(always)] - fn to_raw_sql(self) -> Self::Raw { - match self { - StorageSlotType::Value => 0, - StorageSlotType::Map => 1, - } - } -} - -impl SqlTypeConvert for StorageSlotName { - type Raw = String; - - fn from_raw_sql(raw: Self::Raw) -> Result { - StorageSlotName::new(raw).map_err(Self::map_err) - } - - fn to_raw_sql(self) -> Self::Raw { - String::from(self) - } -} - -// Raw type conversions - eventually introduce wrapper types -// =========================================================== - -#[inline(always)] -pub(crate) fn raw_sql_to_nullifier_prefix(raw: i32) -> u16 { - debug_assert!(raw >= 0); - raw as u16 -} -#[inline(always)] -pub(crate) fn nullifier_prefix_to_raw_sql(prefix: u16) -> i32 { - i32::from(prefix) -} - -#[inline(always)] -pub(crate) fn raw_sql_to_nonce(raw: i64) -> Felt { - debug_assert!(raw >= 0); - // SAFETY: In the store we write `Felt::as_canonical_u64() as i64`, so `raw` is the bit - // reinterpretation of a u64 in the field. Casting back via `raw as u64` recovers that same - // canonical value, which is always a valid (already reduced) field element, so - // `Felt::new_unchecked` is sound. - Felt::new_unchecked(raw as u64) -} -#[inline(always)] -pub(crate) fn nonce_to_raw_sql(nonce: Felt) -> i64 { - nonce.as_canonical_u64() as i64 -} - -#[inline(always)] -pub(crate) fn raw_sql_to_fungible_delta(raw: i64) -> i64 { - raw -} -#[inline(always)] -pub(crate) fn fungible_delta_to_raw_sql(delta: i64) -> i64 { - delta -} - -#[inline(always)] -#[expect(clippy::cast_sign_loss)] -pub(crate) fn raw_sql_to_note_type(raw: i32) -> u8 { - raw as u8 -} -#[inline(always)] -pub(crate) fn note_type_to_raw_sql(note_type: u8) -> i32 { - i32::from(note_type) -} - -#[inline(always)] -pub(crate) fn raw_sql_to_idx(raw: i32) -> usize { - raw as usize -} -#[inline(always)] -pub(crate) fn idx_to_raw_sql(idx: usize) -> i32 { - idx as i32 -} diff --git a/crates/store/src/db/models/mod.rs b/crates/store/src/db/models/mod.rs deleted file mode 100644 index e968480845..0000000000 --- a/crates/store/src/db/models/mod.rs +++ /dev/null @@ -1,23 +0,0 @@ -//! Defines models for usage with the diesel API -//! -//! Note: `select` can either be used as -//! `SelectDsl::select(schema::foo::table, (schema::foo::some_cool_id, ))` -//! or -//! `SelectDsl::select(schema::foo::table, FooRawRow::as_selectable())`. -//! -//! The former can be used to avoid declaring extra types, while the latter -//! is better if a full row is in need of loading and avoids duplicate -//! specification. -//! -//! Note: The fully qualified syntax yields for _much_ better errors. -//! The first step in debugging should always be using the fully qualified -//! calling syntext when dealing with diesel. - -use crate::errors::DatabaseError; - -pub(crate) mod conv; - -pub mod queries; -pub(crate) mod utils; - -pub(crate) use utils::*; diff --git a/crates/store/src/db/models/queries/accounts.rs b/crates/store/src/db/models/queries/accounts.rs deleted file mode 100644 index 90a0401225..0000000000 --- a/crates/store/src/db/models/queries/accounts.rs +++ /dev/null @@ -1,1836 +0,0 @@ -use std::collections::{BTreeMap, HashMap, HashSet}; -use std::num::NonZeroUsize; -use std::ops::RangeInclusive; - -use diesel::prelude::{Queryable, QueryableByName}; -use diesel::query_dsl::methods::SelectDsl; -use diesel::sqlite::Sqlite; -use diesel::{ - AsChangeset, - BoolExpressionMethods, - ExpressionMethods, - Insertable, - JoinOnDsl, - NullableExpressionMethods, - OptionalExtension, - QueryDsl, - RunQueryDsl, - Selectable, - SelectableHelper, - SqliteConnection, -}; -use miden_node_proto::domain::account::{AccountInfo, AccountSummary, AccountVaultDetails}; -use miden_node_utils::limiter::{ - MAX_RESPONSE_PAYLOAD_BYTES, - QueryParamAccountIdLimit, - QueryParamLimiter, -}; -use miden_node_utils::tracing::miden_instrument; -use miden_protocol::Word; -use miden_protocol::account::{ - Account, - AccountCode, - AccountId, - AccountPatch, - AccountStorage, - AccountStorageHeader, - AccountUpdateDetails, - StorageMap, - StorageMapKey, - StorageMapPatchEntries, - StorageSlot, - StorageSlotContent, - StorageSlotName, - StorageSlotType, -}; -use miden_protocol::asset::{Asset, AssetId, AssetVault}; -use miden_protocol::block::{BlockAccountUpdate, BlockNumber}; -use miden_protocol::utils::serde::{Deserializable, Serializable}; -use miden_standards::account::auth::NetworkAccount; - -use crate::COMPONENT; -use crate::db::models::conv::{SqlTypeConvert, nonce_to_raw_sql, raw_sql_to_nonce}; -#[cfg(test)] -use crate::db::models::vec_raw_try_into; -use crate::db::{AccountVaultValue, schema}; -use crate::errors::DatabaseError; - -mod at_block; -pub(crate) use at_block::select_account_header_with_storage_header_at_block; - -mod delta; -use delta::{ - AccountStateForInsert, - LatestAccountStateRow, - PartialAccountState, - PrecomputedFullAccountState, - apply_storage_patch_with_roots, - select_latest_account_state, -}; - -#[cfg(test)] -mod tests; - -type StorageMapValueRow = (i64, String, Vec, Vec); -type StorageHeaderWithEntries = - (AccountStorageHeader, HashMap>); - -/// Sentinel `valid_until` value marking a row as the current, open-ended version of its key. -/// -/// Versioned rows (`accounts`, `account_vault_assets`, `account_storage_map_values`) are -/// applicable for blocks in `[block_num, valid_until)`; updating a key closes the previous row's -/// interval by setting its `valid_until` to the new row's `block_num`. The open end is `i64::MAX` -/// rather than NULL so every validity predicate is a single range comparison that partial indexes -/// can serve. -pub(crate) const VALID_FOREVER: i64 = i64::MAX; - -// NETWORK ACCOUNT TYPE -// ================================================================================================ - -/// Classifies accounts for database storage based on whether they are network accounts. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum NetworkAccountType { - /// Not a network account. - None, - /// A network account. - Network, -} - -// ACCOUNT CODE -// ================================================================================================ - -/// Select account code by its commitment hash from the `account_codes` table. -/// -/// # Returns -/// -/// The account code bytes if found, or `None` if no code exists with that commitment. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT code FROM account_codes WHERE code_commitment = ?1 -/// ``` -pub(crate) fn select_account_code_by_commitment( - conn: &mut SqliteConnection, - code_commitment: Word, -) -> Result>, DatabaseError> { - use schema::account_codes; - - let code_commitment_bytes = code_commitment.to_bytes(); - - let result: Option> = SelectDsl::select( - account_codes::table.filter(account_codes::code_commitment.eq(&code_commitment_bytes)), - account_codes::code, - ) - .first(conn) - .optional()?; - - Ok(result) -} - -// ACCOUNT RETRIEVAL -// ================================================================================================ - -/// Select account by ID from the DB using the given [`SqliteConnection`]. -/// -/// # Returns -/// -/// The latest account info, or an error. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// accounts.account_id, -/// accounts.account_commitment, -/// accounts.block_num -/// FROM -/// accounts -/// WHERE -/// account_id = ?1 -/// AND valid_until = {VALID_FOREVER} -/// ``` -pub(crate) fn select_account( - conn: &mut SqliteConnection, - account_id: AccountId, -) -> Result { - let raw = SelectDsl::select(schema::accounts::table, AccountSummaryRaw::as_select()) - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::valid_until.eq(VALID_FOREVER)) - .get_result::(conn) - .optional()? - .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; - - let summary: AccountSummary = raw.try_into()?; - - // Backfill account details from database For private accounts, we don't store full details in - // the database - let details = if account_id.is_public() { - Some(select_full_account(conn, account_id)?) - } else { - None - }; - - Ok(AccountInfo { summary, details }) -} - -/// Reconstruct full Account from database tables for the latest account state -/// -/// This function queries the database tables to reconstruct a complete Account object: -/// - Code from `account_codes` table -/// - Nonce and storage header from `accounts` table -/// - Storage map entries from `account_storage_map_values` table -/// - Vault from `account_vault_assets` table -/// -/// # Note -/// -/// A stop-gap solution to retain store API and construct `AccountInfo` types. -/// The function should ultimately be removed, and any queries be served from the -/// `State` which contains an `SmtForest` to serve the latest and most recent -/// historical data. -// TODO: remove eventually once refactoring is complete -pub(crate) fn select_full_account( - conn: &mut SqliteConnection, - account_id: AccountId, -) -> Result { - // Get account metadata (nonce, code_commitment) and code in a single join query - let joined = schema::accounts::table.inner_join(schema::account_codes::table.on( - schema::accounts::code_commitment.eq(schema::account_codes::code_commitment.nullable()), - )); - - let (nonce, code_bytes): (Option, Vec) = - SelectDsl::select(joined, (schema::accounts::nonce, schema::account_codes::code)) - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::valid_until.eq(VALID_FOREVER)) - .get_result(conn) - .optional()? - .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; - - let nonce = raw_sql_to_nonce(nonce.ok_or_else(|| { - DatabaseError::DataCorrupted(format!("No nonce found for account {account_id}")) - })?); - - let code = AccountCode::read_from_bytes(&code_bytes)?; - - // Reconstruct storage using existing helper function - let storage = select_latest_account_storage(conn, account_id)?; - - // Reconstruct vault from account_vault_assets table - let vault_entries: Vec<(Vec, Option>)> = SelectDsl::select( - schema::account_vault_assets::table, - (schema::account_vault_assets::vault_key, schema::account_vault_assets::asset), - ) - .filter(schema::account_vault_assets::account_id.eq(account_id.to_bytes())) - .filter(schema::account_vault_assets::valid_until.eq(VALID_FOREVER)) - .load(conn)?; - - let mut assets = Vec::new(); - for (_key_bytes, maybe_asset_bytes) in vault_entries { - if let Some(asset_bytes) = maybe_asset_bytes { - let asset = Asset::read_from_bytes(&asset_bytes)?; - assets.push(asset); - } - } - - let vault = AssetVault::new(&assets)?; - - Ok(Account::new(account_id, vault, storage, code, nonce, None)?) -} - -/// Page of account commitments returned by [`select_account_commitments_paged`]. -#[derive(Debug)] -pub struct AccountCommitmentsPage { - /// The account commitments in this page. - pub commitments: Vec<(AccountId, Word)>, - /// If `Some`, there are more results. Use this as the `after_account_id` for the next page. - pub next_cursor: Option, -} - -/// Selects account commitments with pagination. -/// -/// Returns up to `page_size` account commitments, starting after `after_account_id` if provided. -/// Results are ordered by `account_id` for stable pagination. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// account_id, -/// account_commitment -/// FROM -/// accounts -/// WHERE -/// valid_until = {VALID_FOREVER} -/// AND (account_id > :after_account_id OR :after_account_id IS NULL) -/// ORDER BY -/// account_id ASC -/// LIMIT :page_size + 1 -/// ``` -pub(crate) fn select_account_commitments_paged( - conn: &mut SqliteConnection, - page_size: NonZeroUsize, - after_account_id: Option, -) -> Result { - // Fetch one extra to determine if there are more results - #[expect(clippy::cast_possible_wrap)] - let limit = (page_size.get() + 1) as i64; - - let mut query = SelectDsl::select( - schema::accounts::table, - (schema::accounts::account_id, schema::accounts::account_commitment), - ) - .filter(schema::accounts::valid_until.eq(VALID_FOREVER)) - .order_by(schema::accounts::account_id.asc()) - .limit(limit) - .into_boxed(); - - if let Some(cursor) = after_account_id { - query = query.filter(schema::accounts::account_id.gt(cursor.to_bytes())); - } - - let raw = query.load::<(Vec, Vec)>(conn)?; - - let mut commitments = Result::, DatabaseError>::from_iter(raw.into_iter().map( - |(ref account, ref commitment)| { - Ok((AccountId::read_from_bytes(account)?, Word::read_from_bytes(commitment)?)) - }, - ))?; - - // If we got more than page_size, there are more results - let next_cursor = if commitments.len() > page_size.get() { - commitments.pop(); // Remove the extra element - commitments.last().map(|(id, _)| *id) - } else { - None - }; - - Ok(AccountCommitmentsPage { commitments, next_cursor }) -} - -/// Page of public account IDs returned by [`select_public_account_ids_paged`]. -#[derive(Debug)] -pub struct PublicAccountIdsPage { - /// The public account IDs in this page. - pub account_ids: Vec, - /// If `Some`, there are more results. Use this as the `after_account_id` for the next page. - pub next_cursor: Option, -} - -/// Latest account state forest roots for a public account. -#[derive(Debug)] -pub struct PublicAccountStateRoots { - pub account_id: AccountId, - pub vault_root: Word, - pub storage_header: AccountStorageHeader, -} - -/// Page of public account state roots returned by [`select_public_account_state_roots_paged`]. -#[derive(Debug)] -pub struct PublicAccountStateRootsPage { - /// The public account state roots in this page. - pub accounts: Vec, - /// If `Some`, there are more results. Use this as the `after_account_id` for the next page. - pub next_cursor: Option, -} - -/// Public account state commitments computed by the account state forest before SQLite writes. -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct PrecomputedPublicAccountState { - pub(crate) vault_root: Word, - pub(crate) storage_map_roots: BTreeMap, -} - -pub(crate) type PrecomputedPublicAccountStates = BTreeMap; - -/// Selects public account IDs with pagination. -/// -/// Returns up to `page_size` public account IDs, starting after `after_account_id` if provided. -/// Results are ordered by `account_id` for stable pagination. -/// -/// Public accounts are those with `AccountType::Public`. We identify them by checking -/// against the store. Public accounts store their `code_commitment`, while private accounts only -/// store the `account_commitment`. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// account_id -/// FROM -/// accounts -/// WHERE -/// valid_until = {VALID_FOREVER} -/// AND code_commitment IS NOT NULL -/// AND (account_id > :after_account_id OR :after_account_id IS NULL) -/// ORDER BY -/// account_id ASC -/// LIMIT :page_size + 1 -/// ``` -pub(crate) fn select_public_account_ids_paged( - conn: &mut SqliteConnection, - page_size: NonZeroUsize, - after_account_id: Option, -) -> Result { - #[expect(clippy::cast_possible_wrap)] - let limit = (page_size.get() + 1) as i64; - - let mut query = SelectDsl::select(schema::accounts::table, schema::accounts::account_id) - .filter(schema::accounts::valid_until.eq(VALID_FOREVER)) - .filter(schema::accounts::code_commitment.is_not_null()) - .order_by(schema::accounts::account_id.asc()) - .limit(limit) - .into_boxed(); - - if let Some(cursor) = after_account_id { - query = query.filter(schema::accounts::account_id.gt(cursor.to_bytes())); - } - - let raw = query.load::>(conn)?; - - let mut account_ids: Vec = Result::from_iter(raw.into_iter().map(|bytes| { - AccountId::read_from_bytes(&bytes).map_err(DatabaseError::DeserializationError) - }))?; - - // If we got more than page_size, there are more results - let next_cursor = if account_ids.len() > page_size.get() { - account_ids.pop(); // Remove the extra element - account_ids.last().copied() - } else { - None - }; - - Ok(PublicAccountIdsPage { account_ids, next_cursor }) -} - -/// Selects public account vault roots and storage headers with pagination. -/// -/// Returns up to `page_size` public account states, starting after `after_account_id` if provided. -/// Results are ordered by `account_id` for stable pagination. -/// -/// Public accounts are those with `AccountType::Public`. We identify them by checking -/// against the store. Public accounts store their `code_commitment`, while private accounts only -/// store the `account_commitment`. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// account_id, -/// vault_root, -/// storage_header -/// FROM -/// accounts -/// WHERE -/// valid_until = {VALID_FOREVER} -/// AND code_commitment IS NOT NULL -/// AND (account_id > :after_account_id OR :after_account_id IS NULL) -/// ORDER BY -/// account_id ASC -/// LIMIT :page_size + 1 -/// ``` -pub(crate) fn select_public_account_state_roots_paged( - conn: &mut SqliteConnection, - page_size: NonZeroUsize, - after_account_id: Option, -) -> Result { - #[expect(clippy::cast_possible_wrap)] - let limit = (page_size.get() + 1) as i64; - - let mut query = SelectDsl::select( - schema::accounts::table, - ( - schema::accounts::account_id, - schema::accounts::vault_root, - schema::accounts::storage_header, - ), - ) - .filter(schema::accounts::valid_until.eq(VALID_FOREVER)) - .filter(schema::accounts::code_commitment.is_not_null()) - .order_by(schema::accounts::account_id.asc()) - .limit(limit) - .into_boxed(); - - if let Some(cursor) = after_account_id { - query = query.filter(schema::accounts::account_id.gt(cursor.to_bytes())); - } - - let raw = query.load::<(Vec, Option>, Option>)>(conn)?; - - let mut accounts: Vec = Result::from_iter(raw.into_iter().map( - |(account_id_bytes, vault_root_bytes, storage_header_bytes)| { - let account_id = AccountId::read_from_bytes(&account_id_bytes) - .map_err(DatabaseError::DeserializationError)?; - let vault_root_bytes = vault_root_bytes.ok_or_else(|| { - DatabaseError::DataCorrupted(format!( - "public account {account_id} is missing a vault root" - )) - })?; - let storage_header_bytes = storage_header_bytes.ok_or_else(|| { - DatabaseError::DataCorrupted(format!( - "public account {account_id} is missing a storage header" - )) - })?; - - Ok::<_, DatabaseError>(PublicAccountStateRoots { - account_id, - vault_root: Word::read_from_bytes(&vault_root_bytes)?, - storage_header: AccountStorageHeader::read_from_bytes(&storage_header_bytes)?, - }) - }, - ))?; - - // If we got more than page_size, there are more results. - let next_cursor = if accounts.len() > page_size.get() { - accounts.pop(); - accounts.last().map(|account| account.account_id) - } else { - None - }; - - Ok(PublicAccountStateRootsPage { accounts, next_cursor }) -} - -/// Select account vault assets within a block range (inclusive). -/// -/// # Parameters -/// * `account_id`: Account ID to query -/// * `block_from`: Starting block number -/// * `block_to`: Ending block number -/// * Response payload size: 0 <= size <= 2MB -/// * Vault assets per response: 0 <= count <= (2MB / (2*Word + u32)) + 1 -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// block_num, -/// vault_key, -/// asset -/// FROM -/// account_vault_assets -/// WHERE -/// account_id = ?1 -/// AND block_num >= ?2 -/// AND block_num <= ?3 -/// ORDER BY -/// block_num ASC -/// LIMIT -/// ?4 -/// ``` -pub(crate) fn select_account_vault_assets( - conn: &mut SqliteConnection, - account_id: AccountId, - block_range: RangeInclusive, -) -> Result<(BlockNumber, Vec), DatabaseError> { - use schema::account_vault_assets as t; - // TODO: These limits should be given by the protocol. See miden-protocol/issues/1770 for more - // details - const ROW_OVERHEAD_BYTES: usize = 2 * size_of::() + size_of::(); // key + asset + block_num - const MAX_ROWS: usize = MAX_RESPONSE_PAYLOAD_BYTES / ROW_OVERHEAD_BYTES; - - if !account_id.is_public() { - return Err(DatabaseError::AccountNotPublic(account_id)); - } - - if block_range.is_empty() { - return Err(DatabaseError::InvalidBlockRange { - from: *block_range.start(), - to: *block_range.end(), - }); - } - - let raw: Vec<(i64, Vec, Option>)> = - SelectDsl::select(t::table, (t::block_num, t::vault_key, t::asset)) - .filter( - t::account_id - .eq(account_id.to_bytes()) - .and(t::block_num.ge(block_range.start().to_raw_sql())) - .and(t::block_num.le(block_range.end().to_raw_sql())), - ) - .order(t::block_num.asc()) - .limit(i64::try_from(MAX_ROWS + 1).expect("should fit within i64")) - .load::<(i64, Vec, Option>)>(conn)?; - - // If we got more rows than the limit, the last block may be incomplete so we drop it entirely - // and derive last_block_included from the remaining rows. - let (last_block_included, values) = if let Some(&(last_block_num, ..)) = raw.last() - && raw.len() > MAX_ROWS - { - let values = raw - .into_iter() - .take_while(|(bn, ..)| *bn != last_block_num) - .map(AccountVaultValue::from_raw_row) - .collect::, DatabaseError>>()?; - - let last_block_included = values.last().map_or(*block_range.start(), |v| v.block_num); - - (last_block_included, values) - } else { - ( - *block_range.end(), - raw.into_iter().map(AccountVaultValue::from_raw_row).collect::>()?, - ) - }; - - Ok((last_block_included, values)) -} - -/// Query vault assets at a specific block by finding the most recent update for each `vault_key`. -/// -/// Selects, per vault key, the row whose validity interval covers `block_num`: -/// ```sql -/// SELECT asset FROM account_vault_assets -/// WHERE account_id = ?1 AND block_num <= ?2 AND valid_until > ?2 -/// LIMIT ?3 -/// ``` -/// -/// The read is bounded to [`AccountVaultDetails::MAX_RETURN_ENTRIES`] + 1 rows so an over-the-limit -/// vault can be detected without materializing the whole set. -pub(crate) fn select_account_vault_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, -) -> Result, DatabaseError> { - use diesel::sql_types::{BigInt, Binary}; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - let limit_sql = - i64::try_from(AccountVaultDetails::MAX_RETURN_ENTRIES + 1).expect("should fit within i64"); - - let entries: Vec>> = diesel::sql_query( - r" - SELECT asset FROM account_vault_assets - WHERE account_id = ?1 AND block_num <= ?2 AND valid_until > ?2 - LIMIT ?3 - ", - ) - .bind::(&account_id_bytes) - .bind::(block_num_sql) - .bind::(limit_sql) - .load::(conn)? - .into_iter() - .map(|row| row.asset) - .collect(); - - // Convert to assets, filtering out deletions (None values) - let mut assets = Vec::new(); - for asset_bytes in entries.into_iter().flatten() { - let asset = Asset::read_from_bytes(&asset_bytes)?; - assets.push(asset); - } - - Ok(assets) -} - -#[derive(QueryableByName)] -struct AssetRow { - #[diesel(sql_type = diesel::sql_types::Nullable)] - asset: Option>, -} - -/// Select all accounts from the DB using the given [`SqliteConnection`]. -/// -/// # Returns -/// -/// A vector with accounts, or an error. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// accounts.account_id, -/// accounts.account_commitment, -/// accounts.block_num -/// FROM -/// accounts -/// WHERE -/// valid_until = {VALID_FOREVER} -/// ORDER BY -/// block_num ASC -/// ``` -#[cfg(test)] -pub(crate) fn select_all_accounts( - conn: &mut SqliteConnection, -) -> Result, DatabaseError> { - let raw = SelectDsl::select(schema::accounts::table, AccountSummaryRaw::as_select()) - .filter(schema::accounts::valid_until.eq(VALID_FOREVER)) - .order_by(schema::accounts::block_num.asc()) - .load::(conn)?; - - let summaries: Vec = vec_raw_try_into(raw)?; - - // Backfill account details from database - let account_infos = summaries - .into_iter() - .map(|summary| { - let account_id = summary.account_id; - let details = select_full_account(conn, account_id).ok(); - AccountInfo { summary, details } - }) - .collect(); - - Ok(account_infos) -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct StorageMapValue { - pub block_num: BlockNumber, - pub slot_name: StorageSlotName, - pub key: StorageMapKey, - pub value: Word, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct StorageMapValuesPage { - /// Highest block number included in `rows`. If the page is empty, this will be `block_from`. - pub last_block_included: BlockNumber, - /// Storage map values - pub values: Vec, -} - -impl StorageMapValue { - pub fn from_raw_row(row: StorageMapValueRow) -> Result { - let (block_num, slot_name, key, value) = row; - Ok(Self { - block_num: BlockNumber::from_raw_sql(block_num)?, - slot_name: StorageSlotName::from_raw_sql(slot_name)?, - key: StorageMapKey::read_from_bytes(&key)?, - value: Word::read_from_bytes(&value)?, - }) - } -} - -/// Select account storage map values from the DB using the given [`SqliteConnection`]. -/// -/// # Returns -/// -/// A vector of tuples containing `(block_num, slot, key, value)` for the given account. -/// Each row contains one of: -/// -/// - the historical value for a slot and key specifically on block `block_to` -/// - the latest updated value for the slot and key combination, alongside the block number in which -/// it was updated -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// block_num, -/// slot, -/// key, -/// value -/// FROM -/// account_storage_map_values -/// WHERE -/// account_id = ?1 -/// AND block_num >= ?2 -/// AND block_num <= ?3 -/// ORDER BY -/// block_num ASC -/// LIMIT -/// ?4 -/// ``` -/// Select account storage map values within a block range (inclusive). -/// -/// ## Parameters -/// -/// * `account_id`: Account ID to query -/// * `block_range`: Range of block numbers (inclusive) -/// -/// ## Response -/// -/// * Response payload size: 0 <= size <= 2MB -/// * Storage map values per response: 0 <= count <= (2MB / (2*Word + u32 + u8)) + 1 -pub(crate) fn select_account_storage_map_values_paged( - conn: &mut SqliteConnection, - account_id: AccountId, - block_range: RangeInclusive, - limit: usize, -) -> Result { - use schema::account_storage_map_values as t; - - if !account_id.is_public() { - return Err(DatabaseError::AccountNotPublic(account_id)); - } - - if block_range.is_empty() { - return Err(DatabaseError::InvalidBlockRange { - from: *block_range.start(), - to: *block_range.end(), - }); - } - - let raw: Vec = - SelectDsl::select(t::table, (t::block_num, t::slot_name, t::key, t::value)) - .filter( - t::account_id - .eq(account_id.to_bytes()) - .and(t::block_num.ge(block_range.start().to_raw_sql())) - .and(t::block_num.le(block_range.end().to_raw_sql())), - ) - .order(t::block_num.asc()) - .limit(i64::try_from(limit + 1).expect("limit fits within i64")) - .load(conn)?; - - // If we got more rows than the limit, the last block may be incomplete so we drop it entirely - // and derive last_block_included from the remaining rows. - let (last_block_included, values) = if let Some(&(last_block_num, ..)) = raw.last() - && raw.len() > limit - { - let values = raw - .into_iter() - .take_while(|(bn, ..)| *bn != last_block_num) - .map(StorageMapValue::from_raw_row) - .collect::, DatabaseError>>()?; - - let last_block_included = values.last().map_or(*block_range.start(), |v| v.block_num); - - (last_block_included, values) - } else { - ( - *block_range.end(), - raw.into_iter() - .map(StorageMapValue::from_raw_row) - .collect::, _>>()?, - ) - }; - - Ok(StorageMapValuesPage { last_block_included, values }) -} - -/// Select latest account storage by querying `accounts.storage_header` for the account's -/// open-ended row and reconstructing full storage from the header plus map values from -/// `account_storage_map_values`. -/// -/// Attention: For large accounts it is prohibitively expensive! -pub(crate) fn select_latest_account_storage( - conn: &mut SqliteConnection, - account_id: AccountId, -) -> Result { - let (storage_header, map_entries_by_slot) = - select_latest_account_storage_components(conn, account_id)?; - // Reconstruct StorageSlots from header slots + map entries - let slots = - Result::, DatabaseError>::from_iter(storage_header.slots().map(|slot_header| { - let slot = match slot_header.slot_type() { - StorageSlotType::Value => { - // For value slots, the header value IS the slot value - StorageSlot::with_value(slot_header.name().clone(), slot_header.value()) - }, - StorageSlotType::Map => { - // For map slots, reconstruct from map entries - let entries = - map_entries_by_slot.get(slot_header.name()).cloned().unwrap_or_default(); - let storage_map = StorageMap::with_entries(entries)?; - StorageSlot::with_map(slot_header.name().clone(), storage_map) - }, - }; - Ok(slot) - }))?; - - Ok(AccountStorage::new(slots)?) -} - -/// Fetch account storage header and all storage maps -pub(crate) fn select_latest_account_storage_components( - conn: &mut SqliteConnection, - account_id: AccountId, -) -> Result { - let account_id_bytes = account_id.to_bytes(); - - // Query storage header blob for this account's current (open-ended) row - let storage_blob: Option> = - SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) - .filter(schema::accounts::account_id.eq(&account_id_bytes)) - .filter(schema::accounts::valid_until.eq(VALID_FOREVER)) - .first(conn) - .optional()? - .flatten(); - - let header = match storage_blob { - Some(blob) => AccountStorageHeader::read_from_bytes(&blob)?, - None => AccountStorageHeader::new(Vec::new())?, - }; - - let entries = select_latest_storage_map_entries_all(conn, &account_id)?; - Ok((header, entries)) -} - -// TODO this is expensive and should only be called from tests -fn select_latest_storage_map_entries_all( - conn: &mut SqliteConnection, - account_id: &AccountId, -) -> Result>, DatabaseError> { - use schema::account_storage_map_values as t; - - let map_values: Vec<(String, Vec, Vec)> = - SelectDsl::select(t::table, (t::slot_name, t::key, t::value)) - .filter(t::account_id.eq(&account_id.to_bytes())) - .filter(t::valid_until.eq(VALID_FOREVER)) - .load(conn)?; - - group_storage_map_entries(map_values) -} - -fn group_storage_map_entries( - map_values: Vec<(String, Vec, Vec)>, -) -> Result>, DatabaseError> { - let mut map_entries_by_slot: HashMap> = - HashMap::new(); - for (slot_name_str, key_bytes, value_bytes) in map_values { - let slot_name: StorageSlotName = slot_name_str.parse().map_err(|_| { - DatabaseError::DataCorrupted(format!("Invalid slot name: {slot_name_str}")) - })?; - let key = StorageMapKey::read_from_bytes(&key_bytes)?; - let value = Word::read_from_bytes(&value_bytes)?; - map_entries_by_slot.entry(slot_name).or_default().insert(key, value); - } - - Ok(map_entries_by_slot) -} - -// ACCOUNT MUTATION -// ================================================================================================ - -#[derive(Queryable, Selectable)] -#[diesel(table_name = crate::db::schema::account_vault_assets)] -#[diesel(check_for_backend(diesel::sqlite::Sqlite))] -pub struct AccountVaultUpdateRaw { - pub vault_key: Vec, - pub asset: Option>, - pub block_num: i64, -} - -impl TryFrom for AccountVaultValue { - type Error = DatabaseError; - - fn try_from(raw: AccountVaultUpdateRaw) -> Result { - let vault_key = AssetId::try_from(Word::read_from_bytes(&raw.vault_key)?)?; - let asset = raw.asset.map(|bytes| Asset::read_from_bytes(&bytes)).transpose()?; - let block_num = BlockNumber::from_raw_sql(raw.block_num)?; - - Ok(AccountVaultValue { block_num, vault_key, asset }) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Selectable, Queryable, QueryableByName)] -#[diesel(table_name = schema::accounts)] -#[diesel(check_for_backend(Sqlite))] -pub struct AccountSummaryRaw { - account_id: Vec, // AccountId, - account_commitment: Vec, //RpoDigest, - block_num: i64, //BlockNumber, -} - -impl TryInto for AccountSummaryRaw { - type Error = DatabaseError; - fn try_into(self) -> Result { - let account_id = AccountId::read_from_bytes(&self.account_id[..])?; - let account_commitment = Word::read_from_bytes(&self.account_commitment[..])?; - let block_num = BlockNumber::from_raw_sql(self.block_num)?; - - Ok(AccountSummary { - account_id, - account_commitment, - block_num, - }) - } -} - -/// Insert an account vault asset row into the DB using the given [`SqliteConnection`]. -/// -/// The new row is inserted open-ended (`valid_until = VALID_FOREVER`); any existing open row -/// with the same `(account_id, vault_key)` tuple has its validity interval closed at `block_num`. -/// -/// # Returns -/// -/// The number of affected rows. -pub(crate) fn insert_account_vault_asset( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, - vault_key: AssetId, - asset: Option, -) -> Result { - let record = AccountAssetRowInsert::new(&account_id, &vault_key, block_num, asset); - - diesel::Connection::transaction(conn, |conn| { - // Close the previous version's validity interval at the new row's block. - let vault_key: Word = vault_key.into(); - let vault_key_bytes = vault_key.to_bytes(); - let account_id_bytes = account_id.to_bytes(); - let update_count = diesel::update(schema::account_vault_assets::table) - .filter( - schema::account_vault_assets::account_id - .eq(account_id_bytes) - .and(schema::account_vault_assets::vault_key.eq(vault_key_bytes)) - .and(schema::account_vault_assets::valid_until.eq(VALID_FOREVER)), - ) - .set(schema::account_vault_assets::valid_until.eq(block_num.to_raw_sql())) - .execute(conn)?; - - // Insert the new open-ended row - let insert_count = diesel::insert_into(schema::account_vault_assets::table) - .values(record) - .execute(conn)?; - - Ok(update_count + insert_count) - }) -} - -/// Inserts a versioned account storage-map value using the given [`SqliteConnection`]. -/// -/// The new row is inserted open-ended, and any previous open row for the same -/// `(account_id, slot_name, key)` tuple has its validity interval closed at `block_num` first. -/// -/// # Returns -/// -/// The total number of inserted and invalidated rows. -/// -/// # Errors -/// -/// Returns an error if the previous row cannot be invalidated or the new row cannot be inserted. -pub(crate) fn insert_account_storage_map_value( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, - slot_name: StorageSlotName, - key: StorageMapKey, - value: Word, -) -> Result { - insert_account_storage_map_value_inner(conn, account_id, block_num, slot_name, key, value, true) -} - -/// Inserts a versioned account storage-map value with optional previous-row invalidation. -/// -/// `invalidate_previous` may be disabled when inserting state for a new account, for which no -/// previous open row can exist. The inserted row is always open-ended. -/// -/// # Returns -/// -/// The total number of inserted and invalidated rows. -/// -/// # Errors -/// -/// Returns an error if the requested invalidation or insertion fails. -fn insert_account_storage_map_value_inner( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, - slot_name: StorageSlotName, - key: StorageMapKey, - value: Word, - invalidate_previous: bool, -) -> Result { - let account_id = account_id.to_bytes(); - let key = key.to_bytes(); - let value = value.to_bytes(); - let slot_name = slot_name.to_raw_sql(); - let block_num = block_num.to_raw_sql(); - - let update_count = if invalidate_previous { - diesel::update(schema::account_storage_map_values::table) - .filter( - schema::account_storage_map_values::account_id - .eq(&account_id) - .and(schema::account_storage_map_values::slot_name.eq(&slot_name)) - .and(schema::account_storage_map_values::key.eq(&key)) - .and(schema::account_storage_map_values::valid_until.eq(VALID_FOREVER)), - ) - .set(schema::account_storage_map_values::valid_until.eq(block_num)) - .execute(conn)? - } else { - 0 - }; - - let record = AccountStorageMapRowInsert { - account_id, - key, - value, - slot_name, - block_num, - valid_until: VALID_FOREVER, - }; - let insert_count = diesel::insert_into(schema::account_storage_map_values::table) - .values(record) - .execute(conn)?; - - Ok(update_count + insert_count) -} - -type PendingStorageInserts = Vec<(AccountId, StorageSlotName, StorageMapKey, Word)>; -type PendingAssetInserts = Vec<(AccountId, AssetId, Option)>; - -fn prepare_full_account_update( - update: &BlockAccountUpdate, - account: Account, -) -> Result<(AccountStateForInsert, PendingStorageInserts, PendingAssetInserts), DatabaseError> { - let account_id = account.id(); - - // sanity check the commitment of account matches the final state commitment - if account.to_commitment() != update.final_state_commitment() { - return Err(DatabaseError::AccountCommitmentsMismatch { - calculated: account.to_commitment(), - expected: update.final_state_commitment(), - }); - } - - // collect storage-map inserts to apply after account upsert - let mut storage = Vec::new(); - for slot in account.storage().slots() { - if let StorageSlotContent::Map(storage_map) = slot.content() { - for (key, value) in storage_map.entries() { - storage.push((account_id, slot.name().clone(), *key, *value)); - } - } - } - - // collect vault-asset inserts to apply after account upsert - let mut assets = Vec::new(); - for asset in account.vault().assets() { - // Only insert assets with non-zero values for fungible assets - let should_insert = match asset { - Asset::Fungible(fungible) => fungible.amount().as_u64() > 0, - Asset::NonFungible(_) => true, - }; - if should_insert { - assets.push((account_id, asset.id(), Some(asset))); - } - } - - Ok((AccountStateForInsert::FullAccount(account), storage, assets)) -} - -/// Prepares a full public-account insertion using roots computed by the account-state forest. -/// -/// This avoids reconstructing the account's vault and storage maps in SQLite. The returned state -/// contains the account-row fields, while storage-map entries and vault assets are returned -/// separately for insertion after the account row has satisfied their foreign-key dependency. -/// Empty-word map entries and assets are omitted from the pending inserts. -/// -/// # Errors -/// -/// Returns an error if the full-state patch is missing its code or nonce, a required precomputed -/// storage root is absent, an asset is invalid, or the reconstructed account header does not match -/// the update's final state commitment. -fn prepare_precomputed_full_account_update( - update: &BlockAccountUpdate, - patch: &AccountPatch, - precomputed: &PrecomputedPublicAccountState, -) -> Result<(AccountStateForInsert, PendingStorageInserts, PendingAssetInserts), DatabaseError> { - let account_id = patch.id(); - let code = patch.code().cloned().ok_or_else(|| { - DatabaseError::DataCorrupted(format!( - "full-state patch for account {account_id} is missing account code" - )) - })?; - let nonce = patch.final_nonce().ok_or_else(|| { - DatabaseError::DataCorrupted(format!( - "full-state patch for account {account_id} is missing final nonce" - )) - })?; - - let storage_header = apply_storage_patch_with_roots( - &AccountStorageHeader::new(Vec::new())?, - patch.storage(), - &precomputed.storage_map_roots, - )?; - let account_header = miden_protocol::account::AccountHeader::new( - account_id, - nonce, - precomputed.vault_root, - storage_header.to_commitment(), - code.commitment(), - ); - if account_header.to_commitment() != update.final_state_commitment() { - return Err(DatabaseError::AccountCommitmentsMismatch { - calculated: account_header.to_commitment(), - expected: update.final_state_commitment(), - }); - } - - let storage = patch - .storage() - .maps() - .flat_map(|(slot_name, map_patch)| { - map_patch.entries().into_iter().flat_map(move |entries| { - entries - .as_map() - .iter() - .filter(|(_key, value)| **value != Word::empty()) - .map(move |(key, value)| (account_id, slot_name.clone(), *key, *value)) - }) - }) - .collect(); - let assets = patch - .vault() - .iter() - .filter(|(_asset_id, value)| **value != Word::empty()) - .map(|(asset_id, value)| { - Asset::from_id_and_value(*asset_id, *value) - .map(|asset| (account_id, *asset_id, Some(asset))) - }) - .collect::, _>>()?; - - // The patch carries full state, so it can be turned back into an account and classified with - // the canonical check. - let is_network_account = NetworkAccount::new(Account::try_from(patch)?).is_ok(); - let state = PrecomputedFullAccountState { - nonce, - code, - storage_header, - vault_root: precomputed.vault_root, - is_network_account, - }; - - Ok((AccountStateForInsert::PrecomputedFullState(state), storage, assets)) -} - -/// Prepares a partial public-account update using the latest row and precomputed forest roots. -/// -/// Unchanged header fields are carried forward from `existing`. The returned partial state is used -/// for the next account row, while storage-map values and vault asset updates are returned -/// separately for insertion after that row. Empty vault values are represented as removals. -/// -/// # Errors -/// -/// Returns an error if the existing row is invalid, a required precomputed storage root is absent, -/// a patched asset is invalid, or the reconstructed account header does not match the update's -/// final state commitment. -fn prepare_partial_account_update( - update: &BlockAccountUpdate, - account_id: AccountId, - patch: &AccountPatch, - precomputed: &PrecomputedPublicAccountState, - existing: &LatestAccountStateRow, -) -> Result<(AccountStateForInsert, PendingStorageInserts, PendingAssetInserts), DatabaseError> { - // Build the minimal account state needed for partial patch application from the latest row that - // was loaded with the account's creation metadata. - let state_headers = existing.state_headers(account_id)?; - - // --- Process asset updates. --------------------------------- The patch carries absolute final - // values, so encode `Some` as update and `None` (an empty value word) as removal. - let mut assets = Vec::new(); - for (vault_key, value) in patch.vault().iter() { - let update_or_remove = if *value == Word::empty() { - None - } else { - Some(Asset::from_id_and_value(*vault_key, *value)?) - }; - assets.push((account_id, *vault_key, update_or_remove)); - } - - // --- Collect storage map updates. --------------------------- - - let mut storage = Vec::new(); - for (slot_name, map_patch) in patch.storage().maps() { - for (key, value) in map_patch.entries().into_iter().flat_map(StorageMapPatchEntries::as_map) - { - storage.push((account_id, slot_name.clone(), *key, *value)); - } - } - - // Apply the patch storage to the given storage header. - let new_storage_header = apply_storage_patch_with_roots( - &state_headers.storage_header, - patch.storage(), - &precomputed.storage_map_roots, - )?; - - let new_vault_root = precomputed.vault_root; - - // --- Compute updated account state for the accounts row. --- Use the absolute final nonce. - let new_nonce = patch.final_nonce().unwrap_or(state_headers.nonce); - - // Create minimal account state data for the row insert. - let account_state = PartialAccountState { - nonce: new_nonce, - code_commitment: state_headers.code_commitment, - storage_header: new_storage_header, - vault_root: new_vault_root, - }; - - let account_header = miden_protocol::account::AccountHeader::new( - account_id, - account_state.nonce, - account_state.vault_root, - account_state.storage_header.to_commitment(), - account_state.code_commitment, - ); - - if account_header.to_commitment() != update.final_state_commitment() { - return Err(DatabaseError::AccountCommitmentsMismatch { - calculated: account_header.to_commitment(), - expected: update.final_state_commitment(), - }); - } - - Ok((AccountStateForInsert::PartialState(account_state), storage, assets)) -} - -/// Returns the subset of `account_ids` whose latest committed state is a network account. -/// -/// Unknown ids and non-network accounts are silently omitted. -pub(crate) fn select_network_accounts_subset( - conn: &mut SqliteConnection, - account_ids: &[AccountId], -) -> Result, DatabaseError> { - QueryParamAccountIdLimit::check(account_ids.len())?; - let id_bytes: Vec> = - account_ids.iter().map(miden_crypto::utils::Serializable::to_bytes).collect(); - - let rows: Vec> = - SelectDsl::select(schema::accounts::table, schema::accounts::account_id) - .filter( - schema::accounts::account_id - .eq_any(&id_bytes) - .and( - schema::accounts::network_account_type - .eq(NetworkAccountType::Network.to_raw_sql()), - ) - .and(schema::accounts::valid_until.eq(VALID_FOREVER)), - ) - .load::>(conn) - .map_err(DatabaseError::Diesel)?; - - rows.into_iter() - .map(|bytes| { - AccountId::read_from_bytes(&bytes).map_err(DatabaseError::DeserializationError) - }) - .collect() -} - -/// Attention: Assumes the account details are NOT null! The schema explicitly allows this though! -#[miden_instrument( - target = COMPONENT, - err, -)] -pub(crate) fn upsert_accounts( - conn: &mut SqliteConnection, - accounts: &[BlockAccountUpdate], - block_num: BlockNumber, - precomputed_public_states: &PrecomputedPublicAccountStates, -) -> Result { - let mut count = 0; - for update in accounts { - let account_id = update.account_id(); - let account_id_bytes = account_id.to_bytes(); - - // Pull the latest row once. Partial updates consume the state headers below, while every - // update carries forward creation metadata. - let existing = select_latest_account_state(conn, account_id)?; - let account_is_new = existing.is_none(); - - let created_at_block = match &existing { - Some(row) => row.created_at_block()?, - None => block_num, - }; - - // NOTE: we collect storage / asset inserts to apply them only after the account row is - // written. The storage and vault tables have FKs pointing to accounts `(account_id, - // block_num)`, so inserting them earlier would violate those constraints when inserting a - // brand-new account. - let (account_state, pending_storage_inserts, pending_asset_inserts) = match update.details() - { - AccountUpdateDetails::Private => (AccountStateForInsert::Private, vec![], vec![]), - - // New account is always a full account, but also comes as an update - AccountUpdateDetails::Public(patch) if patch.is_full_state() => { - if block_num == BlockNumber::GENESIS { - let account = Account::try_from(patch) - .expect("Patch to full account always works for full state patches"); - debug_assert_eq!(account_id, account.id()); - prepare_full_account_update(update, account)? - } else { - let precomputed = - precomputed_public_states.get(&account_id).ok_or_else(|| { - DatabaseError::DataCorrupted(format!( - "missing precomputed public account state for account {account_id}" - )) - })?; - prepare_precomputed_full_account_update(update, patch, precomputed)? - } - }, - - // Update of an existing account - AccountUpdateDetails::Public(patch) => { - let precomputed = precomputed_public_states.get(&account_id).ok_or_else(|| { - DatabaseError::DataCorrupted(format!( - "missing precomputed public account state for account {account_id}" - )) - })?; - let existing = - existing.as_ref().ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; - prepare_partial_account_update(update, account_id, patch, precomputed, existing)? - }, - }; - - // Inherit the classification when the account already exists; otherwise classify it once at - // creation based on the new state. - let network_account_type = match &existing { - Some(row) => row.network_account_type()?, - None => match &account_state { - AccountStateForInsert::FullAccount(account) - if NetworkAccount::new(account.clone()).is_ok() => - { - NetworkAccountType::Network - }, - AccountStateForInsert::PrecomputedFullState(state) if state.is_network_account => { - NetworkAccountType::Network - }, - _ => NetworkAccountType::None, - }, - }; - - // Insert account _code_ for full accounts (new account creation) - if let AccountStateForInsert::FullAccount(ref account) = account_state { - let code = account.code(); - let code_value = AccountCodeRowInsert { - code_commitment: code.commitment().to_bytes(), - code: code.to_bytes(), - }; - diesel::insert_into(schema::account_codes::table) - .values(&code_value) - .on_conflict(schema::account_codes::code_commitment) - .do_nothing() - .execute(conn)?; - } - if let AccountStateForInsert::PrecomputedFullState(ref state) = account_state { - let code_value = AccountCodeRowInsert { - code_commitment: state.code.commitment().to_bytes(), - code: state.code.to_bytes(), - }; - diesel::insert_into(schema::account_codes::table) - .values(&code_value) - .on_conflict(schema::account_codes::code_commitment) - .do_nothing() - .execute(conn)?; - } - - // close the previous row's validity interval and insert NEW account row - diesel::update(schema::accounts::table) - .filter( - schema::accounts::account_id - .eq(&account_id_bytes) - .and(schema::accounts::valid_until.eq(VALID_FOREVER)), - ) - .set(schema::accounts::valid_until.eq(block_num.to_raw_sql())) - .execute(conn)?; - - let account_value = match &account_state { - AccountStateForInsert::Private => AccountRowInsert::new_private( - account_id, - network_account_type, - update.final_state_commitment(), - block_num, - created_at_block, - ), - AccountStateForInsert::FullAccount(account) => AccountRowInsert::new_from_account( - account_id, - network_account_type, - update.final_state_commitment(), - block_num, - created_at_block, - account, - ), - AccountStateForInsert::PrecomputedFullState(state) => { - AccountRowInsert::new_from_precomputed_full_state( - account_id, - network_account_type, - update.final_state_commitment(), - block_num, - created_at_block, - state, - ) - }, - AccountStateForInsert::PartialState(state) => AccountRowInsert::new_from_partial( - account_id, - network_account_type, - update.final_state_commitment(), - block_num, - created_at_block, - state, - ), - }; - - diesel::insert_into(schema::accounts::table) - .values(&account_value) - .on_conflict((schema::accounts::account_id, schema::accounts::block_num)) - .do_update() - .set(&account_value) - .execute(conn)?; - - // insert pending storage map entries TODO consider batching - for (acc_id, slot_name, key, value) in pending_storage_inserts { - if account_is_new { - insert_account_storage_map_value_inner( - conn, acc_id, block_num, slot_name, key, value, false, - )?; - } else { - insert_account_storage_map_value(conn, acc_id, block_num, slot_name, key, value)?; - } - } - - for (acc_id, vault_key, update) in pending_asset_inserts { - insert_account_vault_asset(conn, acc_id, block_num, vault_key, update)?; - } - - count += 1; - } - - Ok(count) -} - -#[derive(Insertable, Debug, Clone)] -#[diesel(table_name = schema::account_codes)] -pub(crate) struct AccountCodeRowInsert { - pub(crate) code_commitment: Vec, - pub(crate) code: Vec, -} - -#[derive(Insertable, AsChangeset, Debug, Clone)] -#[diesel(table_name = schema::accounts)] -pub(crate) struct AccountRowInsert { - pub(crate) account_id: Vec, - pub(crate) network_account_type: i32, - pub(crate) block_num: i64, - pub(crate) account_commitment: Vec, - pub(crate) code_commitment: Option>, - pub(crate) nonce: Option, - pub(crate) storage_header: Option>, - pub(crate) vault_root: Option>, - pub(crate) created_at_block: i64, - pub(crate) valid_until: i64, -} - -impl AccountRowInsert { - /// Creates an insert row for a private account (no public state). - pub(crate) fn new_private( - account_id: AccountId, - network_account_type: NetworkAccountType, - account_commitment: Word, - block_num: BlockNumber, - created_at_block: BlockNumber, - ) -> Self { - Self { - account_id: account_id.to_bytes(), - network_account_type: network_account_type.to_raw_sql(), - account_commitment: account_commitment.to_bytes(), - block_num: block_num.to_raw_sql(), - nonce: None, - code_commitment: None, - storage_header: None, - vault_root: None, - created_at_block: created_at_block.to_raw_sql(), - valid_until: VALID_FOREVER, - } - } - - /// Creates an insert row from a full account (new account creation). - fn new_from_account( - account_id: AccountId, - network_account_type: NetworkAccountType, - account_commitment: Word, - block_num: BlockNumber, - created_at_block: BlockNumber, - account: &Account, - ) -> Self { - Self { - account_id: account_id.to_bytes(), - network_account_type: network_account_type.to_raw_sql(), - account_commitment: account_commitment.to_bytes(), - block_num: block_num.to_raw_sql(), - nonce: Some(nonce_to_raw_sql(account.nonce())), - code_commitment: Some(account.code().commitment().to_bytes()), - storage_header: Some(account.storage().to_header().to_bytes()), - vault_root: Some(account.vault().root().to_bytes()), - created_at_block: created_at_block.to_raw_sql(), - valid_until: VALID_FOREVER, - } - } - - fn new_from_precomputed_full_state( - account_id: AccountId, - network_account_type: NetworkAccountType, - account_commitment: Word, - block_num: BlockNumber, - created_at_block: BlockNumber, - state: &PrecomputedFullAccountState, - ) -> Self { - Self { - account_id: account_id.to_bytes(), - network_account_type: network_account_type.to_raw_sql(), - block_num: block_num.to_raw_sql(), - account_commitment: account_commitment.to_bytes(), - code_commitment: Some(state.code.commitment().to_bytes()), - nonce: Some(nonce_to_raw_sql(state.nonce)), - storage_header: Some(state.storage_header.to_bytes()), - vault_root: Some(state.vault_root.to_bytes()), - created_at_block: created_at_block.to_raw_sql(), - valid_until: VALID_FOREVER, - } - } - - /// Creates an insert row from a partial account state (patch update). - fn new_from_partial( - account_id: AccountId, - network_account_type: NetworkAccountType, - account_commitment: Word, - block_num: BlockNumber, - created_at_block: BlockNumber, - state: &PartialAccountState, - ) -> Self { - Self { - account_id: account_id.to_bytes(), - network_account_type: network_account_type.to_raw_sql(), - account_commitment: account_commitment.to_bytes(), - block_num: block_num.to_raw_sql(), - nonce: Some(nonce_to_raw_sql(state.nonce)), - code_commitment: Some(state.code_commitment.to_bytes()), - storage_header: Some(state.storage_header.to_bytes()), - vault_root: Some(state.vault_root.to_bytes()), - created_at_block: created_at_block.to_raw_sql(), - valid_until: VALID_FOREVER, - } - } -} - -#[derive(Insertable, AsChangeset, Debug, Clone)] -#[diesel(table_name = schema::account_vault_assets)] -pub(crate) struct AccountAssetRowInsert { - pub(crate) account_id: Vec, - pub(crate) block_num: i64, - pub(crate) vault_key: Vec, - pub(crate) asset: Option>, - pub(crate) valid_until: i64, -} - -impl AccountAssetRowInsert { - pub(crate) fn new( - account_id: &AccountId, - vault_key: &AssetId, - block_num: BlockNumber, - asset: Option, - ) -> Self { - let account_id = account_id.to_bytes(); - let vault_key: Word = (*vault_key).into(); - let vault_key = vault_key.to_bytes(); - let block_num = block_num.to_raw_sql(); - let asset = asset.map(|asset| asset.to_bytes()); - Self { - account_id, - block_num, - vault_key, - asset, - valid_until: VALID_FOREVER, - } - } -} - -#[derive(Insertable, AsChangeset, Debug, Clone)] -#[diesel(table_name = schema::account_storage_map_values)] -pub(crate) struct AccountStorageMapRowInsert { - pub(crate) account_id: Vec, - pub(crate) block_num: i64, - pub(crate) slot_name: String, - pub(crate) key: Vec, - pub(crate) value: Vec, - pub(crate) valid_until: i64, -} - -// CLEANUP FUNCTIONS -// ================================================================================================ - -/// Number of historical blocks to retain for vault assets, storage map values, and account codes. -/// Rows whose validity interval ends at or below `prune_tip - HISTORICAL_BLOCK_RETENTION` will be -/// deleted; rows still valid anywhere inside the retention window (including all open-ended rows) -/// are retained. -pub const HISTORICAL_BLOCK_RETENTION: u32 = 50; - -/// Clean up old entries for all accounts, deleting entries that can no longer affect state -/// reconstruction at any block within the retention window. -/// -/// A row is applicable for blocks in `[block_num, valid_until)`, so it is deletable exactly when -/// its interval ends at or below the cutoff (`prune_tip - HISTORICAL_BLOCK_RETENTION`): it then -/// cannot cover any block inside the window. `prune_tip` is the effective tip for retention — it -/// lags the chain tip while old snapshot generations are still pinned by readers (see -/// [`crate::db::Db::apply_block`]). Account codes follow the same rule — a code is deleted only -/// when no account row whose interval reaches past the cutoff references it. -/// -/// # Returns -/// A tuple of `(vault_assets_deleted, storage_map_values_deleted, account_codes_deleted)` -#[miden_instrument( - target = COMPONENT, - err, - fields( - cutoff_block, - ), -)] -pub(crate) fn prune_history( - conn: &mut SqliteConnection, - prune_tip: BlockNumber, -) -> Result<(usize, usize, usize), DatabaseError> { - let cutoff_block = i64::from(prune_tip.as_u32().saturating_sub(HISTORICAL_BLOCK_RETENTION)); - tracing::Span::current().record("cutoff_block", cutoff_block); - let vault_deleted = prune_account_vault_assets(conn, cutoff_block)?; - let storage_deleted = prune_account_storage_map_values(conn, cutoff_block)?; - let codes_deleted = prune_account_codes(conn, cutoff_block)?; - - Ok((vault_deleted, storage_deleted, codes_deleted)) -} - -#[miden_instrument( - target = COMPONENT, - err, - fields( - cutoff_block, - ), -)] -fn prune_account_vault_assets( - conn: &mut SqliteConnection, - cutoff_block: i64, -) -> Result { - use diesel::sql_types::BigInt; - - // The literal `!= VALID_FOREVER` term (rather than a bound parameter) lets SQLite prove the - // predicate implies `idx_vault_cleanup`'s partial-index condition. - diesel::sql_query(format!( - "DELETE FROM account_vault_assets \ - WHERE valid_until != {VALID_FOREVER} \ - AND valid_until <= ?1" - )) - .bind::(cutoff_block) - .execute(conn) - .map_err(DatabaseError::Diesel) -} - -#[miden_instrument( - target = COMPONENT, - err, - fields( - cutoff_block, - ), -)] -fn prune_account_storage_map_values( - conn: &mut SqliteConnection, - cutoff_block: i64, -) -> Result { - use diesel::sql_types::BigInt; - - // The literal `!= VALID_FOREVER` term (rather than a bound parameter) lets SQLite prove the - // predicate implies `idx_storage_cleanup`'s partial-index condition. - diesel::sql_query(format!( - "DELETE FROM account_storage_map_values \ - WHERE valid_until != {VALID_FOREVER} \ - AND valid_until <= ?1" - )) - .bind::(cutoff_block) - .execute(conn) - .map_err(DatabaseError::Diesel) -} - -/// Deletes account codes that are no longer referenced by any account row that can serve a read -/// within the retention window. -/// -/// An account code is safe to delete when no `accounts` row whose validity interval reaches past -/// the cutoff (`valid_until > cutoff_block`) references it. That single predicate covers rows -/// inside the window, all open-ended (current) rows, and each account's baseline row — the row -/// still valid at the cutoff even though it was written before it. -/// -/// Rather than re-checking every code on every prune, only codes whose deletability could have -/// changed since the previous prune are examined. A code survived the previous prune because at -/// least one `accounts` row with `valid_until > prev_cutoff` referenced it. For it to be -/// deletable now, all such rows must have expired by the new cutoff — including the longest-lived -/// one, whose `valid_until` therefore lands inside `(prev_cutoff, cutoff_block]`. Scanning the -/// rows that expired in that window thus finds every code that could have become deletable. The -/// scan is an `idx_accounts_code_validity` index range, so its cost scales with the number of -/// account updates since the previous prune, not with total history. Each candidate is deleted -/// only if the `idx_accounts_code_probe` existence probe finds no row still referencing it with -/// `valid_until > cutoff_block`. The previous cutoff is persisted in `prune_progress` within the -/// same transaction; when absent (first prune after migration, or a fresh database) a full pass -/// over all rows valid past the cutoff runs instead. -/// -/// Correctness of the windowed candidate set rests on two invariants: -/// - Rows are only ever closed to the `block_num` of the block currently being applied, which is -/// always above the cutoff, so every expiry crosses the window of some later prune. A write path -/// that back-dated `valid_until` below the current cutoff would leak the code forever. -/// - Every `account_codes` row is inserted alongside an `accounts` row referencing it (see -/// [`upsert_accounts`]); an orphan code with no referencing row would never become a candidate. -#[miden_instrument( - target = COMPONENT, - err, - fields( - cutoff_block, - ), -)] -fn prune_account_codes( - conn: &mut SqliteConnection, - cutoff_block: i64, -) -> Result { - use diesel::sql_types::BigInt; - - let prev_cutoff: Option = - SelectDsl::select(schema::prune_progress::table, schema::prune_progress::codes_cutoff) - .first(conn) - .optional() - .map_err(DatabaseError::Diesel)?; - - let deleted = match prev_cutoff { - // Codes are already pruned through this cutoff and nothing can become collectable while the - // cutoff stands still. Equality is the common case: the cutoff is clamped to zero for the - // first `HISTORICAL_BLOCK_RETENTION` blocks, and a pinned snapshot freezes the prune tip - // across consecutive blocks. A strictly greater `prev_cutoff` is unreachable through - // `apply_block` (the prune tip never regresses) but is guarded against so an out-of-order - // caller cannot move the marker backwards or run the delete with an inverted window. - Some(prev_cutoff) if prev_cutoff >= cutoff_block => return Ok(0), - Some(prev_cutoff) => diesel::sql_query( - "DELETE FROM account_codes \ - WHERE code_commitment IN ( \ - SELECT DISTINCT code_commitment \ - FROM accounts INDEXED BY idx_accounts_code_validity \ - WHERE code_commitment IS NOT NULL \ - AND valid_until > ?1 \ - AND valid_until <= ?2 \ - ) \ - AND NOT EXISTS ( \ - SELECT 1 \ - FROM accounts INDEXED BY idx_accounts_code_probe \ - WHERE accounts.code_commitment = account_codes.code_commitment \ - AND accounts.valid_until > ?2 \ - )", - ) - .bind::(prev_cutoff) - .bind::(cutoff_block) - .execute(conn) - .map_err(DatabaseError::Diesel)?, - // No recorded cutoff: full pass. The forced `idx_accounts_code_validity` covering index - // keeps the subquery an index-only range scan, sized by rows valid at or after the cutoff - // rather than total history. - None => diesel::sql_query( - "DELETE FROM account_codes \ - WHERE code_commitment NOT IN ( \ - SELECT DISTINCT code_commitment \ - FROM accounts INDEXED BY idx_accounts_code_validity \ - WHERE code_commitment IS NOT NULL \ - AND valid_until > ?1 \ - )", - ) - .bind::(cutoff_block) - .execute(conn) - .map_err(DatabaseError::Diesel)?, - }; - - diesel::insert_into(schema::prune_progress::table) - .values(( - schema::prune_progress::id.eq(0), - schema::prune_progress::codes_cutoff.eq(cutoff_block), - )) - .on_conflict(schema::prune_progress::id) - .do_update() - .set(schema::prune_progress::codes_cutoff.eq(cutoff_block)) - .execute(conn) - .map_err(DatabaseError::Diesel)?; - - Ok(deleted) -} diff --git a/crates/store/src/db/models/queries/accounts/at_block.rs b/crates/store/src/db/models/queries/accounts/at_block.rs deleted file mode 100644 index cfe91995cd..0000000000 --- a/crates/store/src/db/models/queries/accounts/at_block.rs +++ /dev/null @@ -1,101 +0,0 @@ -use diesel::prelude::Queryable; -use diesel::query_dsl::methods::SelectDsl; -use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl, SqliteConnection}; -use miden_protocol::account::{AccountHeader, AccountId, AccountStorageHeader}; -use miden_protocol::block::BlockNumber; -use miden_protocol::utils::serde::{Deserializable, Serializable}; -use miden_protocol::{Felt, Word}; - -use crate::db::models::conv::{SqlTypeConvert, raw_sql_to_nonce}; -use crate::db::schema; -use crate::errors::DatabaseError; - -// ACCOUNT HEADER -// ================================================================================================ - -#[derive(Debug, Clone, Queryable)] -struct AccountHeaderDataRaw { - code_commitment: Option>, - nonce: Option, - storage_header: Option>, - vault_root: Option>, -} - -/// Queries the account header for a specific account at a specific block number. -/// -/// This reconstructs the `AccountHeader` by reading from the `accounts` table: -/// - `account_id`, `nonce`, `code_commitment`, `storage_header`, `vault_root` -/// -/// Returns `None` if the account doesn't exist at that block. -/// -/// # Arguments -/// -/// * `conn` - Database connection -/// * `account_id` - The account ID to query -/// * `block_num` - The block number at which to query the account header -/// -/// # Returns -/// -/// * `Ok(Some((AccountHeader, AccountStorageHeader)))` - The headers if found -/// * `Ok(None)` - If account doesn't exist at that block -/// * `Err(DatabaseError)` - If there's a database error -pub(crate) fn select_account_header_with_storage_header_at_block( - conn: &mut SqliteConnection, - account_id: AccountId, - block_num: BlockNumber, -) -> Result, DatabaseError> { - use schema::accounts; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - - let account_data: Option = SelectDsl::select( - accounts::table - .filter(accounts::account_id.eq(&account_id_bytes)) - .filter(accounts::block_num.le(block_num_sql)) - .order(accounts::block_num.desc()) - .limit(1), - ( - accounts::code_commitment, - accounts::nonce, - accounts::storage_header, - accounts::vault_root, - ), - ) - .first(conn) - .optional()?; - - let Some(AccountHeaderDataRaw { - code_commitment: code_commitment_bytes, - nonce: nonce_raw, - storage_header: storage_header_blob, - vault_root: vault_root_bytes, - }) = account_data - else { - return Ok(None); - }; - - let storage_header = match &storage_header_blob { - Some(blob) => AccountStorageHeader::read_from_bytes(blob)?, - None => AccountStorageHeader::new(Vec::new())?, - }; - - let storage_commitment = storage_header.to_commitment(); - - let code_commitment = code_commitment_bytes - .map(|bytes| Word::read_from_bytes(&bytes)) - .transpose()? - .unwrap_or(Word::default()); - - let nonce = nonce_raw.map_or(Felt::ZERO, raw_sql_to_nonce); - - let vault_root = vault_root_bytes - .map(|bytes| Word::read_from_bytes(&bytes)) - .transpose()? - .unwrap_or(Word::default()); - - let account_header = - AccountHeader::new(account_id, nonce, vault_root, storage_commitment, code_commitment); - - Ok(Some((account_header, storage_header))) -} diff --git a/crates/store/src/db/models/queries/block_headers.rs b/crates/store/src/db/models/queries/block_headers.rs deleted file mode 100644 index e147a902ca..0000000000 --- a/crates/store/src/db/models/queries/block_headers.rs +++ /dev/null @@ -1,239 +0,0 @@ -use diesel::prelude::Insertable; -use diesel::query_dsl::methods::SelectDsl; -use diesel::{ - ExpressionMethods, - OptionalExtension, - QueryDsl, - Queryable, - QueryableByName, - RunQueryDsl, - Selectable, - SelectableHelper, - SqliteConnection, -}; -use miden_crypto::Word; -use miden_node_utils::limiter::{QueryParamBlockLimit, QueryParamLimiter}; -use miden_node_utils::tracing::miden_instrument; -use miden_protocol::block::{BlockHeader, BlockNumber, BlockSignatures}; -use miden_protocol::utils::serde::{Deserializable, Serializable}; - -use super::DatabaseError; -use crate::COMPONENT; -use crate::db::models::conv::SqlTypeConvert; -use crate::db::models::vec_raw_try_into; -use crate::db::schema; - -/// Select a [`BlockHeader`] from the DB by its `block_num` using the given [`SqliteConnection`]. -/// -/// # Returns -/// -/// When `block_num` is [None], the latest block header is returned. Otherwise, the block with -/// the given block height is returned. -/// -/// ```sql -/// -- with argument -/// SELECT block_num, block_header -/// FROM block_headers -/// WHERE block_num = ?1 -/// -/// -- without argument -/// SELECT block_num, block_header -/// FROM block_headers -/// ORDER BY block_num DESC -/// LIMIT 1 -/// ``` -pub(crate) fn select_block_header_by_block_num( - conn: &mut SqliteConnection, - maybe_block_num: Option, -) -> Result, DatabaseError> { - let sel = SelectDsl::select(schema::block_headers::table, BlockHeaderRawRow::as_select()); - let row = if let Some(block_num) = maybe_block_num { - sel.filter(schema::block_headers::block_num.eq(block_num.to_raw_sql())) - .get_result::(conn) - .optional()? - // invariant: only one block exists with the given block header, so the length is always - // zero or one - } else { - sel.order(schema::block_headers::block_num.desc()) - .limit(1) - .get_result::(conn) - .optional()? - }; - row.map(std::convert::TryInto::try_into).transpose() -} - -/// Select a [`BlockHeader`] and its [`BlockSignatures`] from the DB by its `block_num` using the -/// given [`SqliteConnection`]. -/// -/// # Returns -/// -/// The block header with the given block height and its validator signatures is returned. -/// -/// ```sql -/// SELECT block_num, block_header, signature -/// FROM block_headers -/// WHERE block_num = ?1 -/// ``` -pub(crate) fn select_block_header_and_signatures_by_block_num( - conn: &mut SqliteConnection, - block_number: BlockNumber, -) -> Result, DatabaseError> { - let sel = SelectDsl::select(schema::block_headers::table, BlockHeaderRawRow::as_select()); - let row = sel - .filter(schema::block_headers::block_num.eq(block_number.to_raw_sql())) - .get_result::(conn) - .optional()?; - row.map(std::convert::TryInto::try_into).transpose() -} - -/// Select block headers for the given block numbers. -/// -/// # Parameters -/// * `blocks`: Iterator of block numbers to retrieve -/// - Limit: 0 <= count <= 1000 -/// -/// # Note -/// -/// Only returns the block headers that are actually present. -/// -/// # Returns -/// -/// A vector of [`BlockHeader`] or an error. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT block_num, block_header -/// FROM block_headers -/// WHERE block_num IN (?1) -/// ``` -pub fn select_block_headers( - conn: &mut SqliteConnection, - blocks: impl Iterator + Send, -) -> Result, DatabaseError> { - // The iterators are all deterministic, so is the conjunction. - // All calling sites do it equivalently, hence the below holds. - // - // - // And the conjunction is truthful: - // - QueryParamBlockLimit::check(blocks.size_hint().0)?; - - let blocks = Vec::from_iter(blocks.map(SqlTypeConvert::to_raw_sql)); - let raw_block_headers = - QueryDsl::select(schema::block_headers::table, BlockHeaderRawRow::as_select()) - .filter(schema::block_headers::block_num.eq_any(blocks)) - .load::(conn)?; - vec_raw_try_into(raw_block_headers) -} - -/// Select all block headers from the DB using the given [`SqliteConnection`]. -/// -/// # Returns -/// -/// A vector of [`BlockHeader`] or an error. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT commitment -/// FROM block_headers -/// ORDER BY block_num ASC -/// ``` -pub fn select_all_block_header_commitments( - conn: &mut SqliteConnection, -) -> Result, DatabaseError> { - let raw_commitments = - QueryDsl::select(schema::block_headers::table, schema::block_headers::commitment) - .order(schema::block_headers::block_num.asc()) - .load::>(conn)?; - let commitments = - Result::from_iter(raw_commitments.into_iter().map(BlockHeaderCommitment::from_raw_sql))?; - Ok(commitments) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[repr(transparent)] -pub struct BlockHeaderCommitment(pub(crate) Word); - -impl BlockHeaderCommitment { - pub fn new(header: &BlockHeader) -> Self { - Self(header.commitment()) - } - pub fn word(self) -> Word { - self.0 - } -} - -#[derive(Debug, Clone, Queryable, QueryableByName, Selectable)] -#[diesel(table_name = schema::block_headers)] -#[diesel(check_for_backend(diesel::sqlite::Sqlite))] -pub struct BlockHeaderRawRow { - #[expect(dead_code)] - pub block_num: i64, - pub block_header: Vec, - pub signature: Vec, - pub commitment: Vec, -} - -impl TryInto for BlockHeaderRawRow { - type Error = DatabaseError; - fn try_into(self) -> Result { - let block_header = BlockHeader::from_raw_sql(self.block_header)?; - // we're bust if this invariant doesn't hold - debug_assert_eq!( - BlockHeaderCommitment::new(&block_header), - BlockHeaderCommitment::from_raw_sql(self.commitment) - .expect("Database always contains valid format commitments") - ); - Ok(block_header) - } -} - -impl TryInto<(BlockHeader, BlockSignatures)> for BlockHeaderRawRow { - type Error = DatabaseError; - fn try_into(self) -> Result<(BlockHeader, BlockSignatures), Self::Error> { - let block_header = BlockHeader::read_from_bytes(&self.block_header[..])?; - let signatures = BlockSignatures::read_from_bytes(&self.signature[..])?; - Ok((block_header, signatures)) - } -} - -#[derive(Debug, Clone, Insertable)] -#[diesel(table_name = schema::block_headers)] -#[diesel(check_for_backend(diesel::sqlite::Sqlite))] -pub struct BlockHeaderInsert { - pub block_num: i64, - pub block_header: Vec, - pub signature: Vec, - pub commitment: Vec, -} - -/// Insert a [`BlockHeader`] to the DB using the given [`SqliteConnection`]. -/// -/// # Returns -/// -/// The number of affected rows. -/// -/// # Note -/// -/// The [`SqliteConnection`] object is not consumed. It's up to the caller to commit or rollback the -/// transaction -#[miden_instrument( - target = COMPONENT, - err, -)] -pub(crate) fn insert_block_header( - conn: &mut SqliteConnection, - block_header: &BlockHeader, - signatures: &BlockSignatures, -) -> Result { - let row = BlockHeaderInsert { - block_num: block_header.block_num().to_raw_sql(), - block_header: block_header.to_bytes(), - signature: signatures.to_bytes(), - commitment: BlockHeaderCommitment::new(block_header).to_raw_sql(), - }; - let count = diesel::insert_into(schema::block_headers::table).values(&[row]).execute(conn)?; - Ok(count) -} diff --git a/crates/store/src/db/models/queries/mod.rs b/crates/store/src/db/models/queries/mod.rs deleted file mode 100644 index 377a20e94d..0000000000 --- a/crates/store/src/db/models/queries/mod.rs +++ /dev/null @@ -1,76 +0,0 @@ -//! Abstracts all relevant queries to individual blocking function calls -//! -//! ## Naming -//! -//! * `fn *` function names have on of three prefixes: `upsert_`, `insert_` or `select_` denoting -//! their nature. If neither fits, then use your best judgment for naming. -//! * `*Insert` types are used for _inserting_ data into table and _must_ implement -//! `diesel::Insertable`. -//! * `*RawRow` types are used for _querying_ a _single_ table an _without_ an explicit row and must -//! implement a `QueryableByName` and `Selectable`. -//! * `*RawJoined` types are used for _querying_ a _left join_ table _without_ an explicit row and -//! must implement a `QueryableByName` and _cannot_ implement `Selectable`. -//! -//! ## Type conversion -//! -//! The database `*Raw` and `*Joined` types use database primitives. In order to convert to correct -//! in-memory representations it's preferable to have new-types which implement [`SqlTypeConvert`]. -//! If that is inconvenient, provide two wrapper methods for the conversion each way. There must be -//! relevant constraints in the table. For convenience, any types that have more complex -//! serialization may use [`Serializable`] and [`Deserializable`] for convenience. -//! -//! ## Assumptions -//! -//! Any call that sits insides of `queries/**/*.rs` can assume it's called within the scope of a -//! transaction, any nesting of further `transaction(conn, || {})` has no effect and should be -//! considered unnecessary boilerplate by default. - -use diesel::SqliteConnection; -use miden_protocol::block::SignedBlock; -use miden_protocol::note::Nullifier; - -use super::DatabaseError; -use crate::db::NoteRecord; - -mod transactions; -pub use transactions::*; -mod block_headers; -pub use block_headers::*; -mod accounts; -pub use accounts::*; -mod nullifiers; -pub use nullifiers::NullifiersPage; -pub(crate) use nullifiers::*; -mod notes; -pub(crate) use notes::*; - -/// Apply a new block to the state. -/// -/// # Returns -/// -/// Number of records inserted and/or updated. -pub(crate) fn apply_block( - conn: &mut SqliteConnection, - block: &SignedBlock, - notes: &[(NoteRecord, Option)], - precomputed_public_states: &PrecomputedPublicAccountStates, -) -> Result { - let mut count = 0; - // Note: ordering here is important as the relevant tables have FK dependencies. - count += insert_block_header(conn, block.header(), block.signatures())?; - count += upsert_accounts( - conn, - block.body().updated_accounts(), - block.header().block_num(), - precomputed_public_states, - )?; - count += insert_scripts(conn, notes.iter().map(|(note, _)| note))?; - count += insert_notes(conn, notes)?; - count += insert_transactions(conn, block.header().block_num(), block.body().transactions())?; - count += insert_nullifiers_for_block( - conn, - block.body().created_nullifiers(), - block.header().block_num(), - )?; - Ok(count) -} diff --git a/crates/store/src/db/models/queries/notes.rs b/crates/store/src/db/models/queries/notes.rs deleted file mode 100644 index f1fd1f2438..0000000000 --- a/crates/store/src/db/models/queries/notes.rs +++ /dev/null @@ -1,848 +0,0 @@ -#![expect( - clippy::cast_possible_wrap, - reason = "We will not approach the item count where i64 and usize cause issues" -)] - -use std::collections::{BTreeMap, BTreeSet, HashSet}; -use std::ops::RangeInclusive; - -use diesel::prelude::{ - ExpressionMethods, - Insertable, - QueryDsl, - Queryable, - QueryableByName, - Selectable, -}; -use diesel::query_dsl::methods::SelectDsl; -use diesel::sqlite::Sqlite; -use diesel::{ - JoinOnDsl, - NullableExpressionMethods, - OptionalExtension, - RunQueryDsl, - SelectableHelper, - SqliteConnection, -}; -use miden_node_utils::limiter::{ - QueryParamLimiter, - QueryParamNoteCommitmentLimit, - QueryParamNoteTagLimit, -}; -use miden_node_utils::tracing::miden_instrument; -use miden_protocol::Word; -use miden_protocol::account::AccountId; -use miden_protocol::block::{BlockNoteIndex, BlockNumber}; -use miden_protocol::crypto::merkle::SparseMerklePath; -use miden_protocol::note::{ - NoteAssets, - NoteAttachments, - NoteDetails, - NoteId, - NoteInclusionProof, - NoteMetadata, - NoteRecipient, - NoteScript, - NoteStorage, - NoteTag, - NoteType, - Nullifier, - PartialNoteMetadata, -}; -use miden_protocol::utils::serde::{Deserializable, Serializable}; -use miden_standards::note::NetworkAccountTarget; - -use crate::COMPONENT; -use crate::db::models::conv::{ - SqlTypeConvert, - idx_to_raw_sql, - note_type_to_raw_sql, - raw_sql_to_idx, -}; -use crate::db::models::queries::select_block_header_by_block_num; -use crate::db::models::{serialize_vec, vec_raw_try_into}; -use crate::db::{DatabaseError, NoteRecord, NoteSyncRecord, NoteSyncUpdate, schema}; -use crate::errors::NoteSyncError; - -/// Estimated byte size of a [`NoteSyncUpdate`] excluding its notes. -/// -/// `BlockHeader` (~341 bytes) + MMR proof with 32 siblings (~1216 bytes). -pub(crate) const NOTE_SYNC_BLOCK_OVERHEAD_BYTES: usize = 1600; - -/// Estimated byte size of a single [`NoteSyncRecord`]. -/// -/// Note ID (~38 bytes) + index + sync metadata with up to four attachment entries (~200 bytes) + -/// sparse merkle path with 16 siblings (~608 bytes). -pub(crate) const NOTE_SYNC_RECORD_BYTES: usize = 900; - -// NETWORK NOTE TYPE -// ================================================================================================ - -/// Classifies network notes for database storage. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[repr(i32)] -pub(crate) enum NetworkNoteType { - /// Not a network note. - None = 0, - /// Single account target network note (has `NetworkAccountTarget` attachment). - SingleTarget = 1, -} - -impl From for i32 { - fn from(value: NetworkNoteType) -> Self { - value as i32 - } -} - -/// Select notes matching the given tags within a block range. -/// -/// # Parameters -/// * `note_tags`: List of note tags to filter by -/// - Limit: 0 <= count <= 1000 -/// * `block_range`: Range of blocks to search (inclusive) -/// -/// # Returns -/// -/// All matching notes from the first block within the range containing a matching note. If no -/// matching notes are found at all, then an empty vector is returned. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// committed_at, -/// batch_index, -/// note_index, -/// note_id, -/// note_type, -/// sender, -/// tag, -/// attachment, -/// inclusion_path -/// FROM -/// notes -/// WHERE -/// committed_at = ( -/// SELECT -/// committed_at -/// FROM -/// notes -/// WHERE -/// tag IN (?1) AND -/// committed_at >= ?2 AND -/// committed_at <= ?3 -/// ORDER BY -/// committed_at ASC -/// LIMIT 1 -/// ) AND -/// tag IN (?1) -/// ORDER BY -/// committed_at ASC, batch_index ASC, note_index ASC -/// ``` -pub(crate) fn select_notes_since_block_by_tag( - conn: &mut SqliteConnection, - note_tags: &[u32], - block_range: RangeInclusive, -) -> Result, DatabaseError> { - QueryParamNoteTagLimit::check(note_tags.len())?; - let desired_note_tags: Vec = note_tags.iter().map(|tag| *tag as i32).collect(); - let start_block_num = block_range.start().to_raw_sql(); - let end_block_num = block_range.end().to_raw_sql(); - - let Some(desired_block_num): Option = - SelectDsl::select(schema::notes::table, schema::notes::committed_at) - .filter(schema::notes::tag.eq_any(&desired_note_tags)) - .filter(schema::notes::committed_at.ge(start_block_num)) - .filter(schema::notes::committed_at.le(end_block_num)) - .order_by(schema::notes::committed_at.asc()) - .limit(1) - .get_result(conn) - .optional()? - else { - return Ok(Vec::new()); - }; - - let notes = SelectDsl::select(schema::notes::table, NoteSyncRecordRawRow::as_select()) - .filter(schema::notes::committed_at.eq(desired_block_num)) - .filter(schema::notes::tag.eq_any(&desired_note_tags)) - .order_by(( - schema::notes::committed_at.asc(), - schema::notes::batch_index.asc(), - schema::notes::note_index.asc(), - )) - .get_results::(conn) - .map_err(DatabaseError::from)?; - - vec_raw_try_into(notes) -} - -/// Select all notes matching the given set of identifiers -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// notes.committed_at, -/// notes.batch_index, -/// notes.note_index, -/// notes.note_id, -/// notes.note_type, -/// notes.sender, -/// notes.tag, -/// notes.attachment, -/// notes.assets, -/// notes.storage, -/// notes.serial_num, -/// notes.inclusion_path, -/// note_scripts.script -/// FROM notes -/// LEFT JOIN note_scripts ON notes.script_root = note_scripts.script_root -/// WHERE note_id IN (?1) -/// ``` -pub(crate) fn select_notes_by_id( - conn: &mut SqliteConnection, - note_ids: &[NoteId], -) -> Result, DatabaseError> { - let note_ids = serialize_vec(note_ids); - let q = schema::notes::table - .left_join( - schema::note_scripts::table - .on(schema::notes::script_root.eq(schema::note_scripts::script_root.nullable())), - ) - .filter(schema::notes::note_id.eq_any(¬e_ids)); - let raw: Vec<_> = SelectDsl::select( - q, - (NoteRecordRawRow::as_select(), schema::note_scripts::script.nullable()), - ) - .load::<(NoteRecordRawRow, Option>)>(conn)?; - let records = vec_raw_try_into::( - raw.into_iter().map(NoteRecordWithScriptRawJoined::from), - )?; - Ok(records) -} - -/// Select the subset of note commitments that already exist in the notes table and were -/// committed at or before `up_to_block`. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// notes.note_commitment -/// FROM notes -/// WHERE note_commitment IN (?1) AND committed_at <= ?2 -/// ``` -pub(crate) fn select_existing_note_commitments( - conn: &mut SqliteConnection, - note_commitments: &[Word], - up_to_block: BlockNumber, -) -> Result, DatabaseError> { - QueryParamNoteCommitmentLimit::check(note_commitments.len())?; - - let note_commitments = serialize_vec(note_commitments.iter()); - - let raw_commitments = SelectDsl::select(schema::notes::table, schema::notes::note_id) - .filter(schema::notes::note_id.eq_any(¬e_commitments)) - .filter(schema::notes::committed_at.le(up_to_block.to_raw_sql())) - .load::>(conn)?; - - let commitments = raw_commitments - .into_iter() - .map(|commitment| Word::read_from_bytes(&commitment[..])) - .collect::, _>>()?; - - Ok(commitments) -} - -/// Select note inclusion proofs matching the note commitments, restricted to notes committed at -/// or before `up_to_block`. -/// -/// # Parameters -/// * `note_ids`: Set of note IDs to query -/// - Limit: 0 <= count <= 1000 -/// * `up_to_block`: Only notes committed at or before this block are returned -/// -/// # Returns -/// -/// - Empty map if no matching `note`. -/// - Otherwise, note inclusion proofs, which `note_id` matches the `NoteId` as bytes. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// committed_at, -/// note_id, -/// batch_index, -/// note_index, -/// inclusion_path -/// FROM -/// notes -/// WHERE -/// note_id IN (?1) AND -/// committed_at <= ?2 -/// ORDER BY -/// committed_at ASC -/// ``` -pub(crate) fn select_note_inclusion_proofs( - conn: &mut SqliteConnection, - note_commitments: &BTreeSet, - up_to_block: BlockNumber, -) -> Result, DatabaseError> { - QueryParamNoteCommitmentLimit::check(note_commitments.len())?; - - let note_commitments = serialize_vec(note_commitments.iter()); - - let raw_notes = SelectDsl::select( - schema::notes::table, - ( - schema::notes::committed_at, - schema::notes::note_id, - schema::notes::batch_index, - schema::notes::note_index, - schema::notes::inclusion_path, - ), - ) - .filter(schema::notes::note_id.eq_any(note_commitments)) - .filter(schema::notes::committed_at.le(up_to_block.to_raw_sql())) - .order_by(schema::notes::committed_at.asc()) - .load::<(i64, Vec, i32, i32, Vec)>(conn)?; - - Result::, _>::from_iter(raw_notes.iter().map( - |(block_num, note_id, batch_index, note_index, merkle_path)| { - let note_id = NoteId::read_from_bytes(¬e_id[..])?; - let block_num = BlockNumber::from_raw_sql(*block_num)?; - let node_index_in_block = - BlockNoteIndex::new(raw_sql_to_idx(*batch_index), raw_sql_to_idx(*note_index)) - .expect("batch and note index from DB should be valid") - .leaf_index_value(); - let merkle_path = SparseMerklePath::read_from_bytes(&merkle_path[..])?; - let proof = NoteInclusionProof::new(block_num, node_index_in_block, merkle_path)?; - Ok((note_id, proof)) - }, - )) -} - -/// Select note sync records matching the given note commitments. -/// -/// # Parameters -/// * `note_commitments`: Slice of note commitments to query -/// - Limit: 0 <= count <= 1000 -/// -/// # Returns -/// -/// - Empty map if no matching `note`. -/// - Otherwise, note sync records keyed by `NoteId`. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// committed_at, -/// batch_index, -/// note_index, -/// note_id, -/// note_commitment, -/// note_type, -/// sender, -/// tag, -/// attachment, -/// inclusion_path -/// FROM -/// notes -/// WHERE -/// note_commitment IN (?1) -/// ORDER BY -/// committed_at ASC -/// ``` -pub(crate) fn select_note_sync_records( - conn: &mut SqliteConnection, - note_ids: &[NoteId], -) -> Result, DatabaseError> { - QueryParamNoteCommitmentLimit::check(note_ids.len())?; - - let note_id_bytes: Vec> = note_ids.iter().map(|id| id.as_word().to_bytes()).collect(); - - let raw_notes = SelectDsl::select(schema::notes::table, NoteSyncRecordRawRow::as_select()) - .filter(schema::notes::note_id.eq_any(note_id_bytes)) - .order_by(schema::notes::committed_at.asc()) - .load::(conn)?; - - raw_notes - .into_iter() - .map(|raw_note| { - let note: NoteSyncRecord = raw_note.try_into()?; - Ok((note.note_id, note)) - }) - .collect() -} - -/// Maps each given nullifier to its note ID. -/// -/// Only public notes have a nullifier stored (`notes.nullifier` is NULL for private notes), so -/// private notes never match and are absent from the result. -/// -/// ```sql -/// SELECT -/// nullifier, -/// note_id -/// FROM -/// notes -/// WHERE -/// nullifier IN (?1) -/// ``` -pub(crate) fn select_note_ids_by_nullifier( - conn: &mut SqliteConnection, - nullifiers: &[Nullifier], -) -> Result, DatabaseError> { - if nullifiers.is_empty() { - return Ok(BTreeMap::new()); - } - - let nullifier_bytes: Vec> = nullifiers.iter().map(Nullifier::to_bytes).collect(); - let pairs = - SelectDsl::select(schema::notes::table, (schema::notes::nullifier, schema::notes::note_id)) - .filter(schema::notes::nullifier.eq_any(nullifier_bytes)) - .load::<(Option>, Vec)>(conn)?; - - let mut note_ids_by_nullifier = BTreeMap::new(); - for (nullifier, note_id) in pairs { - let Some(nullifier) = nullifier else { continue }; - let nullifier = Nullifier::read_from_bytes(&nullifier)?; - let note_id = NoteId::read_from_bytes(¬e_id)?; - note_ids_by_nullifier.insert(nullifier, note_id); - } - Ok(note_ids_by_nullifier) -} - -/// Returns the script for a note by its root. -/// -/// ```sql -/// SELECT -/// script_root, -/// script -/// FROM -/// note_scripts -/// WHERE -/// script_root = ?1 -/// ``` -pub(crate) fn select_note_script_by_root( - conn: &mut SqliteConnection, - root: Word, -) -> Result, DatabaseError> { - let raw = SelectDsl::select(schema::note_scripts::table, schema::note_scripts::script) - .filter(schema::note_scripts::script_root.eq(root.to_bytes())) - .get_result::>(conn) - .optional()?; - - raw.as_ref() - .map(|bytes| NoteScript::from_bytes(bytes)) - .transpose() - .map_err(Into::into) -} - -/// Loads the data necessary for a note sync across all matching blocks in the given range. -/// -/// Returns one [`NoteSyncUpdate`] per block that contains at least one note matching the -/// requested tags, ordered by block number ascending. -pub(crate) fn get_note_sync_multi( - conn: &mut SqliteConnection, - note_tags: &[u32], - block_range: RangeInclusive, - max_response_payload_bytes: usize, -) -> Result, NoteSyncError> { - let mut current_from = *block_range.start(); - let block_end = *block_range.end(); - let mut updates = Vec::new(); - let mut accumulated_size = 0usize; - - loop { - let notes = select_notes_since_block_by_tag(conn, note_tags, current_from..=block_end)?; - - let Some(block_num) = notes.first().map(|note| note.block_num) else { - break; - }; - - accumulated_size += NOTE_SYNC_BLOCK_OVERHEAD_BYTES + notes.len() * NOTE_SYNC_RECORD_BYTES; - - if !updates.is_empty() && accumulated_size > max_response_payload_bytes { - break; - } - - let block_header = select_block_header_by_block_num(conn, Some(block_num))? - .ok_or(NoteSyncError::EmptyBlockHeadersTable)?; - updates.push(NoteSyncUpdate { notes, block_header }); - current_from = block_num + 1; - } - - Ok(updates) -} - -#[derive(Debug, Clone, PartialEq, Selectable, Queryable, QueryableByName)] -#[diesel(table_name = schema::notes)] -#[diesel(check_for_backend(Sqlite))] -pub struct NoteSyncRecordRawRow { - pub committed_at: i64, // BlockNumber - #[diesel(embed)] - pub block_note_index: BlockNoteIndexRawRow, - pub note_id: Vec, // BlobDigest - #[diesel(embed)] - pub metadata: NoteMetadataRawRow, - pub inclusion_path: Vec, // SparseMerklePath -} - -impl TryInto for NoteSyncRecordRawRow { - type Error = DatabaseError; - fn try_into(self) -> Result { - let block_num = BlockNumber::from_raw_sql(self.committed_at)?; - let note_index = self.block_note_index.try_into()?; - - let note_id = NoteId::from_raw(Word::read_from_bytes(&self.note_id[..])?); - let inclusion_path = SparseMerklePath::read_from_bytes(&self.inclusion_path[..])?; - let (metadata, attachments) = self.metadata.try_into()?; - Ok(NoteSyncRecord { - block_num, - note_index, - note_id, - metadata, - attachments, - inclusion_path, - }) - } -} - -#[derive(Debug, Clone, PartialEq, Selectable, Queryable, QueryableByName)] -#[diesel(table_name = schema::notes)] -#[diesel(check_for_backend(Sqlite))] -pub struct NoteDetailsRawRow { - pub assets: Option>, - pub storage: Option>, - pub serial_num: Option>, -} - -// Note: One cannot use `#[diesel(embed)]` to structure this, it will yield a significant amount of -// errors when used with join and debugging is painful to put it mildly. -#[derive(Debug, Clone, PartialEq, Queryable)] -pub struct NoteRecordWithScriptRawJoined { - pub committed_at: i64, - - pub batch_index: i32, - pub note_index: i32, // index within batch - // #[diesel(embed)] - // pub note_index: BlockNoteIndexRaw, - pub note_id: Vec, - - pub note_type: i32, - pub sender: Vec, // AccountId - pub tag: i32, - pub attachment: Vec, - // #[diesel(embed)] - // pub metadata: NoteMetadataRaw, - pub assets: Option>, - pub storage: Option>, - pub serial_num: Option>, - - // #[diesel(embed)] - // pub details: NoteDetailsRaw, - pub inclusion_path: Vec, - pub script: Option>, // not part of notes::table! -} - -impl From<(NoteRecordRawRow, Option>)> for NoteRecordWithScriptRawJoined { - fn from((note, script): (NoteRecordRawRow, Option>)) -> Self { - let NoteRecordRawRow { - committed_at, - batch_index, - note_index, - note_id, - note_type, - sender, - tag, - attachment, - assets, - storage, - serial_num, - inclusion_path, - } = note; - Self { - committed_at, - batch_index, - note_index, - note_id, - note_type, - sender, - tag, - attachment, - assets, - storage, - serial_num, - inclusion_path, - script, - } - } -} - -impl TryInto for NoteRecordWithScriptRawJoined { - type Error = DatabaseError; - fn try_into(self) -> Result { - // let (raw, script) = self; - let raw = self; - let NoteRecordWithScriptRawJoined { - committed_at, - - batch_index, - note_index, - // block note index ^^^ - note_id, - - note_type, - sender, - tag, - attachment, - // metadata ^^^, - assets, - storage, - serial_num, - // details ^^^, - inclusion_path, - script, - .. - } = raw; - let index = BlockNoteIndexRawRow { batch_index, note_index }; - let metadata = NoteMetadataRawRow { note_type, sender, tag, attachment }; - let details = NoteDetailsRawRow { assets, storage, serial_num }; - - let (metadata, attachments) = metadata.try_into()?; - let committed_at = BlockNumber::from_raw_sql(committed_at)?; - let note_id = Word::read_from_bytes(¬e_id[..])?; - let script = script.map(|script| NoteScript::read_from_bytes(&script[..])).transpose()?; - let details = if let NoteDetailsRawRow { - assets: Some(assets), - storage: Some(storage), - serial_num: Some(serial_num), - } = details - { - let storage = NoteStorage::read_from_bytes(&storage[..])?; - let serial_num = Word::read_from_bytes(&serial_num[..])?; - let script = - script.ok_or_else(|| { - miden_node_db::DatabaseError::conversiont_from_sql::< - NoteRecipient, - DatabaseError, - _, - >(None) - })?; - let recipient = NoteRecipient::new(serial_num, script, storage); - let assets = NoteAssets::read_from_bytes(&assets[..])?; - Some(NoteDetails::new(assets, recipient)) - } else { - None - }; - let inclusion_path = SparseMerklePath::read_from_bytes(&inclusion_path[..])?; - let note_index = index.try_into()?; - Ok(NoteRecord { - block_num: committed_at, - note_index, - note_id, - metadata, - details, - attachments, - inclusion_path, - }) - } -} - -#[derive(Debug, Clone, PartialEq, Selectable, Queryable, QueryableByName)] -#[diesel(table_name = schema::notes)] -#[diesel(check_for_backend(Sqlite))] -pub struct NoteRecordRawRow { - pub committed_at: i64, - - pub batch_index: i32, - pub note_index: i32, // index within batch - pub note_id: Vec, - - pub note_type: i32, - pub sender: Vec, // AccountId - pub tag: i32, - pub attachment: Vec, - - pub assets: Option>, - pub storage: Option>, - pub serial_num: Option>, - - pub inclusion_path: Vec, -} - -#[derive(Debug, Clone, PartialEq, Selectable, Queryable, QueryableByName)] -#[diesel(table_name = schema::notes)] -#[diesel(check_for_backend(Sqlite))] -pub struct NoteMetadataRawRow { - note_type: i32, - sender: Vec, // AccountId - tag: i32, - attachment: Vec, -} - -#[expect(clippy::cast_sign_loss, clippy::cast_possible_truncation)] -impl TryInto<(NoteMetadata, NoteAttachments)> for NoteMetadataRawRow { - type Error = DatabaseError; - fn try_into(self) -> Result<(NoteMetadata, NoteAttachments), Self::Error> { - let sender = AccountId::read_from_bytes(&self.sender[..])?; - let note_type = NoteType::try_from(self.note_type as u8) - .map_err(miden_node_db::DatabaseError::conversiont_from_sql::)?; - let tag = NoteTag::new(self.tag as u32); - let attachments = if self.attachment.is_empty() { - NoteAttachments::empty() - } else { - NoteAttachments::read_from_bytes(&self.attachment)? - }; - let partial = PartialNoteMetadata::new(sender, note_type).with_tag(tag); - let metadata = NoteMetadata::new(partial, &attachments); - Ok((metadata, attachments)) - } -} - -#[derive(Debug, Clone, PartialEq, Selectable, Queryable, QueryableByName)] -#[diesel(table_name = schema::notes)] -#[diesel(check_for_backend(Sqlite))] -pub struct BlockNoteIndexRawRow { - pub batch_index: i32, - pub note_index: i32, // index within batch -} - -#[expect(clippy::cast_sign_loss, reason = "Indices are cast to usize for ease of use")] -impl TryInto for BlockNoteIndexRawRow { - type Error = DatabaseError; - fn try_into(self) -> Result { - let batch_index = self.batch_index as usize; - let note_index = self.note_index as usize; - let index = BlockNoteIndex::new(batch_index, note_index).ok_or_else(|| { - miden_node_db::DatabaseError::conversiont_from_sql::( - None, - ) - })?; - Ok(index) - } -} - -/// Insert notes to the DB using the given [`SqliteConnection`]. Public notes should also have a -/// nullifier. -/// -/// # Returns -/// -/// The number of affected rows. -/// -/// # Note -/// -/// The [`SqliteConnection`] object is not consumed. It's up to the caller to commit or rollback the -/// transaction. -#[miden_instrument( - target = COMPONENT, - err, -)] -pub(crate) fn insert_notes( - conn: &mut SqliteConnection, - notes: &[(NoteRecord, Option)], -) -> Result { - let count = diesel::insert_into(schema::notes::table) - .values(Vec::from_iter( - notes - .iter() - .map(|(note, nullifier)| NoteInsertRow::from((note.clone(), *nullifier))), - )) - .execute(conn)?; - Ok(count) -} - -/// Insert scripts to the DB using the given [`SqliteConnection`]. It inserts the scripts held by -/// the notes passed as parameter. If the script root already exists in the DB, it will be ignored. -/// -/// # Returns -/// -/// The number of affected rows. -/// -/// # Note -/// -/// The [`SqliteConnection`] object is not consumed. It's up to the caller to commit or rollback the -/// transaction. -#[miden_instrument( - target = COMPONENT, - err, -)] -pub(crate) fn insert_scripts<'a>( - conn: &mut SqliteConnection, - notes: impl IntoIterator, -) -> Result { - let values = Vec::from_iter(notes.into_iter().filter_map(|note| { - let note_details = note.details.as_ref()?; - Some(( - schema::note_scripts::script_root.eq(note_details.script().root().to_bytes()), - schema::note_scripts::script.eq(note_details.script().to_bytes()), - )) - })); - let count = diesel::insert_or_ignore_into(schema::note_scripts::table) - .values(values) - .execute(conn)?; - - Ok(count) -} - -#[derive(Debug, Clone, PartialEq, Insertable)] -#[diesel(table_name = schema::notes)] -pub struct NoteInsertRow { - pub committed_at: i64, - - pub batch_index: i32, - pub note_index: i32, // index within batch - - pub note_id: Vec, - - pub note_type: i32, - pub sender: Vec, // AccountId - pub tag: i32, - - pub network_note_type: i32, - pub target_account_id: Option>, - pub attachment: Vec, - pub inclusion_path: Vec, - pub consumed_at: Option, - pub nullifier: Option>, - pub assets: Option>, - pub storage: Option>, - pub script_root: Option>, - pub serial_num: Option>, -} - -impl From<(NoteRecord, Option)> for NoteInsertRow { - fn from((note, nullifier): (NoteRecord, Option)) -> Self { - let target_account_id = NetworkAccountTarget::try_from(¬e.attachments).ok(); - let network_note_type = if target_account_id.is_some() && !note.metadata.is_private() { - NetworkNoteType::SingleTarget - } else { - NetworkNoteType::None - }; - - let attachment_bytes = note.attachments.to_bytes(); - - Self { - committed_at: note.block_num.to_raw_sql(), - batch_index: idx_to_raw_sql(note.note_index.batch_idx()), - note_index: idx_to_raw_sql(note.note_index.note_idx_in_batch()), - note_id: note.note_id.to_bytes(), - note_type: note_type_to_raw_sql(note.metadata.note_type() as u8), - sender: note.metadata.sender().to_bytes(), - tag: note.metadata.tag().to_raw_sql(), - network_note_type: network_note_type.into(), - target_account_id: target_account_id.map(|t| t.target_id().to_bytes()), - attachment: attachment_bytes, - inclusion_path: note.inclusion_path.to_bytes(), - consumed_at: None::, // New notes are always unconsumed. - nullifier: nullifier.as_ref().map(Nullifier::to_bytes), - assets: note.details.as_ref().map(|d| d.assets().to_bytes()), - storage: note.details.as_ref().map(|d| d.storage().to_bytes()), - script_root: note.details.as_ref().map(|d| d.script().root().to_bytes()), - serial_num: note.details.as_ref().map(|d| d.serial_num().to_bytes()), - } - } -} diff --git a/crates/store/src/db/models/queries/nullifiers.rs b/crates/store/src/db/models/queries/nullifiers.rs deleted file mode 100644 index 688b5c6f0b..0000000000 --- a/crates/store/src/db/models/queries/nullifiers.rs +++ /dev/null @@ -1,287 +0,0 @@ -use std::num::NonZeroUsize; -use std::ops::RangeInclusive; - -use diesel::query_dsl::methods::SelectDsl; -use diesel::{ - ExpressionMethods, - QueryDsl, - Queryable, - QueryableByName, - RunQueryDsl, - Selectable, - SelectableHelper, - SqliteConnection, -}; -use miden_node_utils::limiter::{ - MAX_RESPONSE_PAYLOAD_BYTES, - QueryParamLimiter, - QueryParamNullifierPrefixLimit, -}; -use miden_node_utils::tracing::miden_instrument; -use miden_protocol::block::BlockNumber; -use miden_protocol::note::Nullifier; -use miden_protocol::utils::serde::{Deserializable, Serializable}; - -use super::DatabaseError; -use crate::COMPONENT; -use crate::db::models::conv::{SqlTypeConvert, nullifier_prefix_to_raw_sql}; -use crate::db::models::utils::{get_nullifier_prefix, vec_raw_try_into}; -use crate::db::{NullifierInfo, schema}; - -/// Returns nullifiers filtered by prefix within a block number range. -/// -/// # Parameters -/// * `prefix_len`: Length of nullifier prefix in bits -/// - Must be exactly 16 bits -/// * `nullifier_prefixes`: List of nullifier prefixes to filter by -/// - Limit: 0 <= count <= 1000 -/// -/// Each value of the `nullifier_prefixes` is only the `prefix_len` most significant bits -/// of the nullifier of interest to the client. This hides the details of the specific -/// nullifier being requested. Currently the only supported prefix length is 16 bits. -/// -/// # Returns -/// -/// A vector of [`NullifierInfo`] with the nullifiers and the block height at which they were -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// nullifier, -/// block_num -/// FROM -/// nullifiers -/// WHERE -/// nullifier_prefix IN (?1) AND -/// block_num >= ?2 AND -/// block_num <= ?3 -/// ORDER BY -/// block_num ASC -/// LIMIT -/// ?4 -/// ``` -pub(crate) fn select_nullifiers_by_prefix( - conn: &mut SqliteConnection, - prefix_len: u8, - nullifier_prefixes: &[u16], - block_range: RangeInclusive, -) -> Result<(Vec, BlockNumber), DatabaseError> { - // Size calculation: max 2^16 nullifiers per block × 36 bytes per nullifier = ~2.25MB - pub const NULLIFIER_BYTES: usize = 32; // digest size (nullifier) - pub const BLOCK_NUM_BYTES: usize = 4; // 32 bits per block number - pub const ROW_OVERHEAD_BYTES: usize = NULLIFIER_BYTES + BLOCK_NUM_BYTES; // 36 bytes - pub const MAX_ROWS: usize = MAX_RESPONSE_PAYLOAD_BYTES / ROW_OVERHEAD_BYTES; - // Pagination reports the last fully-included block, so it only makes progress if every block - // fits within a single page. A block that exceeded `MAX_ROWS` nullifiers would produce an empty - // page and stall clients forever on that block. - const _: () = assert!( - miden_protocol::MAX_INPUT_NOTES_PER_BLOCK <= MAX_ROWS, - "a block's nullifiers must fit in one response page or pagination cannot make progress", - ); - - assert_eq!(prefix_len, 16, "Only 16-bit prefixes are supported"); - - if block_range.is_empty() { - return Err(DatabaseError::InvalidBlockRange { - from: *block_range.start(), - to: *block_range.end(), - }); - } - - QueryParamNullifierPrefixLimit::check(nullifier_prefixes.len())?; - - let prefixes = nullifier_prefixes.iter().map(|prefix| nullifier_prefix_to_raw_sql(*prefix)); - let raw = SelectDsl::select( - schema::nullifiers::table, - NullifierWithoutPrefixRawRow::as_select(), - ) - .filter(schema::nullifiers::nullifier_prefix.eq_any(prefixes)) - .filter(schema::nullifiers::block_num.ge(block_range.start().to_raw_sql())) - .filter(schema::nullifiers::block_num.le(block_range.end().to_raw_sql())) - .order(schema::nullifiers::block_num.asc()) - // Request an additional row so we can determine whether this is the last page. - .limit(i64::try_from(MAX_ROWS + 1).expect("limit fits within i64")) - .load::(conn)?; - - // Discard the last block in the response (assumes more than one block may be present) - if let Some(last) = raw.last() - && raw.len() > MAX_ROWS - { - let last_block_num_i64 = last.block_num; - - let nullifiers = vec_raw_try_into( - raw.into_iter().take_while(|row| row.block_num != last_block_num_i64), - )?; - - let last_block_included = BlockNumber::from_raw_sql(last_block_num_i64.saturating_sub(1))?; - - Ok((nullifiers, last_block_included)) - } else { - Ok((vec_raw_try_into(raw)?, *block_range.end())) - } -} - -/// Select all nullifiers from the DB -/// -/// # Returns -/// -/// A vector with nullifiers and the block height at which they were created, or an error. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// nullifier, -/// block_num -/// FROM -/// nullifiers -/// ORDER BY -/// block_num ASC -/// ``` -#[cfg(test)] -pub(crate) fn select_all_nullifiers( - conn: &mut SqliteConnection, -) -> Result, DatabaseError> { - let nullifiers_raw = - SelectDsl::select(schema::nullifiers::table, NullifierWithoutPrefixRawRow::as_select()) - .load::(conn)?; - vec_raw_try_into(nullifiers_raw) -} - -/// Page of nullifiers returned by [`select_nullifiers_paged`]. -#[derive(Debug)] -pub struct NullifiersPage { - /// The nullifiers in this page. - pub nullifiers: Vec, - /// If `Some`, there are more results. Use this as the `after_nullifier` for the next page. - pub next_cursor: Option, -} - -/// Selects nullifiers with pagination. -/// -/// Returns up to `page_size` nullifiers, starting after `after_nullifier` if provided. -/// Results are ordered by nullifier bytes for stable pagination. -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT -/// nullifier, -/// block_num -/// FROM -/// nullifiers -/// WHERE -/// (nullifier > :after_nullifier OR :after_nullifier IS NULL) -/// ORDER BY -/// nullifier ASC -/// LIMIT :page_size + 1 -/// ``` -pub(crate) fn select_nullifiers_paged( - conn: &mut SqliteConnection, - page_size: NonZeroUsize, - after_nullifier: Option, -) -> Result { - // Fetch one extra to determine if there are more results - #[expect(clippy::cast_possible_wrap)] - let limit = (page_size.get() + 1) as i64; - - let mut query = - SelectDsl::select(schema::nullifiers::table, NullifierWithoutPrefixRawRow::as_select()) - .order_by(schema::nullifiers::nullifier.asc()) - .limit(limit) - .into_boxed(); - - if let Some(cursor) = after_nullifier { - query = query.filter(schema::nullifiers::nullifier.gt(cursor.to_bytes())); - } - - let nullifiers_raw = query.load::(conn)?; - let mut nullifiers: Vec = vec_raw_try_into(nullifiers_raw)?; - - // If we got more than page_size, there are more results - let next_cursor = if nullifiers.len() > page_size.get() { - nullifiers.pop(); // Remove the extra element - nullifiers.last().map(|info| info.nullifier) - } else { - None - }; - - Ok(NullifiersPage { nullifiers, next_cursor }) -} - -/// Insert nullifiers for a block into the database. -/// -/// # Parameters -/// * `nullifiers`: List of nullifiers to insert -/// - Limit: 0 <= count <= 1000 -/// * `block_num`: Block number to associate with the nullifiers -/// -/// # Returns -/// -/// The number of affected rows. -/// -/// # Note -/// -/// The [`SqliteConnection`] object is not consumed. It's up to the caller to commit or rollback the -/// transaction. -/// -/// # Raw SQL -/// -/// ```sql -/// UPDATE notes -/// SET consumed_at = ?1 -/// WHERE nullifier IN (?2); -/// -/// INSERT INTO nullifiers (nullifier, nullifier_prefix, block_num) -/// VALUES (?1, ?2, ?3) -/// ``` -#[miden_instrument( - target = COMPONENT, - err, -)] -pub(crate) fn insert_nullifiers_for_block( - conn: &mut SqliteConnection, - nullifiers: &[Nullifier], - block_num: BlockNumber, -) -> Result { - let serialized_nullifiers = - Vec::>::from_iter(nullifiers.iter().map(Nullifier::to_bytes)); - - let mut count = diesel::update(schema::notes::table) - .filter(schema::notes::nullifier.eq_any(&serialized_nullifiers)) - .set(schema::notes::consumed_at.eq(Some(block_num.to_raw_sql()))) - .execute(conn)?; - - count += diesel::insert_into(schema::nullifiers::table) - .values(Vec::from_iter(nullifiers.iter().zip(serialized_nullifiers.iter()).map( - |(nullifier, bytes)| { - ( - schema::nullifiers::nullifier.eq(bytes), - schema::nullifiers::nullifier_prefix - .eq(nullifier_prefix_to_raw_sql(get_nullifier_prefix(nullifier))), - schema::nullifiers::block_num.eq(block_num.to_raw_sql()), - ) - }, - ))) - .execute(conn)?; - - Ok(count) -} - -#[derive(Debug, Clone, Queryable, QueryableByName, Selectable)] -#[diesel(table_name = schema::nullifiers)] -#[diesel(check_for_backend(diesel::sqlite::Sqlite))] -pub struct NullifierWithoutPrefixRawRow { - pub nullifier: Vec, - pub block_num: i64, -} - -impl TryInto for NullifierWithoutPrefixRawRow { - type Error = DatabaseError; - fn try_into(self) -> Result { - let nullifier = Nullifier::read_from_bytes(&self.nullifier)?; - let block_num = BlockNumber::from_raw_sql(self.block_num)?; - Ok(NullifierInfo { nullifier, block_num }) - } -} diff --git a/crates/store/src/db/models/queries/transactions.rs b/crates/store/src/db/models/queries/transactions.rs deleted file mode 100644 index 249cf75439..0000000000 --- a/crates/store/src/db/models/queries/transactions.rs +++ /dev/null @@ -1,402 +0,0 @@ -use std::ops::RangeInclusive; - -use diesel::prelude::{Insertable, Queryable}; -use diesel::query_dsl::methods::SelectDsl; -use diesel::{ - BoolExpressionMethods, - ExpressionMethods, - QueryDsl, - QueryableByName, - RunQueryDsl, - Selectable, - SelectableHelper, - SqliteConnection, -}; -use miden_node_utils::limiter::{ - MAX_RESPONSE_PAYLOAD_BYTES, - QueryParamAccountIdLimit, - QueryParamLimiter, - QueryParamNoteCommitmentLimit, -}; -use miden_node_utils::tracing::miden_instrument; -use miden_protocol::account::AccountId; -use miden_protocol::block::BlockNumber; -use miden_protocol::note::{NoteHeader, NoteId, Nullifier}; -use miden_protocol::transaction::{ - InputNoteCommitment, - InputNotes, - OrderedTransactionHeaders, - TransactionHeader, - TransactionId, -}; -use miden_protocol::utils::serde::{Deserializable, Serializable}; - -use super::{DatabaseError, select_note_ids_by_nullifier, select_note_sync_records}; -use crate::COMPONENT; -use crate::db::models::conv::SqlTypeConvert; -use crate::db::models::serialize_vec; -use crate::db::schema; - -#[derive(Debug, Clone, PartialEq, Queryable, Selectable, QueryableByName)] -#[diesel(table_name = schema::transactions)] -#[diesel(check_for_backend(diesel::sqlite::Sqlite))] -pub struct TransactionRecordRaw { - account_id: Vec, - block_num: i64, - transaction_id: Vec, - initial_state_commitment: Vec, - final_state_commitment: Vec, - input_notes: Vec, - output_notes: Vec, - size_in_bytes: i64, -} - -/// Insert transactions to the DB using the given [`SqliteConnection`]. -/// -/// # Returns -/// -/// The number of affected rows. -/// -/// # Note -/// -/// The [`SqliteConnection`] object is not consumed. It's up to the caller to commit or rollback the -/// transaction. -#[miden_instrument( - target = COMPONENT, - err, -)] -pub(crate) fn insert_transactions( - conn: &mut SqliteConnection, - block_num: BlockNumber, - transactions: &OrderedTransactionHeaders, -) -> Result { - let rows: Vec<_> = transactions - .as_slice() - .iter() - .map(|tx| TransactionSummaryRowInsert::new(tx, block_num)) - .collect(); - - let count = diesel::insert_into(schema::transactions::table).values(rows).execute(conn)?; - Ok(count) -} - -#[derive(Debug, Clone, PartialEq, Insertable)] -#[diesel(table_name = schema::transactions)] -#[diesel(check_for_backend(diesel::sqlite::Sqlite))] -pub struct TransactionSummaryRowInsert { - transaction_id: Vec, - account_id: Vec, - block_num: i64, - initial_state_commitment: Vec, - final_state_commitment: Vec, - input_notes: Vec, - output_notes: Vec, - size_in_bytes: i64, -} - -impl TransactionSummaryRowInsert { - #[expect( - clippy::cast_possible_wrap, - reason = "We will not approach the item count where i64 and usize cause issues" - )] - fn new( - transaction_header: &miden_protocol::transaction::TransactionHeader, - block_num: BlockNumber, - ) -> Self { - const HEADER_BASE_SIZE_BYTES: usize = 4 + 32 + 16 + 64; - const INPUT_NOTE_COMMITMENT_SIZE_BYTES: usize = 64; - const OUTPUT_NOTE_SYNC_RECORD_SIZE_BYTES: usize = 700; - // Worst case, every input note resolves to a consumed-note reference (nullifier + note id) - // in the sync response. Counting it per input keeps input-heavy transactions under the cap. - const CONSUMED_NOTE_REF_SIZE_BYTES: usize = 64; - - // Serialize input notes as full InputNoteCommitments (nullifier + optional NoteHeader). - let input_notes: Vec = - transaction_header.input_notes().iter().cloned().collect(); - let input_notes_binary = input_notes.to_bytes(); - - // Serialize output notes as full NoteHeaders (NoteId + NoteMetadata). - let output_notes: Vec = transaction_header.output_notes().to_vec(); - let output_notes_binary = output_notes.to_bytes(); - - // Manually calculate the estimated size of the transaction header to avoid - // the cost of serialization. The size estimation includes: - // - 4 bytes for block number - // - 32 bytes for transaction ID - // - 16 bytes for account ID - // - 64 bytes for initial + final state commitments (32 bytes each) - // - ~64 bytes per input note (nullifier + optional NoteHeader) - // - ~64 bytes per input note for its possible consumed-note reference - // - ~700 bytes per output note sync record (metadata header + inclusion proof) - let input_notes_size = (transaction_header.input_notes().num_notes() as usize) - * (INPUT_NOTE_COMMITMENT_SIZE_BYTES + CONSUMED_NOTE_REF_SIZE_BYTES); - let output_notes_size = - transaction_header.output_notes().len() * OUTPUT_NOTE_SYNC_RECORD_SIZE_BYTES; - let size_in_bytes = (HEADER_BASE_SIZE_BYTES + input_notes_size + output_notes_size) as i64; - - Self { - transaction_id: transaction_header.id().to_bytes(), - account_id: transaction_header.account_id().to_bytes(), - block_num: block_num.to_raw_sql(), - initial_state_commitment: transaction_header.initial_state_commitment().to_bytes(), - final_state_commitment: transaction_header.final_state_commitment().to_bytes(), - input_notes: input_notes_binary, - output_notes: output_notes_binary, - size_in_bytes, - } - } -} - -/// Select complete transaction records for the given accounts and block range. -/// -/// # Parameters -/// * `account_ids`: List of account IDs to filter by -/// - Limit: 0 <= size <= 1000 -/// * `block_range`: Range of blocks to include inclusive -/// -/// # Returns -/// A tuple of (`last_block_included`, `transaction_records`) where: -/// - `last_block_included`: The highest block number included in the response -/// - `transaction_records`: Vector of transaction records, limited by payload size -/// -/// # Note -/// This function returns complete transaction record information including state commitments and -/// output note inclusion proofs, allowing for direct conversion to proto `TransactionRecord` -/// without loading full block data. We use a chunked loading strategy to prevent memory -/// exhaustion attacks and ensure predictable resource usage. -/// -/// # Raw SQL -/// ```sql -/// SELECT -/// account_id, -/// block_num, -/// transaction_id, -/// initial_state_commitment, -/// final_state_commitment, -/// input_notes, -/// output_notes, -/// size_in_bytes -/// FROM -/// transactions -/// WHERE -/// block_num >= ?1 -/// AND block_num <= ?2 -/// AND account_id IN (?3) -/// AND ( -/// block_num > ?4 OR (block_num = ?4 AND transaction_id > ?5) -/// ) -/// ORDER BY -/// block_num ASC, -/// transaction_id ASC -/// LIMIT -/// ?6 -/// ``` -/// Notes: -/// - Uses stable ordering (`block_num`, `transaction_id`) to ensure consistent results across -/// paginated queries. -/// - Uses cursor-based pagination. -/// - The query is executed in chunks of 1000 transactions to prevent loading excessive data and to -/// stop as soon as the accumulated size approaches the 4MB limit. -/// - Given the size of note records, 1000 records are guaranteed never to return more than about -/// 60MB of data. -pub fn select_transactions_records( - conn: &mut SqliteConnection, - account_ids: &[AccountId], - block_range: RangeInclusive, -) -> Result<(BlockNumber, Vec), DatabaseError> { - const NUM_TXS_PER_CHUNK: i64 = 1000; // Read 1000 transactions at a time - - QueryParamAccountIdLimit::check(account_ids.len())?; - - let max_payload_bytes = - i64::try_from(MAX_RESPONSE_PAYLOAD_BYTES).expect("payload limit fits within i64"); - - if block_range.is_empty() { - return Err(DatabaseError::InvalidBlockRange { - from: *block_range.start(), - to: *block_range.end(), - }); - } - - let desired_account_ids = serialize_vec(account_ids); - - // Read transactions in chunks to prevent loading excessive data and to stop as soon as we - // approach the size limit - let mut transactions = Vec::new(); - let mut total_size = 0i64; - let mut last_block_num: Option = None; - let mut last_transaction_id: Option> = None; - // Track the block number of the first transaction that did not fit within the payload cap. This - // is the explicit "we truncated" signal; the accumulated byte total cannot be used as a proxy, - // since a transaction can fail to fit while `total_size` is still below the cap. - let mut truncated_at_block: Option = None; - - loop { - let mut query = - SelectDsl::select(schema::transactions::table, TransactionRecordRaw::as_select()) - .filter(schema::transactions::block_num.ge(block_range.start().to_raw_sql())) - .filter(schema::transactions::block_num.le(block_range.end().to_raw_sql())) - .filter(schema::transactions::account_id.eq_any(&desired_account_ids)) - .into_boxed(); - - // Apply cursor-based pagination using the last seen (block_num, transaction_id) - if let (Some(last_block), Some(last_tx_id)) = (last_block_num, &last_transaction_id) { - query = query.filter( - schema::transactions::block_num - .gt(last_block) - .or(schema::transactions::block_num - .eq(last_block) - .and(schema::transactions::transaction_id.gt(last_tx_id))), - ); - } - - let chunk = query - .order(( - schema::transactions::block_num.asc(), - schema::transactions::transaction_id.asc(), - )) - .limit(NUM_TXS_PER_CHUNK) - .load::(conn) - .map_err(DatabaseError::from)?; - - // Add transactions from this chunk one by one until we hit the limit - let mut added_from_chunk = 0; - - for tx in chunk { - if total_size + tx.size_in_bytes <= max_payload_bytes { - total_size += tx.size_in_bytes; - last_block_num = Some(tx.block_num); - last_transaction_id = Some(tx.transaction_id.clone()); - transactions.push(tx); - added_from_chunk += 1; - } else { - // This transaction does not fit, so the response is truncated at its block. - truncated_at_block = Some(tx.block_num); - break; - } - } - - // Break if we truncated due to the payload cap, or the chunk was incomplete (i.e. the - // matching transactions are exhausted). - if truncated_at_block.is_some() || added_from_chunk < NUM_TXS_PER_CHUNK { - break; - } - } - - let Some(truncation_block) = truncated_at_block else { - // Every matching transaction in the range fit within the payload cap. - return Ok((*block_range.end(), with_output_note_proofs(conn, transactions)?)); - }; - - // We stopped within `truncation_block`, so that block may be partial. Block-based pagination - // can only report fully-included blocks, so drop every transaction belonging to the truncation - // block and report the previous block as the cursor. Transactions are ordered ascending by - // block number, so the truncation block's transactions form a contiguous suffix: - // `partition_point` locates the boundary and `truncate` drops the suffix in place, without - // allocating a new vector, with O(log n) complexity. - let complete_len = transactions.partition_point(|row| row.block_num < truncation_block); - transactions.truncate(complete_len); - - if transactions.is_empty() { - // A single block's transactions exceed the payload cap. Reporting `truncation_block - 1` - // here would tell the client to resume from `truncation_block`, which can never fit, so - // pagination would loop forever. Surface the condition instead of silently looping. - return Err(DatabaseError::TransactionPageExceedsPayloadLimit { - block_num: BlockNumber::from_raw_sql(truncation_block)?, - }); - } - - // SAFETY: block_num came from the database and was previously validated. Subtraction is safe - // under the assumption that genesis block (where it could fail) does not have any transactions. - let last_included_block = BlockNumber::from_raw_sql(truncation_block.saturating_sub(1))?; - Ok((last_included_block, with_output_note_proofs(conn, transactions)?)) -} - -fn with_output_note_proofs( - conn: &mut SqliteConnection, - raw_transactions: Vec, -) -> Result, DatabaseError> { - use miden_protocol::Word; - - // Pre-deserialize output notes to collect IDs for the batch lookup. - let mut tx_output_notes = Vec::with_capacity(raw_transactions.len()); - let mut all_note_ids: Vec = Vec::new(); - for raw in &raw_transactions { - let notes: Vec = Deserializable::read_from_bytes(&raw.output_notes)?; - all_note_ids.extend(notes.iter().map(NoteHeader::id)); - tx_output_notes.push(notes); - } - - let mut output_notes_by_id = std::collections::BTreeMap::new(); - for chunk in all_note_ids.chunks(QueryParamNoteCommitmentLimit::LIMIT) { - output_notes_by_id.extend(select_note_sync_records(conn, chunk)?); - } - - // Deserialize each transaction's input notes once and reuse them below. Authenticated inputs - // have no header and carry only a nullifier, so gather those nullifiers to look their note IDs - // up in one batch. - let mut tx_input_notes: Vec> = - Vec::with_capacity(raw_transactions.len()); - let mut authenticated_nullifiers: Vec = Vec::new(); - for raw in &raw_transactions { - let commitments: Vec = - Deserializable::read_from_bytes(&raw.input_notes)?; - for commitment in &commitments { - if commitment.header().is_none() { - authenticated_nullifiers.push(commitment.nullifier()); - } - } - tx_input_notes.push(commitments); - } - - let mut note_ids_by_nullifier = std::collections::BTreeMap::new(); - for chunk in authenticated_nullifiers.chunks(QueryParamNoteCommitmentLimit::LIMIT) { - note_ids_by_nullifier.extend(select_note_ids_by_nullifier(conn, chunk)?); - } - - // Deserialize remaining fields and assemble final records. - raw_transactions - .into_iter() - .zip(tx_output_notes) - .zip(tx_input_notes) - .map(|((raw, output_notes), input_notes)| { - let transaction_id = TransactionId::read_from_bytes(&raw.transaction_id)?; - // Collect inclusion proofs for committed output notes. Notes not found in the `notes` - // table were erased (created and consumed in the same batch). - let output_note_proofs = output_notes - .iter() - .filter_map(|note| { - let key = note.id(); - output_notes_by_id.get(&key).cloned() - }) - .collect(); - - // Build the side-channel refs. The input note commitments are left untouched, so the - // header and its commitment stay exactly as the transaction submitted them. - let consumed_note_refs = input_notes - .iter() - .filter(|commitment| commitment.header().is_none()) - .filter_map(|commitment| { - let nullifier = commitment.nullifier(); - note_ids_by_nullifier.get(&nullifier).map(|note_id| (nullifier, *note_id)) - }) - .collect(); - - let header = TransactionHeader::new_unchecked( - transaction_id, - AccountId::read_from_bytes(&raw.account_id)?, - Word::read_from_bytes(&raw.initial_state_commitment)?, - Word::read_from_bytes(&raw.final_state_commitment)?, - InputNotes::new_unchecked(input_notes), - output_notes, - ); - - Ok(crate::db::TransactionRecord { - block_num: BlockNumber::from_raw_sql(raw.block_num)?, - header, - output_note_proofs, - consumed_note_refs, - }) - }) - .collect() -} diff --git a/crates/store/src/db/models/utils.rs b/crates/store/src/db/models/utils.rs deleted file mode 100644 index 1415ee29eb..0000000000 --- a/crates/store/src/db/models/utils.rs +++ /dev/null @@ -1,66 +0,0 @@ -use diesel::{Connection, RunQueryDsl, SqliteConnection}; -use miden_protocol::note::Nullifier; -use miden_protocol::utils::serde::Serializable; - -use crate::errors::DatabaseError; - -/// Utility to convert an iterable container of containing `R`-typed values to a `Vec` and bail -/// at the first failing conversion -pub(crate) fn vec_raw_try_into>( - raw: impl IntoIterator, -) -> std::result::Result, >::Error> { - std::result::Result::, >::Error>::from_iter( - raw.into_iter().map(>::try_into), - ) -} - -/// Utility to convert an iterable container to a vector of byte blobs -pub(crate) fn serialize_vec<'a, D: Serializable + 'a>( - raw: impl IntoIterator, -) -> Vec> { - Vec::<_>::from_iter(raw.into_iter().map(::to_bytes)) -} - -/// Returns the high 16 bits of the provided nullifier. -pub fn get_nullifier_prefix(nullifier: &Nullifier) -> u16 { - (nullifier.most_significant_felt().as_canonical_u64() >> 48) as u16 -} - -/// Converts a slice of length `N` to an array, returns `None` if invariant -/// isn'crates/store/src/db/mod.rs upheld. -pub fn slice_to_array(bytes: &[u8]) -> Option<[u8; N]> { - if bytes.len() != N { - return None; - } - let mut arr = [0u8; N]; - arr.copy_from_slice(bytes); - Some(arr) -} - -#[expect(dead_code)] -#[inline] -pub fn from_be_to_u32(bytes: &[u8]) -> Option { - slice_to_array::<4>(bytes).map(u32::from_be_bytes) -} - -#[derive(diesel::QueryableByName, Debug)] -#[diesel(table_name = diesel::table)] -pub struct PragmaSchemaVersion { - #[diesel(sql_type = diesel::sql_types::Integer)] - pub schema_version: i32, -} - -/// Returns the schema version of the database. -#[expect(dead_code)] -#[expect( - clippy::cast_sign_loss, - reason = "schema version is always positive and we will never reach 0xEFFF_..._FFFF" -)] -pub fn schema_version(conn: &mut SqliteConnection) -> Result { - let schema_version = conn.transaction(|conn| { - let res = diesel::sql_query("SELECT schema_version FROM pragma_schema_version") - .get_result::(conn)?; - Ok::<_, DatabaseError>(res.schema_version as u32) - })?; - Ok(schema_version) -} diff --git a/crates/store/src/db/queries/account_row.rs b/crates/store/src/db/queries/account_row.rs new file mode 100644 index 0000000000..1437e4c4e1 --- /dev/null +++ b/crates/store/src/db/queries/account_row.rs @@ -0,0 +1,17 @@ +//! Row mapping shared by the `accounts` queries. + +use miden_node_db::DatabaseError; +use miden_node_db::sqlite::Row; +use miden_node_proto::domain::account::AccountSummary; +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::block::BlockNumber; + +/// Maps a row selecting `account_id, account_commitment, block_num` to an [`AccountSummary`]. +pub(super) fn account_summary_from_row(row: &Row<'_>) -> Result { + Ok(AccountSummary { + account_id: row.get::(0)?, + account_commitment: row.get::(1)?, + block_num: row.get::(2)?, + }) +} diff --git a/crates/store/src/db/queries/apply_block.rs b/crates/store/src/db/queries/apply_block.rs new file mode 100644 index 0000000000..41a374b0f4 --- /dev/null +++ b/crates/store/src/db/queries/apply_block.rs @@ -0,0 +1,48 @@ +//! Writes every table a committed block touches. + +use miden_node_db::sqlite::WriteTx; +use miden_protocol::block::SignedBlock; +use miden_protocol::note::Nullifier; + +use crate::db::NoteRecord; +use crate::db::queries::{ + PrecomputedPublicAccountStates, + insert_block_header, + insert_note_scripts, + insert_notes, + insert_nullifiers_for_block, + insert_transactions, + upsert_accounts, +}; +use crate::errors::DatabaseError; + +/// Apply a new block to the state. +/// +/// # Returns +/// +/// Number of records inserted and/or updated. +pub(crate) fn apply_block( + tx: &WriteTx<'_>, + block: &SignedBlock, + notes: &[(NoteRecord, Option)], + precomputed_public_states: &PrecomputedPublicAccountStates, +) -> Result { + let mut count = 0; + // Note: ordering here is important as the relevant tables have FK dependencies. + count += insert_block_header(tx, block.header(), block.signatures())?; + count += upsert_accounts( + tx, + block.body().updated_accounts(), + block.header().block_num(), + precomputed_public_states, + )?; + count += insert_note_scripts(tx, notes.iter().map(|(note, _)| note))?; + count += insert_notes(tx, notes)?; + count += insert_transactions(tx, block.header().block_num(), block.body().transactions())?; + count += insert_nullifiers_for_block( + tx, + block.body().created_nullifiers(), + block.header().block_num(), + )?; + Ok(count) +} diff --git a/crates/store/src/db/queries/block_header_row.rs b/crates/store/src/db/queries/block_header_row.rs new file mode 100644 index 0000000000..228f10f355 --- /dev/null +++ b/crates/store/src/db/queries/block_header_row.rs @@ -0,0 +1,22 @@ +//! Row mapping shared by the `block_headers` queries. + +use miden_node_db::DatabaseError; +use miden_node_db::sqlite::Row; +use miden_protocol::Word; +use miden_protocol::block::BlockHeader; + +use crate::db::BlockHeaderCommitment; + +/// Maps a `SELECT block_header, commitment` row to its [`BlockHeader`]. +/// +/// The stored commitment is only read to assert, in debug builds, that it matches the header it was +/// stored alongside: we are bust if that invariant does not hold. +pub(super) fn block_header_from_row(row: &Row<'_>) -> Result { + let block_header = row.get::(0)?; + debug_assert_eq!( + BlockHeaderCommitment::new(&block_header), + BlockHeaderCommitment(row.get::(1)?), + "stored block header commitment disagrees with the stored header", + ); + Ok(block_header) +} diff --git a/crates/store/src/db/queries/get_note_sync_multi/mod.rs b/crates/store/src/db/queries/get_note_sync_multi/mod.rs new file mode 100644 index 0000000000..23dcea8311 --- /dev/null +++ b/crates/store/src/db/queries/get_note_sync_multi/mod.rs @@ -0,0 +1,58 @@ +//! Loads the data for a note sync across every matching block in a range. + +use std::ops::RangeInclusive; + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::block::BlockNumber; + +use crate::db::NoteSyncUpdate; +use crate::db::queries::{select_block_header_by_block_num, select_notes_since_block_by_tag}; +use crate::errors::NoteSyncError; + +/// Estimated byte size of a [`NoteSyncUpdate`] excluding its notes. +/// +/// `BlockHeader` (~341 bytes) + MMR proof with 32 siblings (~1216 bytes). +pub(crate) const NOTE_SYNC_BLOCK_OVERHEAD_BYTES: usize = 1600; + +/// Estimated byte size of a single [`NoteSyncRecord`](crate::db::NoteSyncRecord). +/// +/// Note ID (~38 bytes) + index + sync metadata with up to four attachment entries (~200 bytes) + +/// sparse merkle path with 16 siblings (~608 bytes). +pub(crate) const NOTE_SYNC_RECORD_BYTES: usize = 900; + +/// Loads the data necessary for a note sync across all matching blocks in the given range. +/// +/// Returns one [`NoteSyncUpdate`] per block that contains at least one note matching the +/// requested tags, ordered by block number ascending. +pub(crate) fn get_note_sync_multi( + tx: &ReadTx<'_>, + note_tags: &[u32], + block_range: RangeInclusive, + max_response_payload_bytes: usize, +) -> Result, NoteSyncError> { + let mut current_from = *block_range.start(); + let block_end = *block_range.end(); + let mut updates = Vec::new(); + let mut accumulated_size = 0usize; + + loop { + let notes = select_notes_since_block_by_tag(tx, note_tags, current_from..=block_end)?; + + let Some(block_num) = notes.first().map(|note| note.block_num) else { + break; + }; + + accumulated_size += NOTE_SYNC_BLOCK_OVERHEAD_BYTES + notes.len() * NOTE_SYNC_RECORD_BYTES; + + if !updates.is_empty() && accumulated_size > max_response_payload_bytes { + break; + } + + let block_header = select_block_header_by_block_num(tx, Some(block_num))? + .ok_or(NoteSyncError::EmptyBlockHeadersTable)?; + updates.push(NoteSyncUpdate { notes, block_header }); + current_from = block_num + 1; + } + + Ok(updates) +} diff --git a/crates/store/src/db/queries/insert_account_storage_map_value/close_storage_map_value_validity.sql b/crates/store/src/db/queries/insert_account_storage_map_value/close_storage_map_value_validity.sql new file mode 100644 index 0000000000..7454a2fe8e --- /dev/null +++ b/crates/store/src/db/queries/insert_account_storage_map_value/close_storage_map_value_validity.sql @@ -0,0 +1,10 @@ +-- Closes the previous version of a storage-map entry at the block that supersedes it. +-- +-- Only the open-ended row (`valid_until` at the sentinel) can be the previous version, so matching +-- on it both selects that row and makes the update idempotent. +UPDATE account_storage_map_values +SET valid_until = ?1 +WHERE account_id = ?2 + AND slot_name = ?3 + AND key = ?4 + AND valid_until = ?5 diff --git a/crates/store/src/db/queries/insert_account_storage_map_value/insert_storage_map_value.sql b/crates/store/src/db/queries/insert_account_storage_map_value/insert_storage_map_value.sql new file mode 100644 index 0000000000..e659612419 --- /dev/null +++ b/crates/store/src/db/queries/insert_account_storage_map_value/insert_storage_map_value.sql @@ -0,0 +1,3 @@ +-- Inserts a storage-map entry as the current version of its key, valid from `block_num` onwards. +INSERT INTO account_storage_map_values (account_id, block_num, slot_name, key, value, valid_until) +VALUES (?1, ?2, ?3, ?4, ?5, ?6) diff --git a/crates/store/src/db/queries/insert_account_storage_map_value/mod.rs b/crates/store/src/db/queries/insert_account_storage_map_value/mod.rs new file mode 100644 index 0000000000..7df2aa79d0 --- /dev/null +++ b/crates/store/src/db/queries/insert_account_storage_map_value/mod.rs @@ -0,0 +1,68 @@ +//! Writes a versioned account storage-map value. + +use miden_node_db::sqlite::WriteTx; +use miden_protocol::Word; +use miden_protocol::account::{AccountId, StorageMapKey, StorageSlotName}; +use miden_protocol::block::BlockNumber; + +use crate::db::queries::VALID_FOREVER; +use crate::errors::DatabaseError; + +const SQL_CLOSE: &str = include_str!("close_storage_map_value_validity.sql"); +const SQL_INSERT: &str = include_str!("insert_storage_map_value.sql"); + +/// Inserts a versioned account storage-map value. +/// +/// The new row is inserted open-ended, and any previous open row for the same +/// `(account_id, slot_name, key)` tuple has its validity interval closed at `block_num` first. +/// +/// # Returns +/// +/// The total number of inserted and invalidated rows. +/// +/// # Errors +/// +/// Returns an error if the previous row cannot be invalidated or the new row cannot be inserted. +pub(crate) fn insert_account_storage_map_value( + tx: &WriteTx<'_>, + account_id: AccountId, + block_num: BlockNumber, + slot_name: &StorageSlotName, + key: StorageMapKey, + value: Word, +) -> Result { + insert_account_storage_map_value_inner(tx, account_id, block_num, slot_name, key, value, true) +} + +/// Inserts a versioned account storage-map value with optional previous-row invalidation. +/// +/// `invalidate_previous` may be disabled when inserting state for a new account, for which no +/// previous open row can exist. The inserted row is always open-ended. +/// +/// # Returns +/// +/// The total number of inserted and invalidated rows. +/// +/// # Errors +/// +/// Returns an error if the requested invalidation or insertion fails. +pub(super) fn insert_account_storage_map_value_inner( + tx: &WriteTx<'_>, + account_id: AccountId, + block_num: BlockNumber, + slot_name: &StorageSlotName, + key: StorageMapKey, + value: Word, + invalidate_previous: bool, +) -> Result { + let mut count = 0; + if invalidate_previous { + count += + tx.execute(SQL_CLOSE, &[&block_num, &account_id, slot_name, &key, &VALID_FOREVER])?; + } + + count += tx + .execute(SQL_INSERT, &[&account_id, &block_num, slot_name, &key, &value, &VALID_FOREVER])?; + + Ok(count) +} diff --git a/crates/store/src/db/queries/insert_account_vault_asset/close_vault_asset_validity.sql b/crates/store/src/db/queries/insert_account_vault_asset/close_vault_asset_validity.sql new file mode 100644 index 0000000000..0f3af6ad84 --- /dev/null +++ b/crates/store/src/db/queries/insert_account_vault_asset/close_vault_asset_validity.sql @@ -0,0 +1,9 @@ +-- Closes the previous version of a vault asset at the block that supersedes it. +-- +-- Only the open-ended row (`valid_until` at the sentinel) can be the previous version, so matching +-- on it both selects that row and makes the update idempotent. +UPDATE account_vault_assets +SET valid_until = ?1 +WHERE account_id = ?2 + AND vault_key = ?3 + AND valid_until = ?4 diff --git a/crates/store/src/db/queries/insert_account_vault_asset/insert_vault_asset.sql b/crates/store/src/db/queries/insert_account_vault_asset/insert_vault_asset.sql new file mode 100644 index 0000000000..5952264c1c --- /dev/null +++ b/crates/store/src/db/queries/insert_account_vault_asset/insert_vault_asset.sql @@ -0,0 +1,4 @@ +-- Inserts a vault asset as the current version of its key, valid from `block_num` onwards. A NULL +-- asset records the removal of that key. +INSERT INTO account_vault_assets (account_id, block_num, vault_key, asset, valid_until) +VALUES (?1, ?2, ?3, ?4, ?5) diff --git a/crates/store/src/db/queries/insert_account_vault_asset/mod.rs b/crates/store/src/db/queries/insert_account_vault_asset/mod.rs new file mode 100644 index 0000000000..fa74c9be48 --- /dev/null +++ b/crates/store/src/db/queries/insert_account_vault_asset/mod.rs @@ -0,0 +1,42 @@ +//! Writes a versioned account vault asset. + +use miden_node_db::sqlite::WriteTx; +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::asset::{Asset, AssetId}; +use miden_protocol::block::BlockNumber; + +use crate::db::queries::VALID_FOREVER; +use crate::errors::DatabaseError; + +const SQL_CLOSE: &str = include_str!("close_vault_asset_validity.sql"); +const SQL_INSERT: &str = include_str!("insert_vault_asset.sql"); + +/// Inserts an account vault asset row. +/// +/// The new row is inserted open-ended (`valid_until = VALID_FOREVER`); any existing open row with +/// the same `(account_id, vault_key)` tuple has its validity interval closed at `block_num`. A +/// `None` asset records the removal of that vault key. +/// +/// # Returns +/// +/// The number of affected rows. +pub(crate) fn insert_account_vault_asset( + tx: &WriteTx<'_>, + account_id: AccountId, + block_num: BlockNumber, + vault_key: AssetId, + asset: Option, +) -> Result { + // The column stores the asset id as its word representation. + let vault_key = Word::from(vault_key); + + // Close the previous version's validity interval at the new row's block. + let mut count = + tx.execute(SQL_CLOSE, &[&block_num, &account_id, &vault_key, &VALID_FOREVER])?; + + count += + tx.execute(SQL_INSERT, &[&account_id, &block_num, &vault_key, &asset, &VALID_FOREVER])?; + + Ok(count) +} diff --git a/crates/store/src/db/queries/insert_block_header/insert_block_header.sql b/crates/store/src/db/queries/insert_block_header/insert_block_header.sql new file mode 100644 index 0000000000..07b1ba8c50 --- /dev/null +++ b/crates/store/src/db/queries/insert_block_header/insert_block_header.sql @@ -0,0 +1,3 @@ +-- Inserts a block header together with the signatures that committed it. +INSERT INTO block_headers (block_num, block_header, signature, commitment) +VALUES (?1, ?2, ?3, ?4) diff --git a/crates/store/src/db/queries/insert_block_header/mod.rs b/crates/store/src/db/queries/insert_block_header/mod.rs new file mode 100644 index 0000000000..e89208b6c3 --- /dev/null +++ b/crates/store/src/db/queries/insert_block_header/mod.rs @@ -0,0 +1,33 @@ +//! Inserts a block header and the signatures that committed it. + +use miden_node_db::sqlite::WriteTx; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::{BlockHeader, BlockSignatures}; + +use crate::COMPONENT; +use crate::db::BlockHeaderCommitment; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("insert_block_header.sql"); + +/// Inserts a [`BlockHeader`] and its [`BlockSignatures`]. +/// +/// The header's commitment is stored alongside it so the chain MMR can be rebuilt without +/// deserializing every header. +/// +/// # Returns +/// +/// The number of affected rows. +#[miden_instrument( + target = COMPONENT, + err, +)] +pub(crate) fn insert_block_header( + tx: &WriteTx<'_>, + block_header: &BlockHeader, + signatures: &BlockSignatures, +) -> Result { + let commitment = BlockHeaderCommitment::new(block_header).word(); + + Ok(tx.execute(SQL, &[&block_header.block_num(), block_header, signatures, &commitment])?) +} diff --git a/crates/store/src/db/queries/insert_note_scripts/insert_note_script.sql b/crates/store/src/db/queries/insert_note_scripts/insert_note_script.sql new file mode 100644 index 0000000000..c5933a00cc --- /dev/null +++ b/crates/store/src/db/queries/insert_note_scripts/insert_note_script.sql @@ -0,0 +1,4 @@ +-- Inserts a note script, keyed by its root. Scripts are shared across notes, so re-inserting a +-- known root is a no-op rather than a constraint violation. +INSERT OR IGNORE INTO note_scripts (script_root, script) +VALUES (?1, ?2) diff --git a/crates/store/src/db/queries/insert_note_scripts/mod.rs b/crates/store/src/db/queries/insert_note_scripts/mod.rs new file mode 100644 index 0000000000..46866e4e7c --- /dev/null +++ b/crates/store/src/db/queries/insert_note_scripts/mod.rs @@ -0,0 +1,38 @@ +//! Inserts the note scripts held by a block's notes. + +use miden_node_db::sqlite::WriteTx; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::Word; + +use crate::COMPONENT; +use crate::db::NoteRecord; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("insert_note_script.sql"); + +/// Inserts the scripts held by the given notes. Notes without details (private notes) carry no +/// script, and a script root already in the table is left untouched. +/// +/// # Returns +/// +/// The number of affected rows. +#[miden_instrument( + target = COMPONENT, + err, +)] +pub(crate) fn insert_note_scripts<'a>( + tx: &WriteTx<'_>, + notes: impl IntoIterator, +) -> Result { + let mut count = 0; + for note in notes { + let Some(details) = note.details.as_ref() else { + continue; + }; + let script = details.script(); + // The column stores the root as its word representation. + let script_root = Word::from(script.root()); + count += tx.execute(SQL, &[&script_root, script])?; + } + Ok(count) +} diff --git a/crates/store/src/db/queries/insert_notes/insert_note.sql b/crates/store/src/db/queries/insert_notes/insert_note.sql new file mode 100644 index 0000000000..36184c08e2 --- /dev/null +++ b/crates/store/src/db/queries/insert_notes/insert_note.sql @@ -0,0 +1,25 @@ +-- Inserts a note committed by a block. +-- +-- Public notes carry their nullifier and detail columns (assets, storage, script root, serial +-- number); private notes store NULL for all of them. `consumed_at` is always NULL here: a freshly +-- committed note is unconsumed until a later block's nullifiers mark it. +INSERT INTO notes ( + committed_at, + batch_index, + note_index, + note_id, + note_type, + sender, + tag, + network_note_type, + target_account_id, + attachment, + inclusion_path, + consumed_at, + nullifier, + assets, + storage, + script_root, + serial_num +) +VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17) diff --git a/crates/store/src/db/queries/insert_notes/mod.rs b/crates/store/src/db/queries/insert_notes/mod.rs new file mode 100644 index 0000000000..680cd70489 --- /dev/null +++ b/crates/store/src/db/queries/insert_notes/mod.rs @@ -0,0 +1,117 @@ +//! Inserts the notes created by a block. + +use miden_node_db::sqlite::{DbValue, ToSqlValue, WriteTx}; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::block::BlockNumber; +use miden_protocol::note::{NoteAssets, NoteDetails, NoteStorage, Nullifier}; +use miden_standards::note::NetworkAccountTarget; + +use crate::COMPONENT; +use crate::db::NoteRecord; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("insert_note.sql"); + +// NETWORK NOTE TYPE +// ================================================================================================ + +/// Classifies network notes for database storage. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i64)] +pub(crate) enum NetworkNoteType { + /// Not a network note. + None = 0, + /// Single account target network note (has `NetworkAccountTarget` attachment). + SingleTarget = 1, +} + +impl ToSqlValue for NetworkNoteType { + fn to_sql_value(&self) -> DbValue { + DbValue::integer(*self as i64) + } +} + +// QUERY +// ================================================================================================ + +/// Inserts the notes created by a block. Public notes are inserted with their nullifier. +/// +/// # Returns +/// +/// The number of affected rows. +#[miden_instrument( + target = COMPONENT, + err, +)] +pub(crate) fn insert_notes( + tx: &WriteTx<'_>, + notes: &[(NoteRecord, Option)], +) -> Result { + let mut count = 0; + for (note, nullifier) in notes { + count += insert_note(tx, note, *nullifier)?; + } + Ok(count) +} + +/// Inserts a single note, deriving its network-note classification from its attachments. +fn insert_note( + tx: &WriteTx<'_>, + note: &NoteRecord, + nullifier: Option, +) -> Result { + let target_account_id: Option = NetworkAccountTarget::try_from(¬e.attachments) + .ok() + .map(|target| target.target_id()); + // A private note is never routed to a network account, even when it carries the attachment. + let network_note_type = if target_account_id.is_some() && !note.metadata.is_private() { + NetworkNoteType::SingleTarget + } else { + NetworkNoteType::None + }; + + let batch_index = index_column(note.note_index.batch_idx()); + let note_index = index_column(note.note_index.note_idx_in_batch()); + let note_type = note.metadata.note_type() as u8; + + // Private notes carry no details, in which case every detail column is NULL. + let details = note.details.as_ref(); + let assets: Option<&NoteAssets> = details.map(NoteDetails::assets); + let storage: Option<&NoteStorage> = details.map(NoteDetails::storage); + // The column stores the script root as its word representation. + let script_root: Option = details.map(|d| Word::from(d.script().root())); + let serial_num: Option = details.map(NoteDetails::serial_num); + + Ok(tx.execute( + SQL, + &[ + ¬e.block_num, + &batch_index, + ¬e_index, + ¬e.note_id, + ¬e_type, + ¬e.metadata.sender(), + ¬e.metadata.tag(), + &network_note_type, + &target_account_id, + ¬e.attachments, + ¬e.inclusion_path, + // New notes are always unconsumed. + &None::, + &nullifier, + &assets, + &storage, + &script_root, + &serial_num, + ], + )?) +} + +/// Narrows a note index to the `u32` the column stores. +/// +/// Both indices are bounded by the block's batch and note limits, which are far below `u32::MAX`. +fn index_column(index: usize) -> u32 { + u32::try_from(index).expect("note indices are bounded well below u32::MAX") +} diff --git a/crates/store/src/db/queries/insert_nullifiers_for_block/insert_nullifier.sql b/crates/store/src/db/queries/insert_nullifiers_for_block/insert_nullifier.sql new file mode 100644 index 0000000000..d734f365b3 --- /dev/null +++ b/crates/store/src/db/queries/insert_nullifiers_for_block/insert_nullifier.sql @@ -0,0 +1,4 @@ +-- Records a nullifier created by a block. The prefix column is indexed so nullifier lookups by +-- prefix never have to scan the full nullifier. +INSERT INTO nullifiers (nullifier, nullifier_prefix, block_num) +VALUES (?1, ?2, ?3) diff --git a/crates/store/src/db/queries/insert_nullifiers_for_block/mark_notes_consumed.sql b/crates/store/src/db/queries/insert_nullifiers_for_block/mark_notes_consumed.sql new file mode 100644 index 0000000000..3bbca7a68c --- /dev/null +++ b/crates/store/src/db/queries/insert_nullifiers_for_block/mark_notes_consumed.sql @@ -0,0 +1,8 @@ +-- Marks the notes spent by a block's nullifiers as consumed at that block. +-- +-- Nullifiers are bound as a single array parameter so the statement text stays constant regardless +-- of how many the block created; see `miden_node_db::sqlite::InList`. Nullifiers whose note is not +-- stored here (a private note, or one committed before this node's history) match nothing. +UPDATE notes +SET consumed_at = ?1 +WHERE nullifier IN (SELECT value FROM rarray(?2)) diff --git a/crates/store/src/db/queries/insert_nullifiers_for_block/mod.rs b/crates/store/src/db/queries/insert_nullifiers_for_block/mod.rs new file mode 100644 index 0000000000..8fde177c5f --- /dev/null +++ b/crates/store/src/db/queries/insert_nullifiers_for_block/mod.rs @@ -0,0 +1,47 @@ +//! Records the nullifiers created by a block and marks the notes they consume. + +use miden_node_db::sqlite::{InList, WriteTx}; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::BlockNumber; +use miden_protocol::note::Nullifier; +use miden_protocol::utils::serde::Serializable; + +use crate::COMPONENT; +use crate::db::utils::get_nullifier_prefix; +use crate::errors::DatabaseError; + +const SQL_MARK_NOTES_CONSUMED: &str = include_str!("mark_notes_consumed.sql"); +const SQL_INSERT_NULLIFIER: &str = include_str!("insert_nullifier.sql"); + +/// Inserts the nullifiers created by a block, and marks the notes they consume as consumed at that +/// block. +/// +/// # Parameters +/// * `nullifiers`: List of nullifiers to insert +/// - Limit: 0 <= count <= 1000 +/// * `block_num`: Block number to associate with the nullifiers +/// +/// # Returns +/// +/// The number of affected rows, counting both the consumed notes and the inserted nullifiers. +#[miden_instrument( + target = COMPONENT, + err, +)] +pub(crate) fn insert_nullifiers_for_block( + tx: &WriteTx<'_>, + nullifiers: &[Nullifier], + block_num: BlockNumber, +) -> Result { + let serialized = Vec::from_iter(nullifiers.iter().map(Serializable::to_bytes)); + let consumed = InList::from_blobs(serialized.iter().map(Vec::as_slice)); + + let mut count = tx.execute(SQL_MARK_NOTES_CONSUMED, &[&block_num, &consumed])?; + + for nullifier in nullifiers { + let prefix = get_nullifier_prefix(nullifier); + count += tx.execute(SQL_INSERT_NULLIFIER, &[nullifier, &prefix, &block_num])?; + } + + Ok(count) +} diff --git a/crates/store/src/db/queries/insert_transactions/insert_transaction.sql b/crates/store/src/db/queries/insert_transactions/insert_transaction.sql new file mode 100644 index 0000000000..2e1ce61bbd --- /dev/null +++ b/crates/store/src/db/queries/insert_transactions/insert_transaction.sql @@ -0,0 +1,16 @@ +-- Records a transaction included in a block. +-- +-- `size_in_bytes` is the estimated size of the sync record this transaction produces; it lets the +-- transaction-record queries stop before they exceed the response payload limit without having to +-- deserialize each row. +INSERT INTO transactions ( + transaction_id, + account_id, + block_num, + initial_state_commitment, + final_state_commitment, + input_notes, + output_notes, + size_in_bytes +) +VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) diff --git a/crates/store/src/db/queries/insert_transactions/mod.rs b/crates/store/src/db/queries/insert_transactions/mod.rs new file mode 100644 index 0000000000..122e66d63b --- /dev/null +++ b/crates/store/src/db/queries/insert_transactions/mod.rs @@ -0,0 +1,95 @@ +//! Inserts the transactions included in a block. + +use miden_node_db::sqlite::WriteTx; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::BlockNumber; +use miden_protocol::note::NoteHeader; +use miden_protocol::transaction::{ + InputNoteCommitment, + OrderedTransactionHeaders, + TransactionHeader, +}; +use miden_protocol::utils::serde::Serializable; + +use crate::COMPONENT; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("insert_transaction.sql"); + +/// Inserts the transactions included in a block. +/// +/// # Returns +/// +/// The number of affected rows. +#[miden_instrument( + target = COMPONENT, + err, +)] +pub(crate) fn insert_transactions( + tx: &WriteTx<'_>, + block_num: BlockNumber, + transactions: &OrderedTransactionHeaders, +) -> Result { + let mut count = 0; + for header in transactions.as_slice() { + count += insert_transaction(tx, block_num, header)?; + } + Ok(count) +} + +/// Inserts a single transaction header. +fn insert_transaction( + tx: &WriteTx<'_>, + block_num: BlockNumber, + header: &TransactionHeader, +) -> Result { + // Serialize input notes as full InputNoteCommitments (nullifier + optional NoteHeader). + let input_notes: Vec = header.input_notes().iter().cloned().collect(); + let input_notes = input_notes.to_bytes(); + + // Serialize output notes as full NoteHeaders (NoteId + NoteMetadata). + let output_notes: Vec = header.output_notes().to_vec(); + let output_notes = output_notes.to_bytes(); + + Ok(tx.execute( + SQL, + &[ + &header.id(), + &header.account_id(), + &block_num, + &header.initial_state_commitment(), + &header.final_state_commitment(), + &input_notes, + &output_notes, + &estimated_sync_record_size(header), + ], + )?) +} + +/// Estimates the size of the sync record this transaction produces. +/// +/// The estimate is computed from note counts rather than by serializing the record, which would +/// cost far more than the estimate is worth. It is deliberately an over-estimate, so a response +/// assembled under the limit is always within it. +#[expect( + clippy::cast_possible_wrap, + reason = "We will not approach the item count where i64 and usize cause issues" +)] +fn estimated_sync_record_size(header: &TransactionHeader) -> i64 { + // - 4 bytes for block number + // - 32 bytes for transaction ID + // - 16 bytes for account ID + // - 64 bytes for initial + final state commitments (32 bytes each) + const HEADER_BASE_SIZE_BYTES: usize = 4 + 32 + 16 + 64; + const INPUT_NOTE_COMMITMENT_SIZE_BYTES: usize = 64; + const OUTPUT_NOTE_SYNC_RECORD_SIZE_BYTES: usize = 700; + // Worst case, every input note resolves to a consumed-note reference (nullifier + note id) in + // the sync response. Counting it per input keeps input-heavy transactions under the cap. + const CONSUMED_NOTE_REF_SIZE_BYTES: usize = 64; + + let input_notes_size = (header.input_notes().num_notes() as usize) + * (INPUT_NOTE_COMMITMENT_SIZE_BYTES + CONSUMED_NOTE_REF_SIZE_BYTES); + let output_notes_size = header.output_notes().len() * OUTPUT_NOTE_SYNC_RECORD_SIZE_BYTES; + + (HEADER_BASE_SIZE_BYTES + input_notes_size + output_notes_size) as i64 +} diff --git a/crates/store/src/db/queries/mod.rs b/crates/store/src/db/queries/mod.rs new file mode 100644 index 0000000000..93ef4e8f44 --- /dev/null +++ b/crates/store/src/db/queries/mod.rs @@ -0,0 +1,219 @@ +//! Database query functions for the store. +//! +//! Each function takes a [`ReadTx`](miden_node_db::sqlite::ReadTx) or +//! [`WriteTx`](miden_node_db::sqlite::WriteTx) and is driven from a [`Db`](crate::db::Db) method +//! through [`DbReader::read`](miden_node_db::sqlite::DbReader::read) / +//! [`DbWriter::write`](miden_node_db::sqlite::DbWriter::write). One module per query, holding the +//! function and the `.sql` file it executes. + +mod account_row; +mod block_header_row; +mod note_row; + +use miden_node_db::DatabaseError; +use miden_node_db::sqlite::{DbValue, DbValueRef, FromSqlValue, InList, ToSqlValue}; + +// SHARED COLUMN TYPES +// ================================================================================================= + +/// Sentinel `valid_until` value marking a row as the current, open-ended version of its key. +/// +/// Versioned rows (`accounts`, `account_vault_assets`, `account_storage_map_values`) are +/// applicable for blocks in `[block_num, valid_until)`; updating a key closes the previous row's +/// interval by setting its `valid_until` to the new row's `block_num`. The open end is `i64::MAX` +/// rather than NULL so every validity predicate is a single range comparison that partial indexes +/// can serve. +pub(crate) const VALID_FOREVER: i64 = i64::MAX; + +/// Classifies accounts for database storage based on whether they are network accounts. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(i64)] +pub(crate) enum NetworkAccountType { + /// Not a network account. + None = 0, + /// A network account. + Network = 1, +} + +impl ToSqlValue for NetworkAccountType { + fn to_sql_value(&self) -> DbValue { + DbValue::integer(*self as i64) + } +} + +impl FromSqlValue for NetworkAccountType { + fn from_sql_value(value: DbValueRef<'_>) -> Result { + match value.as_i64()? { + 0 => Ok(Self::None), + 1 => Ok(Self::Network), + other => Err(DatabaseError::deserialization( + "NetworkAccountType", + InvalidNetworkAccountType(other), + )), + } + } +} + +#[derive(Debug, thiserror::Error)] +#[error("invalid network account type value {0}")] +struct InvalidNetworkAccountType(i64); + +/// Binds note tags for an `IN` list. +/// +/// Tags occupy the full `u32` range and are stored unsigned (see the `NoteTag` codec), so widening +/// them to `i64` is the same value the column holds. +fn note_tag_in_list(note_tags: &[u32]) -> InList { + InList::from_i64s(note_tags.iter().copied().map(i64::from)) +} + +// BLOCK QUERIES +// ================================================================================================= + +mod insert_block_header; +pub(crate) use insert_block_header::insert_block_header; + +mod select_all_block_header_commitments; +pub(crate) use select_all_block_header_commitments::select_all_block_header_commitments; + +mod select_block_header_and_signatures_by_block_num; +pub(crate) use select_block_header_and_signatures_by_block_num::select_block_header_and_signatures_by_block_num; + +mod select_block_header_by_block_num; +pub(crate) use select_block_header_by_block_num::select_block_header_by_block_num; + +mod select_block_headers; +pub(crate) use select_block_headers::select_block_headers; + +// NOTE QUERIES +// ================================================================================================= + +mod insert_note_scripts; +pub(crate) use insert_note_scripts::insert_note_scripts; + +mod insert_notes; +pub(crate) use insert_notes::insert_notes; + +mod get_note_sync_multi; +pub(crate) use get_note_sync_multi::get_note_sync_multi; +#[cfg(test)] +pub(crate) use get_note_sync_multi::{NOTE_SYNC_BLOCK_OVERHEAD_BYTES, NOTE_SYNC_RECORD_BYTES}; + +mod select_existing_note_commitments; +pub(crate) use select_existing_note_commitments::select_existing_note_commitments; + +mod select_note_ids_by_nullifier; +pub(crate) use select_note_ids_by_nullifier::select_note_ids_by_nullifier; + +mod select_note_inclusion_proofs; +pub(crate) use select_note_inclusion_proofs::select_note_inclusion_proofs; + +mod select_note_script_by_root; +pub(crate) use select_note_script_by_root::select_note_script_by_root; + +mod select_note_sync_records; +pub(crate) use select_note_sync_records::select_note_sync_records; + +mod select_notes_by_id; +pub(crate) use select_notes_by_id::select_notes_by_id; + +mod select_notes_since_block_by_tag; +pub(crate) use select_notes_since_block_by_tag::select_notes_since_block_by_tag; + +// NULLIFIER QUERIES +// ================================================================================================= + +mod insert_nullifiers_for_block; +pub(crate) use insert_nullifiers_for_block::insert_nullifiers_for_block; + +#[cfg(test)] +mod select_all_nullifiers; +#[cfg(test)] +pub(crate) use select_all_nullifiers::select_all_nullifiers; + +mod select_nullifiers_by_prefix; +pub(crate) use select_nullifiers_by_prefix::select_nullifiers_by_prefix; + +mod select_nullifiers_paged; +// `NullifiersPage` is part of the store's public API, so it is re-exported `pub` through the +// private module chain and made public again by `crate::db`. +pub use select_nullifiers_paged::NullifiersPage; +pub(crate) use select_nullifiers_paged::select_nullifiers_paged; + +// TRANSACTION QUERIES +// ================================================================================================= + +mod insert_transactions; +pub(crate) use insert_transactions::insert_transactions; + +mod select_transactions_records; +pub(crate) use select_transactions_records::select_transactions_records; + +// ACCOUNT QUERIES +// ================================================================================================= + +mod insert_account_storage_map_value; +pub(crate) use insert_account_storage_map_value::insert_account_storage_map_value; + +mod insert_account_vault_asset; +pub(crate) use insert_account_vault_asset::insert_account_vault_asset; + +mod prune_history; +pub use prune_history::HISTORICAL_BLOCK_RETENTION; +pub(crate) use prune_history::prune_history; + +mod upsert_accounts; +pub(crate) use upsert_accounts::{AccountRow, upsert_accounts}; +pub use upsert_accounts::{PrecomputedPublicAccountState, PrecomputedPublicAccountStates}; + +mod select_account_code_by_commitment; +pub(crate) use select_account_code_by_commitment::select_account_code_by_commitment; + +mod select_account_commitments_paged; +pub use select_account_commitments_paged::AccountCommitmentsPage; +pub(crate) use select_account_commitments_paged::select_account_commitments_paged; + +mod select_network_accounts_subset; +pub(crate) use select_network_accounts_subset::select_network_accounts_subset; + +mod select_public_account_ids_paged; +pub use select_public_account_ids_paged::PublicAccountIdsPage; +pub(crate) use select_public_account_ids_paged::select_public_account_ids_paged; + +mod select_public_account_state_roots_paged; +pub use select_public_account_state_roots_paged::PublicAccountStateRootsPage; +pub(crate) use select_public_account_state_roots_paged::select_public_account_state_roots_paged; + +mod select_account; +pub(crate) use select_account::select_account; + +mod select_account_header_with_storage_header_at_block; +pub(crate) use select_account_header_with_storage_header_at_block::select_account_header_with_storage_header_at_block; + +mod select_account_storage_map_values_paged; +#[cfg(test)] +pub(crate) use select_account_storage_map_values_paged::StorageMapValue; +pub use select_account_storage_map_values_paged::StorageMapValuesPage; +pub(crate) use select_account_storage_map_values_paged::select_account_storage_map_values_paged; + +mod select_account_vault_assets; +pub(crate) use select_account_vault_assets::select_account_vault_assets; + +mod select_account_vault_at_block; +pub(crate) use select_account_vault_at_block::select_account_vault_at_block; + +#[cfg(test)] +mod select_all_accounts; +#[cfg(test)] +pub(crate) use select_all_accounts::select_all_accounts; + +mod select_full_account; +pub(crate) use select_full_account::select_full_account; + +mod select_latest_account_storage; +pub(crate) use select_latest_account_storage::select_latest_account_storage; + +// BLOCK APPLICATION +// ================================================================================================= + +mod apply_block; +pub(crate) use apply_block::apply_block; diff --git a/crates/store/src/db/queries/note_row.rs b/crates/store/src/db/queries/note_row.rs new file mode 100644 index 0000000000..ee757d0575 --- /dev/null +++ b/crates/store/src/db/queries/note_row.rs @@ -0,0 +1,124 @@ +//! Row mapping shared by the `notes` queries. +//! +//! The `notes` queries all select their columns in one of two fixed orders, so a single mapper can +//! serve them: the sync-record order (see [`note_sync_record_from_row`]) and the full-record order +//! (see [`note_record_from_row`]), which extends it with the detail columns and the joined script. + +use miden_node_db::DatabaseError; +use miden_node_db::sqlite::Row; +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::block::{BlockNoteIndex, BlockNumber}; +use miden_protocol::crypto::merkle::SparseMerklePath; +use miden_protocol::note::{ + NoteAssets, + NoteAttachments, + NoteDetails, + NoteId, + NoteMetadata, + NoteRecipient, + NoteScript, + NoteStorage, + NoteTag, + NoteType, + PartialNoteMetadata, +}; + +use crate::db::{NoteRecord, NoteSyncRecord}; + +/// Maps a row selecting `committed_at, batch_index, note_index, note_id, note_type, sender, tag, +/// attachment, inclusion_path` to a [`NoteSyncRecord`]. +pub(super) fn note_sync_record_from_row(row: &Row<'_>) -> Result { + let (metadata, attachments) = note_metadata_from_row(row, 4)?; + + Ok(NoteSyncRecord { + block_num: row.get::(0)?, + note_index: block_note_index_from_row(row, 1)?, + note_id: NoteId::from_raw(row.get::(3)?), + metadata, + attachments, + inclusion_path: row.get::(8)?, + }) +} + +/// Maps a row selecting `committed_at, batch_index, note_index, note_id, note_type, sender, tag, +/// attachment, assets, storage, serial_num, inclusion_path, script` to a [`NoteRecord`]. +/// +/// The script column comes from the left join on `note_scripts` and is therefore nullable, as are +/// the detail columns; a note carries details only when all of them are present. +pub(super) fn note_record_from_row(row: &Row<'_>) -> Result { + let (metadata, attachments) = note_metadata_from_row(row, 4)?; + let details = note_details_from_row(row, 8)?; + + Ok(NoteRecord { + block_num: row.get::(0)?, + note_index: block_note_index_from_row(row, 1)?, + note_id: row.get::(3)?, + metadata, + details, + attachments, + inclusion_path: row.get::(11)?, + }) +} + +/// Maps `note_type, sender, tag, attachment` starting at `offset` to a note's metadata. +fn note_metadata_from_row( + row: &Row<'_>, + offset: usize, +) -> Result<(NoteMetadata, NoteAttachments), DatabaseError> { + let note_type = NoteType::try_from(row.get::(offset)?) + .map_err(|err| DatabaseError::deserialization("NoteType", err))?; + let sender = row.get::(offset + 1)?; + let tag = row.get::(offset + 2)?; + + // An empty blob means the note has no attachments, rather than being a serialized empty value. + let attachment = row.get::>(offset + 3)?; + let attachments = if attachment.is_empty() { + NoteAttachments::empty() + } else { + row.get::(offset + 3)? + }; + + let partial = PartialNoteMetadata::new(sender, note_type).with_tag(tag); + Ok((NoteMetadata::new(partial, &attachments), attachments)) +} + +/// Maps `batch_index, note_index` starting at `offset` to a [`BlockNoteIndex`]. +fn block_note_index_from_row( + row: &Row<'_>, + offset: usize, +) -> Result { + let batch_index = row.get::(offset)? as usize; + let note_index = row.get::(offset + 1)? as usize; + + BlockNoteIndex::new(batch_index, note_index).ok_or_else(|| { + DatabaseError::conversiont_from_sql::(None) + }) +} + +/// Maps `assets, storage, serial_num` starting at `offset`, plus the joined `script` column that +/// follows them, to a note's details. +/// +/// Private notes store none of these, in which case there are no details to reconstruct. +fn note_details_from_row( + row: &Row<'_>, + offset: usize, +) -> Result, DatabaseError> { + let assets = row.get::>(offset)?; + let storage = row.get::>(offset + 1)?; + let serial_num = row.get::>(offset + 2)?; + // The script sits after the `inclusion_path` column, which the details do not use. + let script = row.get::>(offset + 4)?; + + let (Some(assets), Some(storage), Some(serial_num)) = (assets, storage, serial_num) else { + return Ok(None); + }; + // A note with details must have a script; the join failing to find one means the note's script + // was never stored. + let script = script.ok_or_else(|| { + DatabaseError::conversiont_from_sql::(None) + })?; + + let recipient = NoteRecipient::new(serial_num, script, storage); + Ok(Some(NoteDetails::new(assets, recipient))) +} diff --git a/crates/store/src/db/queries/prune_history/mod.rs b/crates/store/src/db/queries/prune_history/mod.rs new file mode 100644 index 0000000000..5a453f7546 --- /dev/null +++ b/crates/store/src/db/queries/prune_history/mod.rs @@ -0,0 +1,119 @@ +//! Deletes account history that can no longer serve a read inside the retention window. + +use miden_node_db::sqlite::WriteTx; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::block::BlockNumber; + +use crate::COMPONENT; +use crate::db::queries::VALID_FOREVER; +use crate::errors::DatabaseError; + +const SQL_VAULT_ASSETS: &str = include_str!("prune_account_vault_assets.sql"); +const SQL_STORAGE_MAP_VALUES: &str = include_str!("prune_account_storage_map_values.sql"); +const SQL_ACCOUNT_CODES_FULL: &str = include_str!("prune_account_codes_full.sql"); +const SQL_ACCOUNT_CODES_WINDOWED: &str = include_str!("prune_account_codes_windowed.sql"); +const SQL_SELECT_PRUNE_PROGRESS: &str = include_str!("select_prune_progress.sql"); +const SQL_UPSERT_PRUNE_PROGRESS: &str = include_str!("upsert_prune_progress.sql"); + +/// The two pruning statements spell the open-ended sentinel out as a literal so SQLite can match it +/// against the partial cleanup indexes; that literal has to stay in step with [`VALID_FOREVER`]. +const _: () = assert!( + VALID_FOREVER == 9_223_372_036_854_775_807, + "the `valid_until != ` literal in the pruning statements is out of date" +); + +/// Number of historical blocks to retain for vault assets, storage map values, and account codes. +/// Rows whose validity interval ends at or below `prune_tip - HISTORICAL_BLOCK_RETENTION` will be +/// deleted; rows still valid anywhere inside the retention window (including all open-ended rows) +/// are retained. +pub const HISTORICAL_BLOCK_RETENTION: u32 = 50; + +/// Clean up old entries for all accounts, deleting entries that can no longer affect state +/// reconstruction at any block within the retention window. +/// +/// A row is applicable for blocks in `[block_num, valid_until)`, so it is deletable exactly when +/// its interval ends at or below the cutoff (`prune_tip - HISTORICAL_BLOCK_RETENTION`): it then +/// cannot cover any block inside the window. `prune_tip` is the effective tip for retention — it +/// lags the chain tip while old snapshot generations are still pinned by readers (see +/// [`crate::db::Db::apply_block`]). Account codes follow the same rule — a code is deleted only +/// when no account row whose interval reaches past the cutoff references it. +/// +/// # Returns +/// A tuple of `(vault_assets_deleted, storage_map_values_deleted, account_codes_deleted)` +#[miden_instrument( + target = COMPONENT, + err, + fields( + cutoff_block, + ), +)] +pub(crate) fn prune_history( + tx: &WriteTx<'_>, + prune_tip: BlockNumber, +) -> Result<(usize, usize, usize), DatabaseError> { + let cutoff_block = i64::from(prune_tip.as_u32().saturating_sub(HISTORICAL_BLOCK_RETENTION)); + tracing::Span::current().record("cutoff_block", cutoff_block); + + let vault_deleted = tx.execute(SQL_VAULT_ASSETS, &[&cutoff_block])?; + let storage_deleted = tx.execute(SQL_STORAGE_MAP_VALUES, &[&cutoff_block])?; + let codes_deleted = prune_account_codes(tx, cutoff_block)?; + + Ok((vault_deleted, storage_deleted, codes_deleted)) +} + +/// Deletes account codes that are no longer referenced by any account row that can serve a read +/// within the retention window. +/// +/// An account code is safe to delete when no `accounts` row whose validity interval reaches past +/// the cutoff (`valid_until > cutoff_block`) references it. That single predicate covers rows +/// inside the window, all open-ended (current) rows, and each account's baseline row — the row +/// still valid at the cutoff even though it was written before it. +/// +/// Rather than re-checking every code on every prune, only codes whose deletability could have +/// changed since the previous prune are examined. A code survived the previous prune because at +/// least one `accounts` row with `valid_until > prev_cutoff` referenced it. For it to be +/// deletable now, all such rows must have expired by the new cutoff — including the longest-lived +/// one, whose `valid_until` therefore lands inside `(prev_cutoff, cutoff_block]`. Scanning the +/// rows that expired in that window thus finds every code that could have become deletable. The +/// previous cutoff is persisted in `prune_progress` within the same transaction; when absent +/// (first prune after migration, or a fresh database) a full pass over all rows valid past the +/// cutoff runs instead. +/// +/// Correctness of the windowed candidate set rests on two invariants: +/// - Rows are only ever closed to the `block_num` of the block currently being applied, which is +/// always above the cutoff, so every expiry crosses the window of some later prune. A write path +/// that back-dated `valid_until` below the current cutoff would leak the code forever. +/// - Every `account_codes` row is inserted alongside an `accounts` row referencing it (see +/// [`upsert_accounts`](super::upsert_accounts)); an orphan code with no referencing row would +/// never become a candidate. +#[miden_instrument( + target = COMPONENT, + err, + fields( + cutoff_block, + ), +)] +fn prune_account_codes(tx: &WriteTx<'_>, cutoff_block: i64) -> Result { + let prev_cutoff = tx + .query(SQL_SELECT_PRUNE_PROGRESS, &[], |row| row.get::(0))? + .into_iter() + .next(); + + let deleted = match prev_cutoff { + // Codes are already pruned through this cutoff and nothing can become collectable while the + // cutoff stands still. Equality is the common case: the cutoff is clamped to zero for the + // first `HISTORICAL_BLOCK_RETENTION` blocks, and a pinned snapshot freezes the prune tip + // across consecutive blocks. A strictly greater `prev_cutoff` is unreachable through + // `apply_block` (the prune tip never regresses) but is guarded against so an out-of-order + // caller cannot move the marker backwards or run the delete with an inverted window. + Some(prev_cutoff) if prev_cutoff >= cutoff_block => return Ok(0), + Some(prev_cutoff) => { + tx.execute(SQL_ACCOUNT_CODES_WINDOWED, &[&prev_cutoff, &cutoff_block])? + }, + None => tx.execute(SQL_ACCOUNT_CODES_FULL, &[&cutoff_block])?, + }; + + tx.execute(SQL_UPSERT_PRUNE_PROGRESS, &[&cutoff_block])?; + + Ok(deleted) +} diff --git a/crates/store/src/db/queries/prune_history/prune_account_codes_full.sql b/crates/store/src/db/queries/prune_history/prune_account_codes_full.sql new file mode 100644 index 0000000000..b71bdf51fd --- /dev/null +++ b/crates/store/src/db/queries/prune_history/prune_account_codes_full.sql @@ -0,0 +1,16 @@ +-- Deletes account codes that no account row reaching past the retention cutoff still references. +-- +-- The full pass, used when no previous cutoff has been recorded (the first prune after migration, +-- or a fresh database). The single `valid_until > ?1` predicate covers rows inside the window, all +-- open-ended (current) rows, and each account's baseline row — the row still valid at the cutoff +-- even though it was written before it. +-- +-- The forced `idx_accounts_code_validity` covering index keeps the subquery an index-only range +-- scan, sized by rows valid at or after the cutoff rather than by total history. +DELETE FROM account_codes +WHERE code_commitment NOT IN ( + SELECT DISTINCT code_commitment + FROM accounts INDEXED BY idx_accounts_code_validity + WHERE code_commitment IS NOT NULL + AND valid_until > ?1 +) diff --git a/crates/store/src/db/queries/prune_history/prune_account_codes_windowed.sql b/crates/store/src/db/queries/prune_history/prune_account_codes_windowed.sql new file mode 100644 index 0000000000..78db2a4eef --- /dev/null +++ b/crates/store/src/db/queries/prune_history/prune_account_codes_windowed.sql @@ -0,0 +1,25 @@ +-- Deletes account codes that became collectable since the previous prune. +-- +-- Candidates are the codes referenced by rows whose validity interval ended inside +-- `(?1, ?2]` — the window between the previous cutoff and this one. A code that survived the +-- previous prune did so because some row with `valid_until > ?1` referenced it; for it to be +-- deletable now, the longest-lived such row must have expired by `?2`, which puts its +-- `valid_until` in exactly that window. The scan is an `idx_accounts_code_validity` index range, +-- so its cost scales with account updates since the previous prune, not with total history. +-- +-- Each candidate is then deleted only if the `idx_accounts_code_probe` existence probe finds no +-- row still referencing it past the new cutoff. +DELETE FROM account_codes +WHERE code_commitment IN ( + SELECT DISTINCT code_commitment + FROM accounts INDEXED BY idx_accounts_code_validity + WHERE code_commitment IS NOT NULL + AND valid_until > ?1 + AND valid_until <= ?2 +) +AND NOT EXISTS ( + SELECT 1 + FROM accounts INDEXED BY idx_accounts_code_probe + WHERE accounts.code_commitment = account_codes.code_commitment + AND accounts.valid_until > ?2 +) diff --git a/crates/store/src/db/queries/prune_history/prune_account_storage_map_values.sql b/crates/store/src/db/queries/prune_history/prune_account_storage_map_values.sql new file mode 100644 index 0000000000..4c67bcf433 --- /dev/null +++ b/crates/store/src/db/queries/prune_history/prune_account_storage_map_values.sql @@ -0,0 +1,8 @@ +-- Deletes storage-map rows whose validity interval ends at or below the retention cutoff. +-- +-- The literal sentinel term (rather than a bound parameter) lets SQLite prove the predicate implies +-- `idx_storage_cleanup`'s partial-index condition. It is kept in sync with `VALID_FOREVER` by a +-- compile-time assertion in this module. +DELETE FROM account_storage_map_values +WHERE valid_until != 9223372036854775807 + AND valid_until <= ?1 diff --git a/crates/store/src/db/queries/prune_history/prune_account_vault_assets.sql b/crates/store/src/db/queries/prune_history/prune_account_vault_assets.sql new file mode 100644 index 0000000000..3e2d98827d --- /dev/null +++ b/crates/store/src/db/queries/prune_history/prune_account_vault_assets.sql @@ -0,0 +1,8 @@ +-- Deletes vault-asset rows whose validity interval ends at or below the retention cutoff. +-- +-- The literal sentinel term (rather than a bound parameter) lets SQLite prove the predicate implies +-- `idx_vault_cleanup`'s partial-index condition. It is kept in sync with `VALID_FOREVER` by a +-- compile-time assertion in this module. +DELETE FROM account_vault_assets +WHERE valid_until != 9223372036854775807 + AND valid_until <= ?1 diff --git a/crates/store/src/db/queries/prune_history/select_prune_progress.sql b/crates/store/src/db/queries/prune_history/select_prune_progress.sql new file mode 100644 index 0000000000..edcb2acdb9 --- /dev/null +++ b/crates/store/src/db/queries/prune_history/select_prune_progress.sql @@ -0,0 +1,3 @@ +-- Returns the cutoff through which account-code pruning has completed, if any prune has run under +-- this schema. The table holds at most one row, pinned to `id = 0`. +SELECT codes_cutoff FROM prune_progress diff --git a/crates/store/src/db/queries/prune_history/upsert_prune_progress.sql b/crates/store/src/db/queries/prune_history/upsert_prune_progress.sql new file mode 100644 index 0000000000..a111760097 --- /dev/null +++ b/crates/store/src/db/queries/prune_history/upsert_prune_progress.sql @@ -0,0 +1,5 @@ +-- Records the cutoff through which account-code pruning has completed. Written in the same +-- transaction as the prune itself, so the marker is exact and crash-consistent. +INSERT INTO prune_progress (id, codes_cutoff) +VALUES (0, ?1) +ON CONFLICT(id) DO UPDATE SET codes_cutoff = excluded.codes_cutoff diff --git a/crates/store/src/db/queries/select_account/mod.rs b/crates/store/src/db/queries/select_account/mod.rs new file mode 100644 index 0000000000..b473c00449 --- /dev/null +++ b/crates/store/src/db/queries/select_account/mod.rs @@ -0,0 +1,37 @@ +//! Returns an account's latest committed summary, with full details for public accounts. + +use miden_node_db::sqlite::ReadTx; +use miden_node_proto::domain::account::AccountInfo; +use miden_protocol::account::AccountId; + +use crate::db::queries::account_row::account_summary_from_row; +use crate::db::queries::{VALID_FOREVER, select_full_account}; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_account_summary.sql"); + +/// Select account by ID. +/// +/// # Returns +/// +/// The latest account info, or an error. +pub(crate) fn select_account( + tx: &ReadTx<'_>, + account_id: AccountId, +) -> Result { + let summary = tx + .query(SQL, &[&account_id, &VALID_FOREVER], account_summary_from_row)? + .into_iter() + .next() + .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; + + // Backfill account details from database. For private accounts, we don't store full details in + // the database + let details = if account_id.is_public() { + Some(select_full_account(tx, account_id)?) + } else { + None + }; + + Ok(AccountInfo { summary, details }) +} diff --git a/crates/store/src/db/queries/select_account/select_account_summary.sql b/crates/store/src/db/queries/select_account/select_account_summary.sql new file mode 100644 index 0000000000..318a613a06 --- /dev/null +++ b/crates/store/src/db/queries/select_account/select_account_summary.sql @@ -0,0 +1,5 @@ +-- Returns the latest committed summary of the given account. +SELECT account_id, account_commitment, block_num +FROM accounts +WHERE account_id = ?1 + AND valid_until = ?2; diff --git a/crates/store/src/db/queries/select_account_code_by_commitment/mod.rs b/crates/store/src/db/queries/select_account_code_by_commitment/mod.rs new file mode 100644 index 0000000000..f737122d13 --- /dev/null +++ b/crates/store/src/db/queries/select_account_code_by_commitment/mod.rs @@ -0,0 +1,24 @@ +//! Returns account code by its commitment. + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::Word; + +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_account_code_by_commitment.sql"); + +/// Select account code by its commitment hash from the `account_codes` table. +/// +/// # Returns +/// +/// The account code bytes if found, or `None` if no code exists with that commitment. +pub(crate) fn select_account_code_by_commitment( + tx: &ReadTx<'_>, + code_commitment: Word, +) -> Result>, DatabaseError> { + // Invariant: `code_commitment` is the primary key, so there is at most one row. + Ok(tx + .query(SQL, &[&code_commitment], |row| row.get::>(0))? + .into_iter() + .next()) +} diff --git a/crates/store/src/db/queries/select_account_code_by_commitment/select_account_code_by_commitment.sql b/crates/store/src/db/queries/select_account_code_by_commitment/select_account_code_by_commitment.sql new file mode 100644 index 0000000000..f7559135cc --- /dev/null +++ b/crates/store/src/db/queries/select_account_code_by_commitment/select_account_code_by_commitment.sql @@ -0,0 +1,4 @@ +-- Returns the account code stored under the given commitment. +SELECT code +FROM account_codes +WHERE code_commitment = ?1; diff --git a/crates/store/src/db/queries/select_account_commitments_paged/mod.rs b/crates/store/src/db/queries/select_account_commitments_paged/mod.rs new file mode 100644 index 0000000000..c1dd68139d --- /dev/null +++ b/crates/store/src/db/queries/select_account_commitments_paged/mod.rs @@ -0,0 +1,56 @@ +//! Returns a page of latest account commitments, for rebuilding the account tree at startup. + +use std::num::NonZeroUsize; + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::utils::serde::Serializable; + +use crate::db::queries::VALID_FOREVER; +use crate::errors::DatabaseError; + +const SQL_FIRST_PAGE: &str = include_str!("select_account_commitments_page.sql"); +const SQL_AFTER_CURSOR: &str = include_str!("select_account_commitments_page_after.sql"); + +/// Page of account commitments returned by [`select_account_commitments_paged`]. +#[derive(Debug)] +pub struct AccountCommitmentsPage { + /// The account commitments in this page. + pub commitments: Vec<(AccountId, Word)>, + /// If `Some`, there are more results. Use this as the `after_account_id` for the next page. + pub next_cursor: Option, +} + +/// Selects account commitments with pagination. +/// +/// Returns up to `page_size` account commitments, starting after `after_account_id` if provided. +/// Results are ordered by `account_id` for stable pagination. +pub(crate) fn select_account_commitments_paged( + tx: &ReadTx<'_>, + page_size: NonZeroUsize, + after_account_id: Option, +) -> Result { + // Fetch one extra to determine if there are more results + let limit = i64::try_from(page_size.get() + 1).expect("page size fits within i64"); + + let map = + |row: &miden_node_db::sqlite::Row<'_>| Ok((row.get::(0)?, row.get::(1)?)); + let mut commitments = match after_account_id { + Some(cursor) => { + let cursor = cursor.to_bytes(); + tx.query(SQL_AFTER_CURSOR, &[&limit, &VALID_FOREVER, &cursor], map)? + }, + None => tx.query(SQL_FIRST_PAGE, &[&limit, &VALID_FOREVER], map)?, + }; + + // If we got more than page_size, there are more results + let next_cursor = if commitments.len() > page_size.get() { + commitments.pop(); // Remove the extra element + commitments.last().map(|(id, _)| *id) + } else { + None + }; + + Ok(AccountCommitmentsPage { commitments, next_cursor }) +} diff --git a/crates/store/src/db/queries/select_account_commitments_paged/select_account_commitments_page.sql b/crates/store/src/db/queries/select_account_commitments_paged/select_account_commitments_page.sql new file mode 100644 index 0000000000..ace1e08875 --- /dev/null +++ b/crates/store/src/db/queries/select_account_commitments_paged/select_account_commitments_page.sql @@ -0,0 +1,6 @@ +-- Returns the first page of latest account commitments, ordered by account id. +SELECT account_id, account_commitment +FROM accounts +WHERE valid_until = ?2 +ORDER BY account_id ASC +LIMIT ?1; diff --git a/crates/store/src/db/queries/select_account_commitments_paged/select_account_commitments_page_after.sql b/crates/store/src/db/queries/select_account_commitments_paged/select_account_commitments_page_after.sql new file mode 100644 index 0000000000..3b55db28d6 --- /dev/null +++ b/crates/store/src/db/queries/select_account_commitments_paged/select_account_commitments_page_after.sql @@ -0,0 +1,10 @@ +-- Returns the page of latest account commitments following the cursor. +-- +-- The cursor is a bare `>` comparison rather than a nullable parameter so the range scan can use the +-- index on `account_id`. +SELECT account_id, account_commitment +FROM accounts +WHERE valid_until = ?2 + AND account_id > ?3 +ORDER BY account_id ASC +LIMIT ?1; diff --git a/crates/store/src/db/queries/select_account_header_with_storage_header_at_block/mod.rs b/crates/store/src/db/queries/select_account_header_with_storage_header_at_block/mod.rs new file mode 100644 index 0000000000..1207718a44 --- /dev/null +++ b/crates/store/src/db/queries/select_account_header_with_storage_header_at_block/mod.rs @@ -0,0 +1,62 @@ +//! Returns an account's header as of a block. + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::account::{AccountHeader, AccountId, AccountStorageHeader}; +use miden_protocol::block::BlockNumber; +use miden_protocol::{Felt, Word}; + +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_account_header_at_block.sql"); + +/// The header columns as stored; every one of them is nullable for private accounts. +type AccountHeaderRow = (Option, Option, Option, Option); + +/// Queries the account header for a specific account at a specific block number. +/// +/// This reconstructs the [`AccountHeader`] by reading from the `accounts` table: +/// `account_id`, `nonce`, `code_commitment`, `storage_header`, `vault_root`. +/// +/// # Returns +/// +/// * `Ok(Some((AccountHeader, AccountStorageHeader)))` - The headers if found +/// * `Ok(None)` - If account doesn't exist at that block +/// * `Err(DatabaseError)` - If there's a database error +pub(crate) fn select_account_header_with_storage_header_at_block( + tx: &ReadTx<'_>, + account_id: AccountId, + block_num: BlockNumber, +) -> Result, DatabaseError> { + let row = tx + .query(SQL, &[&account_id, &block_num], |row| -> Result { + Ok(( + row.get::>(0)?, + row.get::>(1)?, + row.get::>(2)?, + row.get::>(3)?, + )) + })? + .into_iter() + .next(); + + let Some((code_commitment, nonce, storage_header, vault_root)) = row else { + return Ok(None); + }; + + // A private account stores none of these, in which case the header reads as empty/default. + let storage_header = match storage_header { + Some(header) => header, + None => AccountStorageHeader::new(Vec::new())?, + }; + let storage_commitment = storage_header.to_commitment(); + + let account_header = AccountHeader::new( + account_id, + nonce.unwrap_or(Felt::ZERO), + vault_root.unwrap_or_default(), + storage_commitment, + code_commitment.unwrap_or_default(), + ); + + Ok(Some((account_header, storage_header))) +} diff --git a/crates/store/src/db/queries/select_account_header_with_storage_header_at_block/select_account_header_at_block.sql b/crates/store/src/db/queries/select_account_header_with_storage_header_at_block/select_account_header_at_block.sql new file mode 100644 index 0000000000..e43199b80e --- /dev/null +++ b/crates/store/src/db/queries/select_account_header_with_storage_header_at_block/select_account_header_at_block.sql @@ -0,0 +1,9 @@ +-- Returns the columns making up the given account's header as of a block. +-- +-- The most recent row at or before the block holds the state in force then. +SELECT code_commitment, nonce, storage_header, vault_root +FROM accounts +WHERE account_id = ?1 + AND block_num <= ?2 +ORDER BY block_num DESC +LIMIT 1; diff --git a/crates/store/src/db/queries/select_account_storage_map_values_paged/mod.rs b/crates/store/src/db/queries/select_account_storage_map_values_paged/mod.rs new file mode 100644 index 0000000000..4b66abe537 --- /dev/null +++ b/crates/store/src/db/queries/select_account_storage_map_values_paged/mod.rs @@ -0,0 +1,103 @@ +//! Returns an account's storage map updates within a block range. + +use std::ops::RangeInclusive; + +use miden_node_db::SqlTypeConvert; +use miden_node_db::sqlite::ReadTx; +use miden_protocol::Word; +use miden_protocol::account::{AccountId, StorageMapKey, StorageSlotName}; +use miden_protocol::block::BlockNumber; + +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_account_storage_map_values_paged.sql"); + +/// A single storage map value at the block it was written. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StorageMapValue { + pub block_num: BlockNumber, + pub slot_name: StorageSlotName, + pub key: StorageMapKey, + pub value: Word, +} + +/// Page of storage map values returned by [`select_account_storage_map_values_paged`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct StorageMapValuesPage { + /// Highest block number included in `values`. If the page is empty, this will be `block_from`. + pub last_block_included: BlockNumber, + /// Storage map values + pub values: Vec, +} + +/// A storage map value row, with the block number left raw for the trimming below. +type StorageMapValueRow = (i64, StorageSlotName, StorageMapKey, Word); + +/// Select account storage map values within a block range (inclusive). +/// +/// ## Response +/// +/// * Response payload size: 0 <= size <= 2MB +/// * Storage map values per response: 0 <= count <= (2MB / (2*Word + u32 + u8)) + 1 +pub(crate) fn select_account_storage_map_values_paged( + tx: &ReadTx<'_>, + account_id: AccountId, + block_range: RangeInclusive, + limit: usize, +) -> Result { + if !account_id.is_public() { + return Err(DatabaseError::AccountNotPublic(account_id)); + } + + if block_range.is_empty() { + return Err(DatabaseError::InvalidBlockRange { + from: *block_range.start(), + to: *block_range.end(), + }); + } + + let row_limit = i64::try_from(limit + 1).expect("limit fits within i64"); + let raw = + tx.query(SQL, &[&account_id, block_range.start(), block_range.end(), &row_limit], |row| { + Ok(( + row.get::(0)?, + row.get::(1)?, + row.get::(2)?, + row.get::(3)?, + )) + })?; + + // If we got more rows than the limit, the last block may be incomplete so we drop it entirely + // and derive last_block_included from the remaining rows. + let last_block_num = raw.last().map(|(block_num, ..)| *block_num); + let (last_block_included, values) = if let Some(last_block_num) = last_block_num + && raw.len() > limit + { + let values = collect_storage_map_values( + raw.into_iter().take_while(|(block_num, ..)| *block_num != last_block_num), + )?; + let last_block_included = values.last().map_or(*block_range.start(), |v| v.block_num); + + (last_block_included, values) + } else { + (*block_range.end(), collect_storage_map_values(raw)?) + }; + + Ok(StorageMapValuesPage { last_block_included, values }) +} + +/// Converts raw `(block_num, slot_name, key, value)` rows into [`StorageMapValue`]s. +fn collect_storage_map_values( + rows: impl IntoIterator, +) -> Result, DatabaseError> { + rows.into_iter() + .map(|(block_num, slot_name, key, value)| { + Ok(StorageMapValue { + block_num: BlockNumber::from_raw_sql(block_num)?, + slot_name, + key, + value, + }) + }) + .collect() +} diff --git a/crates/store/src/db/queries/select_account_storage_map_values_paged/select_account_storage_map_values_paged.sql b/crates/store/src/db/queries/select_account_storage_map_values_paged/select_account_storage_map_values_paged.sql new file mode 100644 index 0000000000..f8ed2dc518 --- /dev/null +++ b/crates/store/src/db/queries/select_account_storage_map_values_paged/select_account_storage_map_values_paged.sql @@ -0,0 +1,8 @@ +-- Returns the given account's storage map updates within a block range, oldest first. +SELECT block_num, slot_name, key, value +FROM account_storage_map_values +WHERE account_id = ?1 + AND block_num >= ?2 + AND block_num <= ?3 +ORDER BY block_num ASC +LIMIT ?4; diff --git a/crates/store/src/db/queries/select_account_vault_assets/mod.rs b/crates/store/src/db/queries/select_account_vault_assets/mod.rs new file mode 100644 index 0000000000..32bed45a33 --- /dev/null +++ b/crates/store/src/db/queries/select_account_vault_assets/mod.rs @@ -0,0 +1,92 @@ +//! Returns an account's vault updates within a block range. + +use std::mem::size_of; +use std::ops::RangeInclusive; + +use miden_node_db::SqlTypeConvert; +use miden_node_db::sqlite::ReadTx; +use miden_node_utils::limiter::MAX_RESPONSE_PAYLOAD_BYTES; +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::asset::{Asset, AssetId}; +use miden_protocol::block::BlockNumber; + +use crate::db::AccountVaultValue; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_account_vault_assets.sql"); + +/// A vault update row, with the block number left raw for the trimming below. +type VaultAssetRow = (i64, Word, Option); + +/// Select account vault assets within a block range (inclusive). +/// +/// # Parameters +/// * `account_id`: Account ID to query +/// * `block_range`: Range of block numbers (inclusive) +/// * Response payload size: 0 <= size <= 2MB +/// +/// # Returns +/// +/// The updates, and the last block the response covers. When the rows would exceed the payload +/// limit, the trailing block is dropped whole. +pub(crate) fn select_account_vault_assets( + tx: &ReadTx<'_>, + account_id: AccountId, + block_range: RangeInclusive, +) -> Result<(BlockNumber, Vec), DatabaseError> { + // TODO: These limits should be given by the protocol. See miden-protocol/issues/1770 for more + // details + const ROW_OVERHEAD_BYTES: usize = 2 * size_of::() + size_of::(); // key + asset + block_num + const MAX_ROWS: usize = MAX_RESPONSE_PAYLOAD_BYTES / ROW_OVERHEAD_BYTES; + + if !account_id.is_public() { + return Err(DatabaseError::AccountNotPublic(account_id)); + } + + if block_range.is_empty() { + return Err(DatabaseError::InvalidBlockRange { + from: *block_range.start(), + to: *block_range.end(), + }); + } + + let limit = i64::try_from(MAX_ROWS + 1).expect("should fit within i64"); + let raw = + tx.query(SQL, &[&account_id, block_range.start(), block_range.end(), &limit], |row| { + Ok((row.get::(0)?, row.get::(1)?, row.get::>(2)?)) + })?; + + // If we got more rows than the limit, the last block may be incomplete so we drop it entirely + // and derive last_block_included from the remaining rows. + let last_block_num = raw.last().map(|(block_num, ..)| *block_num); + let (last_block_included, values) = if let Some(last_block_num) = last_block_num + && raw.len() > MAX_ROWS + { + let values = collect_vault_values( + raw.into_iter().take_while(|(block_num, ..)| *block_num != last_block_num), + )?; + let last_block_included = values.last().map_or(*block_range.start(), |v| v.block_num); + + (last_block_included, values) + } else { + (*block_range.end(), collect_vault_values(raw)?) + }; + + Ok((last_block_included, values)) +} + +/// Converts raw `(block_num, vault_key, asset)` rows into [`AccountVaultValue`]s. +fn collect_vault_values( + rows: impl IntoIterator, +) -> Result, DatabaseError> { + rows.into_iter() + .map(|(block_num, vault_key, asset)| { + Ok(AccountVaultValue { + block_num: BlockNumber::from_raw_sql(block_num)?, + vault_key: AssetId::try_from(vault_key)?, + asset, + }) + }) + .collect() +} diff --git a/crates/store/src/db/queries/select_account_vault_assets/select_account_vault_assets.sql b/crates/store/src/db/queries/select_account_vault_assets/select_account_vault_assets.sql new file mode 100644 index 0000000000..8e077dca74 --- /dev/null +++ b/crates/store/src/db/queries/select_account_vault_assets/select_account_vault_assets.sql @@ -0,0 +1,8 @@ +-- Returns the given account's vault updates within a block range, oldest first. +SELECT block_num, vault_key, asset +FROM account_vault_assets +WHERE account_id = ?1 + AND block_num >= ?2 + AND block_num <= ?3 +ORDER BY block_num ASC +LIMIT ?4; diff --git a/crates/store/src/db/queries/select_account_vault_at_block/mod.rs b/crates/store/src/db/queries/select_account_vault_at_block/mod.rs new file mode 100644 index 0000000000..0b0f27217a --- /dev/null +++ b/crates/store/src/db/queries/select_account_vault_at_block/mod.rs @@ -0,0 +1,31 @@ +//! Returns the assets in an account's vault as of a block. + +use miden_node_db::sqlite::ReadTx; +use miden_node_proto::domain::account::AccountVaultDetails; +use miden_protocol::account::AccountId; +use miden_protocol::asset::Asset; +use miden_protocol::block::BlockNumber; + +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_account_vault_at_block.sql"); + +/// Query vault assets at a specific block by finding the most recent update for each `vault_key`. +/// +/// The read is bounded to [`AccountVaultDetails::MAX_RETURN_ENTRIES`] + 1 rows so an over-the-limit +/// vault can be detected without materializing the whole set. +pub(crate) fn select_account_vault_at_block( + tx: &ReadTx<'_>, + account_id: AccountId, + block_num: BlockNumber, +) -> Result, DatabaseError> { + let limit = + i64::try_from(AccountVaultDetails::MAX_RETURN_ENTRIES + 1).expect("should fit within i64"); + + // A NULL asset marks a removal, and is filtered out here. + Ok(tx + .query(SQL, &[&account_id, &block_num, &limit], |row| row.get::>(0))? + .into_iter() + .flatten() + .collect()) +} diff --git a/crates/store/src/db/queries/select_account_vault_at_block/select_account_vault_at_block.sql b/crates/store/src/db/queries/select_account_vault_at_block/select_account_vault_at_block.sql new file mode 100644 index 0000000000..146a52f95f --- /dev/null +++ b/crates/store/src/db/queries/select_account_vault_at_block/select_account_vault_at_block.sql @@ -0,0 +1,10 @@ +-- Returns the assets in the given account's vault as of a block. +-- +-- Selects, per vault key, the row whose validity interval covers the block; a NULL asset marks a +-- removal and is skipped by the caller. +SELECT asset +FROM account_vault_assets +WHERE account_id = ?1 + AND block_num <= ?2 + AND valid_until > ?2 +LIMIT ?3; diff --git a/crates/store/src/db/queries/select_all_accounts/mod.rs b/crates/store/src/db/queries/select_all_accounts/mod.rs new file mode 100644 index 0000000000..1868ee81d2 --- /dev/null +++ b/crates/store/src/db/queries/select_all_accounts/mod.rs @@ -0,0 +1,26 @@ +//! Returns every account's latest committed state. + +use miden_node_db::sqlite::ReadTx; +use miden_node_proto::domain::account::AccountInfo; + +use crate::db::queries::account_row::account_summary_from_row; +use crate::db::queries::{VALID_FOREVER, select_full_account}; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_all_account_summaries.sql"); + +/// Select all accounts from the DB. +/// +/// Details are backfilled per account on a best-effort basis, as private accounts have none. +#[cfg(test)] +pub(crate) fn select_all_accounts(tx: &ReadTx<'_>) -> Result, DatabaseError> { + let summaries = tx.query(SQL, &[&VALID_FOREVER], account_summary_from_row)?; + + Ok(summaries + .into_iter() + .map(|summary| { + let details = select_full_account(tx, summary.account_id).ok(); + AccountInfo { summary, details } + }) + .collect()) +} diff --git a/crates/store/src/db/queries/select_all_accounts/select_all_account_summaries.sql b/crates/store/src/db/queries/select_all_accounts/select_all_account_summaries.sql new file mode 100644 index 0000000000..27e73d14f4 --- /dev/null +++ b/crates/store/src/db/queries/select_all_accounts/select_all_account_summaries.sql @@ -0,0 +1,5 @@ +-- Returns the latest committed summary of every account, oldest update first. +SELECT account_id, account_commitment, block_num +FROM accounts +WHERE valid_until = ?1 +ORDER BY block_num ASC; diff --git a/crates/store/src/db/queries/select_all_block_header_commitments/mod.rs b/crates/store/src/db/queries/select_all_block_header_commitments/mod.rs new file mode 100644 index 0000000000..d1a2be21ff --- /dev/null +++ b/crates/store/src/db/queries/select_all_block_header_commitments/mod.rs @@ -0,0 +1,16 @@ +//! Returns every stored block header commitment, for rebuilding the chain MMR at startup. + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::Word; + +use crate::db::BlockHeaderCommitment; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_all_block_header_commitments.sql"); + +/// Returns every stored block header commitment, ordered by block number ascending. +pub(crate) fn select_all_block_header_commitments( + tx: &ReadTx<'_>, +) -> Result, DatabaseError> { + Ok(tx.query(SQL, &[], |row| row.get::(0).map(BlockHeaderCommitment))?) +} diff --git a/crates/store/src/db/queries/select_all_block_header_commitments/select_all_block_header_commitments.sql b/crates/store/src/db/queries/select_all_block_header_commitments/select_all_block_header_commitments.sql new file mode 100644 index 0000000000..0cfd48d509 --- /dev/null +++ b/crates/store/src/db/queries/select_all_block_header_commitments/select_all_block_header_commitments.sql @@ -0,0 +1,4 @@ +-- Returns every stored block header commitment, ordered by block number. +SELECT commitment +FROM block_headers +ORDER BY block_num ASC; diff --git a/crates/store/src/db/queries/select_all_nullifiers/mod.rs b/crates/store/src/db/queries/select_all_nullifiers/mod.rs new file mode 100644 index 0000000000..7085f50490 --- /dev/null +++ b/crates/store/src/db/queries/select_all_nullifiers/mod.rs @@ -0,0 +1,21 @@ +//! Returns every stored nullifier. + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::block::BlockNumber; +use miden_protocol::note::Nullifier; + +use crate::db::NullifierInfo; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_all_nullifiers.sql"); + +/// Returns every stored nullifier with the block at which it was created, in no particular order. +#[cfg(test)] +pub(crate) fn select_all_nullifiers(tx: &ReadTx<'_>) -> Result, DatabaseError> { + Ok(tx.query(SQL, &[], |row| { + Ok(NullifierInfo { + nullifier: row.get::(0)?, + block_num: row.get::(1)?, + }) + })?) +} diff --git a/crates/store/src/db/queries/select_all_nullifiers/select_all_nullifiers.sql b/crates/store/src/db/queries/select_all_nullifiers/select_all_nullifiers.sql new file mode 100644 index 0000000000..a4120a5326 --- /dev/null +++ b/crates/store/src/db/queries/select_all_nullifiers/select_all_nullifiers.sql @@ -0,0 +1,3 @@ +-- Returns every stored nullifier with the block at which it was created. +SELECT nullifier, block_num +FROM nullifiers; diff --git a/crates/store/src/db/queries/select_block_header_and_signatures_by_block_num/mod.rs b/crates/store/src/db/queries/select_block_header_and_signatures_by_block_num/mod.rs new file mode 100644 index 0000000000..730818d021 --- /dev/null +++ b/crates/store/src/db/queries/select_block_header_and_signatures_by_block_num/mod.rs @@ -0,0 +1,21 @@ +//! Returns a block header together with the validator signatures it was committed with. + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::block::{BlockHeader, BlockNumber, BlockSignatures}; + +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_block_header_and_signatures_by_block_num.sql"); + +/// Returns the block header at `block_num` and its validator signatures. +pub(crate) fn select_block_header_and_signatures_by_block_num( + tx: &ReadTx<'_>, + block_num: BlockNumber, +) -> Result, DatabaseError> { + // Invariant: `block_num` is the primary key, so there is at most one row. + let rows = tx.query(SQL, &[&block_num], |row| { + Ok((row.get::(0)?, row.get::(1)?)) + })?; + + Ok(rows.into_iter().next()) +} diff --git a/crates/store/src/db/queries/select_block_header_and_signatures_by_block_num/select_block_header_and_signatures_by_block_num.sql b/crates/store/src/db/queries/select_block_header_and_signatures_by_block_num/select_block_header_and_signatures_by_block_num.sql new file mode 100644 index 0000000000..2c1419d468 --- /dev/null +++ b/crates/store/src/db/queries/select_block_header_and_signatures_by_block_num/select_block_header_and_signatures_by_block_num.sql @@ -0,0 +1,4 @@ +-- Returns the block header stored at the given block number together with its validator signatures. +SELECT block_header, signature +FROM block_headers +WHERE block_num = ?1; diff --git a/crates/store/src/db/queries/select_block_header_by_block_num/mod.rs b/crates/store/src/db/queries/select_block_header_by_block_num/mod.rs new file mode 100644 index 0000000000..3b5b2b7f24 --- /dev/null +++ b/crates/store/src/db/queries/select_block_header_by_block_num/mod.rs @@ -0,0 +1,27 @@ +//! Returns a single block header, either at a given block number or at the chain tip. + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::block::{BlockHeader, BlockNumber}; + +use crate::db::queries::block_header_row::block_header_from_row; +use crate::errors::DatabaseError; + +const SQL_BY_BLOCK_NUM: &str = include_str!("select_block_header_by_block_num.sql"); +const SQL_LATEST: &str = include_str!("select_latest_block_header.sql"); + +/// Returns the block header at `maybe_block_num`, or the latest block header when it is `None`. +/// +/// The two cases are separate statements rather than one statement with a nullable parameter, so the +/// lookup by block number stays an equality match on the primary key instead of an ordered scan. +pub(crate) fn select_block_header_by_block_num( + tx: &ReadTx<'_>, + maybe_block_num: Option, +) -> Result, DatabaseError> { + // Invariant: `block_num` is the primary key, so either statement returns at most one row. + let rows = match maybe_block_num { + Some(block_num) => tx.query(SQL_BY_BLOCK_NUM, &[&block_num], block_header_from_row)?, + None => tx.query(SQL_LATEST, &[], block_header_from_row)?, + }; + + Ok(rows.into_iter().next()) +} diff --git a/crates/store/src/db/queries/select_block_header_by_block_num/select_block_header_by_block_num.sql b/crates/store/src/db/queries/select_block_header_by_block_num/select_block_header_by_block_num.sql new file mode 100644 index 0000000000..a08dea4e9d --- /dev/null +++ b/crates/store/src/db/queries/select_block_header_by_block_num/select_block_header_by_block_num.sql @@ -0,0 +1,4 @@ +-- Returns the block header stored at the given block number. +SELECT block_header, commitment +FROM block_headers +WHERE block_num = ?1; diff --git a/crates/store/src/db/queries/select_block_header_by_block_num/select_latest_block_header.sql b/crates/store/src/db/queries/select_block_header_by_block_num/select_latest_block_header.sql new file mode 100644 index 0000000000..a6a3e32c0b --- /dev/null +++ b/crates/store/src/db/queries/select_block_header_by_block_num/select_latest_block_header.sql @@ -0,0 +1,5 @@ +-- Returns the block header of the chain tip. +SELECT block_header, commitment +FROM block_headers +ORDER BY block_num DESC +LIMIT 1; diff --git a/crates/store/src/db/queries/select_block_headers/mod.rs b/crates/store/src/db/queries/select_block_headers/mod.rs new file mode 100644 index 0000000000..587f490b6d --- /dev/null +++ b/crates/store/src/db/queries/select_block_headers/mod.rs @@ -0,0 +1,35 @@ +//! Returns the block headers for a set of block numbers. + +use miden_node_db::SqlTypeConvert; +use miden_node_db::sqlite::{InList, ReadTx}; +use miden_node_utils::limiter::{QueryParamBlockLimit, QueryParamLimiter}; +use miden_protocol::block::{BlockHeader, BlockNumber}; + +use crate::db::queries::block_header_row::block_header_from_row; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_block_headers.sql"); + +/// Returns the block headers stored at `blocks`, ordered by block number. +/// +/// Block numbers without a stored header are skipped, so the result may be shorter than `blocks`. +/// +/// # Parameters +/// +/// * `blocks`: the block numbers to retrieve, at most [`QueryParamBlockLimit`] of them. +pub(crate) fn select_block_headers( + tx: &ReadTx<'_>, + blocks: impl Iterator + Send, +) -> Result, DatabaseError> { + // The iterators are all deterministic, so is the conjunction. + // All calling sites do it equivalently, hence the below holds. + // + // + // And the conjunction is truthful: + // + QueryParamBlockLimit::check(blocks.size_hint().0)?; + + let blocks = InList::from_i64s(blocks.map(SqlTypeConvert::to_raw_sql)); + + Ok(tx.query(SQL, &[&blocks], block_header_from_row)?) +} diff --git a/crates/store/src/db/queries/select_block_headers/select_block_headers.sql b/crates/store/src/db/queries/select_block_headers/select_block_headers.sql new file mode 100644 index 0000000000..5bc1500aa0 --- /dev/null +++ b/crates/store/src/db/queries/select_block_headers/select_block_headers.sql @@ -0,0 +1,8 @@ +-- Returns the block headers stored at the given block numbers, ordered by block number. +-- +-- Block numbers are bound as a single array parameter so the statement text stays constant +-- regardless of how many are requested; see `miden_node_db::sqlite::InList`. +SELECT block_header, commitment +FROM block_headers +WHERE block_num IN (SELECT value FROM rarray(?1)) +ORDER BY block_num ASC; diff --git a/crates/store/src/db/queries/select_existing_note_commitments/mod.rs b/crates/store/src/db/queries/select_existing_note_commitments/mod.rs new file mode 100644 index 0000000000..ad646ab45e --- /dev/null +++ b/crates/store/src/db/queries/select_existing_note_commitments/mod.rs @@ -0,0 +1,31 @@ +//! Returns which of the given note commitments are already stored. + +use std::collections::HashSet; + +use miden_node_db::sqlite::{InList, ReadTx}; +use miden_node_utils::limiter::{QueryParamLimiter, QueryParamNoteCommitmentLimit}; +use miden_protocol::Word; +use miden_protocol::block::BlockNumber; +use miden_protocol::utils::serde::Serializable; + +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_existing_note_commitments.sql"); + +/// Select the subset of note commitments that already exist in the notes table and were committed +/// at or before `up_to_block`. +pub(crate) fn select_existing_note_commitments( + tx: &ReadTx<'_>, + note_commitments: &[Word], + up_to_block: BlockNumber, +) -> Result, DatabaseError> { + QueryParamNoteCommitmentLimit::check(note_commitments.len())?; + + let commitments = Vec::from_iter(note_commitments.iter().map(Serializable::to_bytes)); + let commitments = InList::from_blobs(commitments.iter().map(Vec::as_slice)); + + Ok(tx + .query(SQL, &[&commitments, &up_to_block], |row| row.get::(0))? + .into_iter() + .collect()) +} diff --git a/crates/store/src/db/queries/select_existing_note_commitments/select_existing_note_commitments.sql b/crates/store/src/db/queries/select_existing_note_commitments/select_existing_note_commitments.sql new file mode 100644 index 0000000000..6ee1dbe93b --- /dev/null +++ b/crates/store/src/db/queries/select_existing_note_commitments/select_existing_note_commitments.sql @@ -0,0 +1,5 @@ +-- Returns the subset of the given note commitments that is already stored at or before the block. +SELECT note_id +FROM notes +WHERE note_id IN (SELECT value FROM rarray(?1)) + AND committed_at <= ?2; diff --git a/crates/store/src/db/queries/select_full_account/mod.rs b/crates/store/src/db/queries/select_full_account/mod.rs new file mode 100644 index 0000000000..e8aa16e2ec --- /dev/null +++ b/crates/store/src/db/queries/select_full_account/mod.rs @@ -0,0 +1,59 @@ +//! Reconstructs a full account from the tables holding its latest committed state. + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::Felt; +use miden_protocol::account::{Account, AccountCode, AccountId}; +use miden_protocol::asset::{Asset, AssetVault}; + +use crate::db::queries::{VALID_FOREVER, select_latest_account_storage}; +use crate::errors::DatabaseError; + +const SQL_NONCE_AND_CODE: &str = include_str!("select_account_nonce_and_code.sql"); +const SQL_VAULT: &str = include_str!("select_account_vault.sql"); + +/// Reconstruct full Account from database tables for the latest account state +/// +/// This function queries the database tables to reconstruct a complete Account object: +/// - Code from `account_codes` table +/// - Nonce and storage header from `accounts` table +/// - Storage map entries from `account_storage_map_values` table +/// - Vault from `account_vault_assets` table +/// +/// # Note +/// +/// A stop-gap solution to retain store API and construct `AccountInfo` types. +/// The function should ultimately be removed, and any queries be served from the +/// `State` which contains an `SmtForest` to serve the latest and most recent +/// historical data. +// TODO: remove eventually once refactoring is complete +pub(crate) fn select_full_account( + tx: &ReadTx<'_>, + account_id: AccountId, +) -> Result { + // Get account metadata (nonce, code_commitment) and code in a single join query + let (nonce, code) = tx + .query(SQL_NONCE_AND_CODE, &[&account_id, &VALID_FOREVER], |row| { + Ok((row.get::>(0)?, row.get::(1)?)) + })? + .into_iter() + .next() + .ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; + + let nonce = nonce.ok_or_else(|| { + DatabaseError::DataCorrupted(format!("No nonce found for account {account_id}")) + })?; + + // Reconstruct storage using existing helper function + let storage = select_latest_account_storage(tx, account_id)?; + + // Reconstruct vault from account_vault_assets table; a NULL asset marks a removal. + let assets = tx + .query(SQL_VAULT, &[&account_id, &VALID_FOREVER], |row| row.get::>(0))? + .into_iter() + .flatten() + .collect::>(); + + let vault = AssetVault::new(&assets)?; + + Ok(Account::new(account_id, vault, storage, code, nonce, None)?) +} diff --git a/crates/store/src/db/queries/select_full_account/select_account_nonce_and_code.sql b/crates/store/src/db/queries/select_full_account/select_account_nonce_and_code.sql new file mode 100644 index 0000000000..958360ddd0 --- /dev/null +++ b/crates/store/src/db/queries/select_full_account/select_account_nonce_and_code.sql @@ -0,0 +1,6 @@ +-- Returns the nonce and code of the given account's latest committed state. +SELECT accounts.nonce, account_codes.code +FROM accounts +INNER JOIN account_codes ON accounts.code_commitment = account_codes.code_commitment +WHERE accounts.account_id = ?1 + AND accounts.valid_until = ?2; diff --git a/crates/store/src/db/queries/select_full_account/select_account_vault.sql b/crates/store/src/db/queries/select_full_account/select_account_vault.sql new file mode 100644 index 0000000000..2b71aec9cb --- /dev/null +++ b/crates/store/src/db/queries/select_full_account/select_account_vault.sql @@ -0,0 +1,7 @@ +-- Returns the assets currently held in the given account's vault. +-- +-- A NULL asset marks a removal, and is skipped by the caller. +SELECT asset +FROM account_vault_assets +WHERE account_id = ?1 + AND valid_until = ?2; diff --git a/crates/store/src/db/queries/select_latest_account_storage/mod.rs b/crates/store/src/db/queries/select_latest_account_storage/mod.rs new file mode 100644 index 0000000000..71d84ca91a --- /dev/null +++ b/crates/store/src/db/queries/select_latest_account_storage/mod.rs @@ -0,0 +1,105 @@ +//! Reconstructs an account's current storage from the header and its map entries. + +use std::collections::{BTreeMap, HashMap}; + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::Word; +use miden_protocol::account::{ + AccountId, + AccountStorage, + AccountStorageHeader, + StorageMap, + StorageMapKey, + StorageSlot, + StorageSlotName, + StorageSlotType, +}; + +use crate::db::queries::VALID_FOREVER; +use crate::errors::DatabaseError; + +const SQL_STORAGE_HEADER: &str = include_str!("select_account_storage_header.sql"); +const SQL_MAP_ENTRIES: &str = include_str!("select_account_storage_map_entries.sql"); + +/// An account's storage header together with its map entries, keyed by slot. +pub(crate) type StorageHeaderWithEntries = + (AccountStorageHeader, HashMap>); + +/// Reconstructs the account's current storage: value slots come from the header, map slots from the +/// stored map entries. +pub(crate) fn select_latest_account_storage( + tx: &ReadTx<'_>, + account_id: AccountId, +) -> Result { + let (storage_header, map_entries_by_slot) = + select_latest_account_storage_components(tx, account_id)?; + + // Reconstruct StorageSlots from header slots + map entries + let slots = storage_header + .slots() + .map(|slot_header| { + let slot = match slot_header.slot_type() { + StorageSlotType::Value => { + // For value slots, the header value IS the slot value + StorageSlot::with_value(slot_header.name().clone(), slot_header.value()) + }, + StorageSlotType::Map => { + // For map slots, reconstruct from map entries + let entries = + map_entries_by_slot.get(slot_header.name()).cloned().unwrap_or_default(); + StorageSlot::with_map( + slot_header.name().clone(), + StorageMap::with_entries(entries)?, + ) + }, + }; + Ok(slot) + }) + .collect::, DatabaseError>>()?; + + Ok(AccountStorage::new(slots)?) +} + +/// Fetch account storage header and all storage maps +pub(crate) fn select_latest_account_storage_components( + tx: &ReadTx<'_>, + account_id: AccountId, +) -> Result { + // The column is nullable, and the account may have no row at all. + let storage_blob = tx + .query(SQL_STORAGE_HEADER, &[&account_id, &VALID_FOREVER], |row| { + row.get::>(0) + })? + .into_iter() + .next() + .flatten(); + + let header = match storage_blob { + Some(header) => header, + None => AccountStorageHeader::new(Vec::new())?, + }; + + Ok((header, select_latest_storage_map_entries_all(tx, account_id)?)) +} + +// TODO this is expensive and should only be called from tests +fn select_latest_storage_map_entries_all( + tx: &ReadTx<'_>, + account_id: AccountId, +) -> Result>, DatabaseError> { + let map_values = tx.query(SQL_MAP_ENTRIES, &[&account_id, &VALID_FOREVER], |row| { + Ok(( + row.get::(0)?, + row.get::(1)?, + row.get::(2)?, + )) + })?; + + let mut map_entries_by_slot: HashMap> = + HashMap::new(); + for (slot_name, key, value) in map_values { + map_entries_by_slot.entry(slot_name).or_default().insert(key, value); + } + + Ok(map_entries_by_slot) +} diff --git a/crates/store/src/db/queries/select_latest_account_storage/select_account_storage_header.sql b/crates/store/src/db/queries/select_latest_account_storage/select_account_storage_header.sql new file mode 100644 index 0000000000..3e9cc3e9d9 --- /dev/null +++ b/crates/store/src/db/queries/select_latest_account_storage/select_account_storage_header.sql @@ -0,0 +1,5 @@ +-- Returns the storage header of the given account's latest committed state. +SELECT storage_header +FROM accounts +WHERE account_id = ?1 + AND valid_until = ?2; diff --git a/crates/store/src/db/queries/select_latest_account_storage/select_account_storage_map_entries.sql b/crates/store/src/db/queries/select_latest_account_storage/select_account_storage_map_entries.sql new file mode 100644 index 0000000000..5482d51cd9 --- /dev/null +++ b/crates/store/src/db/queries/select_latest_account_storage/select_account_storage_map_entries.sql @@ -0,0 +1,5 @@ +-- Returns every current storage map entry of the given account. +SELECT slot_name, key, value +FROM account_storage_map_values +WHERE account_id = ?1 + AND valid_until = ?2; diff --git a/crates/store/src/db/queries/select_network_accounts_subset/mod.rs b/crates/store/src/db/queries/select_network_accounts_subset/mod.rs new file mode 100644 index 0000000000..561e72e263 --- /dev/null +++ b/crates/store/src/db/queries/select_network_accounts_subset/mod.rs @@ -0,0 +1,33 @@ +//! Filters a set of accounts down to the network accounts among them. + +use std::collections::HashSet; + +use miden_node_db::sqlite::{InList, ReadTx}; +use miden_node_utils::limiter::{QueryParamAccountIdLimit, QueryParamLimiter}; +use miden_protocol::account::AccountId; +use miden_protocol::utils::serde::Serializable; + +use crate::db::queries::{NetworkAccountType, VALID_FOREVER}; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_network_accounts_subset.sql"); + +/// Returns the subset of `account_ids` whose latest committed state is a network account. +/// +/// Unknown ids and non-network accounts are silently omitted. +pub(crate) fn select_network_accounts_subset( + tx: &ReadTx<'_>, + account_ids: &[AccountId], +) -> Result, DatabaseError> { + QueryParamAccountIdLimit::check(account_ids.len())?; + + let id_bytes = Vec::from_iter(account_ids.iter().map(Serializable::to_bytes)); + let ids = InList::from_blobs(id_bytes.iter().map(Vec::as_slice)); + + Ok(tx + .query(SQL, &[&ids, &NetworkAccountType::Network, &VALID_FOREVER], |row| { + row.get::(0) + })? + .into_iter() + .collect()) +} diff --git a/crates/store/src/db/queries/select_network_accounts_subset/select_network_accounts_subset.sql b/crates/store/src/db/queries/select_network_accounts_subset/select_network_accounts_subset.sql new file mode 100644 index 0000000000..60da11adaa --- /dev/null +++ b/crates/store/src/db/queries/select_network_accounts_subset/select_network_accounts_subset.sql @@ -0,0 +1,9 @@ +-- Returns which of the given accounts are network accounts in their latest committed state. +-- +-- Account ids are bound as a single array parameter so the statement text stays constant regardless +-- of how many are requested; see `miden_node_db::sqlite::InList`. +SELECT account_id +FROM accounts +WHERE account_id IN (SELECT value FROM rarray(?1)) + AND network_account_type = ?2 + AND valid_until = ?3; diff --git a/crates/store/src/db/queries/select_note_ids_by_nullifier/mod.rs b/crates/store/src/db/queries/select_note_ids_by_nullifier/mod.rs new file mode 100644 index 0000000000..fa1eeca63f --- /dev/null +++ b/crates/store/src/db/queries/select_note_ids_by_nullifier/mod.rs @@ -0,0 +1,37 @@ +//! Maps nullifiers to the ids of the notes they consume. + +use std::collections::BTreeMap; + +use miden_node_db::sqlite::{InList, ReadTx}; +use miden_protocol::Word; +use miden_protocol::note::{NoteId, Nullifier}; +use miden_protocol::utils::serde::Serializable; + +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_note_ids_by_nullifier.sql"); + +/// Maps each given nullifier to its note ID. +/// +/// Only public notes have a nullifier stored (`notes.nullifier` is NULL for private notes), so +/// private notes never match and are absent from the result. +pub(crate) fn select_note_ids_by_nullifier( + tx: &ReadTx<'_>, + nullifiers: &[Nullifier], +) -> Result, DatabaseError> { + if nullifiers.is_empty() { + return Ok(BTreeMap::new()); + } + + let nullifier_bytes = Vec::from_iter(nullifiers.iter().map(Serializable::to_bytes)); + let nullifier_bytes = InList::from_blobs(nullifier_bytes.iter().map(Vec::as_slice)); + + let pairs = tx.query(SQL, &[&nullifier_bytes], |row| { + Ok((row.get::>(0)?, NoteId::from_raw(row.get::(1)?))) + })?; + + Ok(pairs + .into_iter() + .filter_map(|(nullifier, note_id)| nullifier.map(|nullifier| (nullifier, note_id))) + .collect()) +} diff --git a/crates/store/src/db/queries/select_note_ids_by_nullifier/select_note_ids_by_nullifier.sql b/crates/store/src/db/queries/select_note_ids_by_nullifier/select_note_ids_by_nullifier.sql new file mode 100644 index 0000000000..60ea99607e --- /dev/null +++ b/crates/store/src/db/queries/select_note_ids_by_nullifier/select_note_ids_by_nullifier.sql @@ -0,0 +1,6 @@ +-- Maps the given nullifiers to their note ids. +-- +-- Only public notes store a nullifier, so private notes never match. +SELECT nullifier, note_id +FROM notes +WHERE nullifier IN (SELECT value FROM rarray(?1)); diff --git a/crates/store/src/db/queries/select_note_inclusion_proofs/mod.rs b/crates/store/src/db/queries/select_note_inclusion_proofs/mod.rs new file mode 100644 index 0000000000..aeb333482f --- /dev/null +++ b/crates/store/src/db/queries/select_note_inclusion_proofs/mod.rs @@ -0,0 +1,58 @@ +//! Returns inclusion proofs for a set of notes. + +use std::collections::{BTreeMap, BTreeSet}; + +use miden_node_db::sqlite::{InList, ReadTx}; +use miden_node_utils::limiter::{QueryParamLimiter, QueryParamNoteCommitmentLimit}; +use miden_protocol::Word; +use miden_protocol::block::{BlockNoteIndex, BlockNumber}; +use miden_protocol::crypto::merkle::SparseMerklePath; +use miden_protocol::note::{NoteId, NoteInclusionProof}; +use miden_protocol::utils::serde::Serializable; + +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_note_inclusion_proofs.sql"); + +/// Select note inclusion proofs matching the note commitments, restricted to notes committed at +/// or before `up_to_block`. +/// +/// # Parameters +/// * `note_commitments`: Set of note commitments to query +/// - Limit: 0 <= count <= 1000 +/// * `up_to_block`: Only notes committed at or before this block are returned +/// +/// # Returns +/// +/// - Empty map if no matching `note`. +/// - Otherwise, note inclusion proofs keyed by [`NoteId`]. +pub(crate) fn select_note_inclusion_proofs( + tx: &ReadTx<'_>, + note_commitments: &BTreeSet, + up_to_block: BlockNumber, +) -> Result, DatabaseError> { + QueryParamNoteCommitmentLimit::check(note_commitments.len())?; + + let commitments = Vec::from_iter(note_commitments.iter().map(Serializable::to_bytes)); + let commitments = InList::from_blobs(commitments.iter().map(Vec::as_slice)); + + let rows = tx.query(SQL, &[&commitments, &up_to_block], |row| { + Ok(( + row.get::(0)?, + NoteId::from_raw(row.get::(1)?), + row.get::(2)? as usize, + row.get::(3)? as usize, + row.get::(4)?, + )) + })?; + + rows.into_iter() + .map(|(block_num, note_id, batch_index, note_index, merkle_path)| { + let node_index_in_block = BlockNoteIndex::new(batch_index, note_index) + .expect("batch and note index from DB should be valid") + .leaf_index_value(); + let proof = NoteInclusionProof::new(block_num, node_index_in_block, merkle_path)?; + Ok((note_id, proof)) + }) + .collect() +} diff --git a/crates/store/src/db/queries/select_note_inclusion_proofs/select_note_inclusion_proofs.sql b/crates/store/src/db/queries/select_note_inclusion_proofs/select_note_inclusion_proofs.sql new file mode 100644 index 0000000000..97e4275dae --- /dev/null +++ b/crates/store/src/db/queries/select_note_inclusion_proofs/select_note_inclusion_proofs.sql @@ -0,0 +1,6 @@ +-- Returns the inclusion proof data for the given notes, restricted to those already committed. +SELECT committed_at, note_id, batch_index, note_index, inclusion_path +FROM notes +WHERE note_id IN (SELECT value FROM rarray(?1)) + AND committed_at <= ?2 +ORDER BY committed_at ASC; diff --git a/crates/store/src/db/queries/select_note_script_by_root/mod.rs b/crates/store/src/db/queries/select_note_script_by_root/mod.rs new file mode 100644 index 0000000000..9f74d0de7b --- /dev/null +++ b/crates/store/src/db/queries/select_note_script_by_root/mod.rs @@ -0,0 +1,18 @@ +//! Returns a note script by its root. + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::Word; +use miden_protocol::note::NoteScript; + +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_note_script_by_root.sql"); + +/// Returns the script for a note by its root. +pub(crate) fn select_note_script_by_root( + tx: &ReadTx<'_>, + root: Word, +) -> Result, DatabaseError> { + // Invariant: `script_root` is the primary key, so there is at most one row. + Ok(tx.query(SQL, &[&root], |row| row.get::(0))?.into_iter().next()) +} diff --git a/crates/store/src/db/queries/select_note_script_by_root/select_note_script_by_root.sql b/crates/store/src/db/queries/select_note_script_by_root/select_note_script_by_root.sql new file mode 100644 index 0000000000..6661a32a08 --- /dev/null +++ b/crates/store/src/db/queries/select_note_script_by_root/select_note_script_by_root.sql @@ -0,0 +1,4 @@ +-- Returns the note script stored under the given root. +SELECT script +FROM note_scripts +WHERE script_root = ?1; diff --git a/crates/store/src/db/queries/select_note_sync_records/mod.rs b/crates/store/src/db/queries/select_note_sync_records/mod.rs new file mode 100644 index 0000000000..802f9c1faa --- /dev/null +++ b/crates/store/src/db/queries/select_note_sync_records/mod.rs @@ -0,0 +1,41 @@ +//! Returns note sync records for a set of note ids. + +use std::collections::BTreeMap; + +use miden_node_db::sqlite::{InList, ReadTx}; +use miden_node_utils::limiter::{QueryParamLimiter, QueryParamNoteCommitmentLimit}; +use miden_protocol::note::NoteId; +use miden_protocol::utils::serde::Serializable; + +use crate::db::NoteSyncRecord; +use crate::db::queries::note_row::note_sync_record_from_row; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_note_sync_records.sql"); + +/// Select note sync records matching the given note ids. +/// +/// # Parameters +/// * `note_ids`: Slice of note ids to query +/// - Limit: 0 <= count <= 1000 +/// +/// # Returns +/// +/// - Empty map if no matching `note`. +/// - Otherwise, note sync records keyed by [`NoteId`]. +pub(crate) fn select_note_sync_records( + tx: &ReadTx<'_>, + note_ids: &[NoteId], +) -> Result, DatabaseError> { + QueryParamNoteCommitmentLimit::check(note_ids.len())?; + + // The stored `note_id` column holds the note's word, not the serialized `NoteId`. + let note_ids = Vec::from_iter(note_ids.iter().map(|id| id.as_word().to_bytes())); + let note_ids = InList::from_blobs(note_ids.iter().map(Vec::as_slice)); + + Ok(tx + .query(SQL, &[¬e_ids], note_sync_record_from_row)? + .into_iter() + .map(|note| (note.note_id, note)) + .collect()) +} diff --git a/crates/store/src/db/queries/select_note_sync_records/select_note_sync_records.sql b/crates/store/src/db/queries/select_note_sync_records/select_note_sync_records.sql new file mode 100644 index 0000000000..432c190582 --- /dev/null +++ b/crates/store/src/db/queries/select_note_sync_records/select_note_sync_records.sql @@ -0,0 +1,6 @@ +-- Returns the sync records for the given notes, oldest block first. +SELECT committed_at, batch_index, note_index, note_id, note_type, sender, tag, attachment, + inclusion_path +FROM notes +WHERE note_id IN (SELECT value FROM rarray(?1)) +ORDER BY committed_at ASC; diff --git a/crates/store/src/db/queries/select_notes_by_id/mod.rs b/crates/store/src/db/queries/select_notes_by_id/mod.rs new file mode 100644 index 0000000000..dd6b66896b --- /dev/null +++ b/crates/store/src/db/queries/select_notes_by_id/mod.rs @@ -0,0 +1,22 @@ +//! Returns full note records, including details and script, for a set of note ids. + +use miden_node_db::sqlite::{InList, ReadTx}; +use miden_protocol::note::NoteId; +use miden_protocol::utils::serde::Serializable; + +use crate::db::NoteRecord; +use crate::db::queries::note_row::note_record_from_row; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_notes_by_id.sql"); + +/// Select all notes matching the given set of identifiers. +pub(crate) fn select_notes_by_id( + tx: &ReadTx<'_>, + note_ids: &[NoteId], +) -> Result, DatabaseError> { + let note_ids = Vec::from_iter(note_ids.iter().map(Serializable::to_bytes)); + let note_ids = InList::from_blobs(note_ids.iter().map(Vec::as_slice)); + + Ok(tx.query(SQL, &[¬e_ids], note_record_from_row)?) +} diff --git a/crates/store/src/db/queries/select_notes_by_id/select_notes_by_id.sql b/crates/store/src/db/queries/select_notes_by_id/select_notes_by_id.sql new file mode 100644 index 0000000000..a535a5289b --- /dev/null +++ b/crates/store/src/db/queries/select_notes_by_id/select_notes_by_id.sql @@ -0,0 +1,10 @@ +-- Returns the notes with the given ids, including their details and script where stored. +-- +-- The script lives in `note_scripts`, keyed by the note's script root; the join is outer because +-- private notes store no script. +SELECT notes.committed_at, notes.batch_index, notes.note_index, notes.note_id, notes.note_type, + notes.sender, notes.tag, notes.attachment, notes.assets, notes.storage, notes.serial_num, + notes.inclusion_path, note_scripts.script +FROM notes +LEFT JOIN note_scripts ON notes.script_root = note_scripts.script_root +WHERE notes.note_id IN (SELECT value FROM rarray(?1)); diff --git a/crates/store/src/db/queries/select_notes_since_block_by_tag/mod.rs b/crates/store/src/db/queries/select_notes_since_block_by_tag/mod.rs new file mode 100644 index 0000000000..376b6f97ab --- /dev/null +++ b/crates/store/src/db/queries/select_notes_since_block_by_tag/mod.rs @@ -0,0 +1,48 @@ +//! Returns the notes matching a set of tags, from the first block in range that has any. + +use std::ops::RangeInclusive; + +use miden_node_db::sqlite::ReadTx; +use miden_node_utils::limiter::{QueryParamLimiter, QueryParamNoteTagLimit}; +use miden_protocol::block::BlockNumber; + +use crate::db::NoteSyncRecord; +use crate::db::queries::note_row::note_sync_record_from_row; +use crate::db::queries::note_tag_in_list; +use crate::errors::DatabaseError; + +const SQL_FIRST_BLOCK: &str = include_str!("select_first_block_with_tags.sql"); +const SQL_NOTES_IN_BLOCK: &str = include_str!("select_notes_in_block_by_tag.sql"); + +/// Select notes matching the given tags within a block range. +/// +/// # Parameters +/// * `note_tags`: List of note tags to filter by +/// - Limit: 0 <= count <= 1000 +/// * `block_range`: Range of blocks to search (inclusive) +/// +/// # Returns +/// +/// All matching notes from the first block within the range containing a matching note. If no +/// matching notes are found at all, then an empty vector is returned. +pub(crate) fn select_notes_since_block_by_tag( + tx: &ReadTx<'_>, + note_tags: &[u32], + block_range: RangeInclusive, +) -> Result, DatabaseError> { + QueryParamNoteTagLimit::check(note_tags.len())?; + + let tags = note_tag_in_list(note_tags); + let first_block = tx + .query(SQL_FIRST_BLOCK, &[&tags, block_range.start(), block_range.end()], |row| { + row.get::(0) + })? + .into_iter() + .next(); + + let Some(first_block) = first_block else { + return Ok(Vec::new()); + }; + + Ok(tx.query(SQL_NOTES_IN_BLOCK, &[&tags, &first_block], note_sync_record_from_row)?) +} diff --git a/crates/store/src/db/queries/select_notes_since_block_by_tag/select_first_block_with_tags.sql b/crates/store/src/db/queries/select_notes_since_block_by_tag/select_first_block_with_tags.sql new file mode 100644 index 0000000000..8c7cec80cc --- /dev/null +++ b/crates/store/src/db/queries/select_notes_since_block_by_tag/select_first_block_with_tags.sql @@ -0,0 +1,11 @@ +-- Returns the earliest block within the range that contains a note matching one of the tags. +-- +-- Tags are bound as a single array parameter so the statement text stays constant regardless of how +-- many are requested; see `miden_node_db::sqlite::InList`. +SELECT committed_at +FROM notes +WHERE tag IN (SELECT value FROM rarray(?1)) + AND committed_at >= ?2 + AND committed_at <= ?3 +ORDER BY committed_at ASC +LIMIT 1; diff --git a/crates/store/src/db/queries/select_notes_since_block_by_tag/select_notes_in_block_by_tag.sql b/crates/store/src/db/queries/select_notes_since_block_by_tag/select_notes_in_block_by_tag.sql new file mode 100644 index 0000000000..aa12bd3d0d --- /dev/null +++ b/crates/store/src/db/queries/select_notes_since_block_by_tag/select_notes_in_block_by_tag.sql @@ -0,0 +1,7 @@ +-- Returns every note in the given block matching one of the tags, in block note order. +SELECT committed_at, batch_index, note_index, note_id, note_type, sender, tag, attachment, + inclusion_path +FROM notes +WHERE committed_at = ?2 + AND tag IN (SELECT value FROM rarray(?1)) +ORDER BY committed_at ASC, batch_index ASC, note_index ASC; diff --git a/crates/store/src/db/queries/select_nullifiers_by_prefix/mod.rs b/crates/store/src/db/queries/select_nullifiers_by_prefix/mod.rs new file mode 100644 index 0000000000..601505dc10 --- /dev/null +++ b/crates/store/src/db/queries/select_nullifiers_by_prefix/mod.rs @@ -0,0 +1,107 @@ +//! Returns the nullifiers matching a set of prefixes within a block range. + +use std::ops::RangeInclusive; + +use miden_node_db::SqlTypeConvert; +use miden_node_db::sqlite::{InList, ReadTx}; +use miden_node_utils::limiter::{ + MAX_RESPONSE_PAYLOAD_BYTES, + QueryParamLimiter, + QueryParamNullifierPrefixLimit, +}; +use miden_protocol::block::BlockNumber; +use miden_protocol::note::Nullifier; + +use crate::db::NullifierInfo; +use crate::errors::DatabaseError; + +const SQL: &str = include_str!("select_nullifiers_by_prefix.sql"); + +/// Returns nullifiers filtered by prefix within a block number range. +/// +/// # Parameters +/// * `prefix_len`: Length of nullifier prefix in bits +/// - Must be exactly 16 bits +/// * `nullifier_prefixes`: List of nullifier prefixes to filter by +/// - Limit: 0 <= count <= 1000 +/// +/// Each value of the `nullifier_prefixes` is only the `prefix_len` most significant bits +/// of the nullifier of interest to the client. This hides the details of the specific +/// nullifier being requested. Currently the only supported prefix length is 16 bits. +/// +/// # Returns +/// +/// The matching nullifiers with the block at which they were created, and the last block the +/// response covers. When the rows would exceed the payload limit, the trailing block is dropped +/// whole and the returned block number reports how far the response actually reaches. +pub(crate) fn select_nullifiers_by_prefix( + tx: &ReadTx<'_>, + prefix_len: u8, + nullifier_prefixes: &[u16], + block_range: RangeInclusive, +) -> Result<(Vec, BlockNumber), DatabaseError> { + // Size calculation: max 2^16 nullifiers per block × 36 bytes per nullifier = ~2.25MB + pub const NULLIFIER_BYTES: usize = 32; // digest size (nullifier) + pub const BLOCK_NUM_BYTES: usize = 4; // 32 bits per block number + pub const ROW_OVERHEAD_BYTES: usize = NULLIFIER_BYTES + BLOCK_NUM_BYTES; // 36 bytes + pub const MAX_ROWS: usize = MAX_RESPONSE_PAYLOAD_BYTES / ROW_OVERHEAD_BYTES; + // Pagination reports the last fully-included block, so it only makes progress if every block + // fits within a single page. A block that exceeded `MAX_ROWS` nullifiers would produce an empty + // page and stall clients forever on that block. + const _: () = assert!( + miden_protocol::MAX_INPUT_NOTES_PER_BLOCK <= MAX_ROWS, + "a block's nullifiers must fit in one response page or pagination cannot make progress", + ); + + assert_eq!(prefix_len, 16, "Only 16-bit prefixes are supported"); + + if block_range.is_empty() { + return Err(DatabaseError::InvalidBlockRange { + from: *block_range.start(), + to: *block_range.end(), + }); + } + + QueryParamNullifierPrefixLimit::check(nullifier_prefixes.len())?; + + let prefixes = InList::from_i64s(nullifier_prefixes.iter().copied().map(i64::from)); + // Request an additional row so we can determine whether this is the last page. + let limit = i64::try_from(MAX_ROWS + 1).expect("limit fits within i64"); + + // The block number is read raw because the trimming below steps one block back, which is not + // representable as a `BlockNumber` when the trailing block is the genesis block. + let raw = + tx.query(SQL, &[&prefixes, block_range.start(), block_range.end(), &limit], |row| { + Ok((row.get::(0)?, row.get::(1)?)) + })?; + + let last_block_num = raw.last().map(|(_, block_num)| *block_num); + + // Discard the last block in the response (assumes more than one block may be present) + if let Some(last_block_num) = last_block_num + && raw.len() > MAX_ROWS + { + let nullifiers = collect_nullifier_infos( + raw.into_iter().take_while(|(_, block_num)| *block_num != last_block_num), + )?; + let last_block_included = BlockNumber::from_raw_sql(last_block_num.saturating_sub(1))?; + + Ok((nullifiers, last_block_included)) + } else { + Ok((collect_nullifier_infos(raw)?, *block_range.end())) + } +} + +/// Converts raw `(nullifier, block_num)` rows into [`NullifierInfo`]s. +fn collect_nullifier_infos( + rows: impl IntoIterator, +) -> Result, DatabaseError> { + rows.into_iter() + .map(|(nullifier, block_num)| { + Ok(NullifierInfo { + nullifier, + block_num: BlockNumber::from_raw_sql(block_num)?, + }) + }) + .collect() +} diff --git a/crates/store/src/db/queries/select_nullifiers_by_prefix/select_nullifiers_by_prefix.sql b/crates/store/src/db/queries/select_nullifiers_by_prefix/select_nullifiers_by_prefix.sql new file mode 100644 index 0000000000..094f0576cb --- /dev/null +++ b/crates/store/src/db/queries/select_nullifiers_by_prefix/select_nullifiers_by_prefix.sql @@ -0,0 +1,12 @@ +-- Returns the nullifiers whose prefix is in the requested set and that were created within the +-- given block range, oldest first. +-- +-- Prefixes are bound as a single array parameter so the statement text stays constant regardless of +-- how many are requested; see `miden_node_db::sqlite::InList`. +SELECT nullifier, block_num +FROM nullifiers +WHERE nullifier_prefix IN (SELECT value FROM rarray(?1)) + AND block_num >= ?2 + AND block_num <= ?3 +ORDER BY block_num ASC +LIMIT ?4; diff --git a/crates/store/src/db/queries/select_nullifiers_paged/mod.rs b/crates/store/src/db/queries/select_nullifiers_paged/mod.rs new file mode 100644 index 0000000000..40f2db755f --- /dev/null +++ b/crates/store/src/db/queries/select_nullifiers_paged/mod.rs @@ -0,0 +1,56 @@ +//! Returns a page of nullifiers, for rebuilding the nullifier tree at startup. + +use std::num::NonZeroUsize; + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::block::BlockNumber; +use miden_protocol::note::Nullifier; + +use crate::db::NullifierInfo; +use crate::errors::DatabaseError; + +const SQL_FIRST_PAGE: &str = include_str!("select_nullifiers_page.sql"); +const SQL_AFTER_CURSOR: &str = include_str!("select_nullifiers_page_after.sql"); + +/// Page of nullifiers returned by [`select_nullifiers_paged`]. +#[derive(Debug)] +pub struct NullifiersPage { + /// The nullifiers in this page. + pub nullifiers: Vec, + /// If `Some`, there are more results. Use this as the `after_nullifier` for the next page. + pub next_cursor: Option, +} + +/// Selects nullifiers with pagination. +/// +/// Returns up to `page_size` nullifiers, starting after `after_nullifier` if provided. +/// Results are ordered by nullifier bytes for stable pagination. +pub(crate) fn select_nullifiers_paged( + tx: &ReadTx<'_>, + page_size: NonZeroUsize, + after_nullifier: Option, +) -> Result { + // Fetch one extra to determine if there are more results + let limit = i64::try_from(page_size.get() + 1).expect("page size fits within i64"); + + let map = |row: &miden_node_db::sqlite::Row<'_>| { + Ok(NullifierInfo { + nullifier: row.get::(0)?, + block_num: row.get::(1)?, + }) + }; + let mut nullifiers = match after_nullifier { + Some(cursor) => tx.query(SQL_AFTER_CURSOR, &[&limit, &cursor], map)?, + None => tx.query(SQL_FIRST_PAGE, &[&limit], map)?, + }; + + // If we got more than page_size, there are more results + let next_cursor = if nullifiers.len() > page_size.get() { + nullifiers.pop(); // Remove the extra element + nullifiers.last().map(|info| info.nullifier) + } else { + None + }; + + Ok(NullifiersPage { nullifiers, next_cursor }) +} diff --git a/crates/store/src/db/queries/select_nullifiers_paged/select_nullifiers_page.sql b/crates/store/src/db/queries/select_nullifiers_paged/select_nullifiers_page.sql new file mode 100644 index 0000000000..f56a96b5f9 --- /dev/null +++ b/crates/store/src/db/queries/select_nullifiers_paged/select_nullifiers_page.sql @@ -0,0 +1,5 @@ +-- Returns the first page of nullifiers, ordered by nullifier for stable pagination. +SELECT nullifier, block_num +FROM nullifiers +ORDER BY nullifier ASC +LIMIT ?1; diff --git a/crates/store/src/db/queries/select_nullifiers_paged/select_nullifiers_page_after.sql b/crates/store/src/db/queries/select_nullifiers_paged/select_nullifiers_page_after.sql new file mode 100644 index 0000000000..7555674147 --- /dev/null +++ b/crates/store/src/db/queries/select_nullifiers_paged/select_nullifiers_page_after.sql @@ -0,0 +1,9 @@ +-- Returns the page of nullifiers following the cursor, ordered by nullifier for stable pagination. +-- +-- The cursor is a bare `>` comparison rather than a nullable parameter so the range scan can use the +-- primary key index; pagination over the whole table would otherwise be quadratic. +SELECT nullifier, block_num +FROM nullifiers +WHERE nullifier > ?2 +ORDER BY nullifier ASC +LIMIT ?1; diff --git a/crates/store/src/db/queries/select_public_account_ids_paged/mod.rs b/crates/store/src/db/queries/select_public_account_ids_paged/mod.rs new file mode 100644 index 0000000000..9154a0286d --- /dev/null +++ b/crates/store/src/db/queries/select_public_account_ids_paged/mod.rs @@ -0,0 +1,58 @@ +//! Returns a page of public account ids. + +use std::num::NonZeroUsize; + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::account::AccountId; +use miden_protocol::utils::serde::Serializable; + +use crate::db::queries::VALID_FOREVER; +use crate::errors::DatabaseError; + +const SQL_FIRST_PAGE: &str = include_str!("select_public_account_ids_page.sql"); +const SQL_AFTER_CURSOR: &str = include_str!("select_public_account_ids_page_after.sql"); + +/// Page of public account IDs returned by [`select_public_account_ids_paged`]. +#[derive(Debug)] +pub struct PublicAccountIdsPage { + /// The public account IDs in this page. + pub account_ids: Vec, + /// If `Some`, there are more results. Use this as the `after_account_id` for the next page. + pub next_cursor: Option, +} + +/// Selects public account IDs with pagination. +/// +/// Returns up to `page_size` public account IDs, starting after `after_account_id` if provided. +/// Results are ordered by `account_id` for stable pagination. +/// +/// Public accounts are those with `AccountType::Public`. We identify them by checking +/// against the store. Public accounts store their `code_commitment`, while private accounts only +/// store the `account_commitment`. +pub(crate) fn select_public_account_ids_paged( + tx: &ReadTx<'_>, + page_size: NonZeroUsize, + after_account_id: Option, +) -> Result { + // Fetch one extra to determine if there are more results + let limit = i64::try_from(page_size.get() + 1).expect("page size fits within i64"); + + let map = |row: &miden_node_db::sqlite::Row<'_>| row.get::(0); + let mut account_ids = match after_account_id { + Some(cursor) => { + let cursor = cursor.to_bytes(); + tx.query(SQL_AFTER_CURSOR, &[&limit, &VALID_FOREVER, &cursor], map)? + }, + None => tx.query(SQL_FIRST_PAGE, &[&limit, &VALID_FOREVER], map)?, + }; + + // If we got more than page_size, there are more results + let next_cursor = if account_ids.len() > page_size.get() { + account_ids.pop(); // Remove the extra element + account_ids.last().copied() + } else { + None + }; + + Ok(PublicAccountIdsPage { account_ids, next_cursor }) +} diff --git a/crates/store/src/db/queries/select_public_account_ids_paged/select_public_account_ids_page.sql b/crates/store/src/db/queries/select_public_account_ids_paged/select_public_account_ids_page.sql new file mode 100644 index 0000000000..88eb41aac1 --- /dev/null +++ b/crates/store/src/db/queries/select_public_account_ids_paged/select_public_account_ids_page.sql @@ -0,0 +1,10 @@ +-- Returns the first page of public account ids, ordered by account id. +-- +-- Public accounts are those that store a code commitment; private accounts store only their +-- account commitment. +SELECT account_id +FROM accounts +WHERE valid_until = ?2 + AND code_commitment IS NOT NULL +ORDER BY account_id ASC +LIMIT ?1; diff --git a/crates/store/src/db/queries/select_public_account_ids_paged/select_public_account_ids_page_after.sql b/crates/store/src/db/queries/select_public_account_ids_paged/select_public_account_ids_page_after.sql new file mode 100644 index 0000000000..051cc23221 --- /dev/null +++ b/crates/store/src/db/queries/select_public_account_ids_paged/select_public_account_ids_page_after.sql @@ -0,0 +1,8 @@ +-- Returns the page of public account ids following the cursor. +SELECT account_id +FROM accounts +WHERE valid_until = ?2 + AND code_commitment IS NOT NULL + AND account_id > ?3 +ORDER BY account_id ASC +LIMIT ?1; diff --git a/crates/store/src/db/queries/select_public_account_state_roots_paged/mod.rs b/crates/store/src/db/queries/select_public_account_state_roots_paged/mod.rs new file mode 100644 index 0000000000..e58ffbc819 --- /dev/null +++ b/crates/store/src/db/queries/select_public_account_state_roots_paged/mod.rs @@ -0,0 +1,96 @@ +//! Returns a page of public account state roots, for rebuilding the account state forest. + +use std::num::NonZeroUsize; + +use miden_node_db::sqlite::ReadTx; +use miden_protocol::Word; +use miden_protocol::account::{AccountId, AccountStorageHeader}; +use miden_protocol::utils::serde::Serializable; + +use crate::db::queries::VALID_FOREVER; +use crate::errors::DatabaseError; + +const SQL_FIRST_PAGE: &str = include_str!("select_public_account_state_roots_page.sql"); +const SQL_AFTER_CURSOR: &str = include_str!("select_public_account_state_roots_page_after.sql"); + +/// Latest account state forest roots for a public account. +#[derive(Debug)] +pub struct PublicAccountStateRoots { + pub account_id: AccountId, + pub vault_root: Word, + pub storage_header: AccountStorageHeader, +} + +/// Page of public account state roots returned by [`select_public_account_state_roots_paged`]. +#[derive(Debug)] +pub struct PublicAccountStateRootsPage { + /// The public account state roots in this page. + pub accounts: Vec, + /// If `Some`, there are more results. Use this as the `after_account_id` for the next page. + pub next_cursor: Option, +} + +/// A public account's state roots as stored, before the nullable columns are checked. +type StateRootsRow = (AccountId, Option, Option); + +/// Selects public account vault roots and storage headers with pagination. +/// +/// Returns up to `page_size` public account states, starting after `after_account_id` if provided. +/// Results are ordered by `account_id` for stable pagination. +/// +/// Public accounts are those with `AccountType::Public`. We identify them by checking +/// against the store. Public accounts store their `code_commitment`, while private accounts only +/// store the `account_commitment`. +pub(crate) fn select_public_account_state_roots_paged( + tx: &ReadTx<'_>, + page_size: NonZeroUsize, + after_account_id: Option, +) -> Result { + // Fetch one extra to determine if there are more results + let limit = i64::try_from(page_size.get() + 1).expect("page size fits within i64"); + + let map = |row: &miden_node_db::sqlite::Row<'_>| -> Result { + Ok(( + row.get::(0)?, + row.get::>(1)?, + row.get::>(2)?, + )) + }; + let raw = match after_account_id { + Some(cursor) => { + let cursor = cursor.to_bytes(); + tx.query(SQL_AFTER_CURSOR, &[&limit, &VALID_FOREVER, &cursor], map)? + }, + None => tx.query(SQL_FIRST_PAGE, &[&limit, &VALID_FOREVER], map)?, + }; + + // The columns are nullable in the schema, but a public account always has both. + let mut accounts = raw + .into_iter() + .map(|(account_id, vault_root, storage_header)| { + Ok(PublicAccountStateRoots { + account_id, + vault_root: vault_root.ok_or_else(|| { + DatabaseError::DataCorrupted(format!( + "public account {account_id} is missing a vault root" + )) + })?, + storage_header: storage_header.ok_or_else(|| { + DatabaseError::DataCorrupted(format!( + "public account {account_id} is missing a storage header" + )) + })?, + }) + }) + .collect::, DatabaseError>>()?; + + // If we got more than page_size, there are more results. + let next_cursor = if accounts.len() > page_size.get() { + accounts.pop(); + accounts.last().map(|account| account.account_id) + } else { + None + }; + + Ok(PublicAccountStateRootsPage { accounts, next_cursor }) +} diff --git a/crates/store/src/db/queries/select_public_account_state_roots_paged/select_public_account_state_roots_page.sql b/crates/store/src/db/queries/select_public_account_state_roots_paged/select_public_account_state_roots_page.sql new file mode 100644 index 0000000000..861e16fa34 --- /dev/null +++ b/crates/store/src/db/queries/select_public_account_state_roots_paged/select_public_account_state_roots_page.sql @@ -0,0 +1,7 @@ +-- Returns the first page of public account vault roots and storage headers, ordered by account id. +SELECT account_id, vault_root, storage_header +FROM accounts +WHERE valid_until = ?2 + AND code_commitment IS NOT NULL +ORDER BY account_id ASC +LIMIT ?1; diff --git a/crates/store/src/db/queries/select_public_account_state_roots_paged/select_public_account_state_roots_page_after.sql b/crates/store/src/db/queries/select_public_account_state_roots_paged/select_public_account_state_roots_page_after.sql new file mode 100644 index 0000000000..ea2e6b54b6 --- /dev/null +++ b/crates/store/src/db/queries/select_public_account_state_roots_paged/select_public_account_state_roots_page_after.sql @@ -0,0 +1,8 @@ +-- Returns the page of public account vault roots and storage headers following the cursor. +SELECT account_id, vault_root, storage_header +FROM accounts +WHERE valid_until = ?2 + AND code_commitment IS NOT NULL + AND account_id > ?3 +ORDER BY account_id ASC +LIMIT ?1; diff --git a/crates/store/src/db/queries/select_transactions_records/mod.rs b/crates/store/src/db/queries/select_transactions_records/mod.rs new file mode 100644 index 0000000000..6805549466 --- /dev/null +++ b/crates/store/src/db/queries/select_transactions_records/mod.rs @@ -0,0 +1,260 @@ +//! Returns full transaction records for a set of accounts within a block range. + +use std::collections::BTreeMap; +use std::ops::RangeInclusive; + +use miden_node_db::SqlTypeConvert; +use miden_node_db::sqlite::{InList, ReadTx}; +use miden_node_utils::limiter::{ + MAX_RESPONSE_PAYLOAD_BYTES, + QueryParamAccountIdLimit, + QueryParamLimiter, + QueryParamNoteCommitmentLimit, +}; +use miden_protocol::Word; +use miden_protocol::account::AccountId; +use miden_protocol::block::BlockNumber; +use miden_protocol::note::{NoteHeader, NoteId, Nullifier}; +use miden_protocol::transaction::{ + InputNoteCommitment, + InputNotes, + TransactionHeader, + TransactionId, +}; +use miden_protocol::utils::serde::{Deserializable, Serializable}; + +use crate::db::TransactionRecord; +use crate::db::queries::{select_note_ids_by_nullifier, select_note_sync_records}; +use crate::errors::DatabaseError; + +const SQL_FIRST_CHUNK: &str = include_str!("select_transactions_records_chunk.sql"); +const SQL_AFTER_CURSOR: &str = include_str!("select_transactions_records_chunk_after.sql"); + +/// A transaction row, before its notes are resolved. +struct TransactionRow { + account_id: AccountId, + block_num: i64, + transaction_id: TransactionId, + initial_state_commitment: Word, + final_state_commitment: Word, + input_notes: Vec, + output_notes: Vec, + size_in_bytes: i64, +} + +/// Returns the transactions of `account_ids` within `block_range`, and the last block the response +/// covers. +/// +/// Notes: +/// - Uses stable ordering (`block_num`, `transaction_id`) to ensure consistent results across +/// paginated queries. +/// - Uses cursor-based pagination. +/// - The query is executed in chunks of 1000 transactions to prevent loading excessive data and to +/// stop as soon as the accumulated size approaches the 4MB limit. +/// - Given the size of note records, 1000 records are guaranteed never to return more than about +/// 60MB of data. +pub(crate) fn select_transactions_records( + tx: &ReadTx<'_>, + account_ids: &[AccountId], + block_range: RangeInclusive, +) -> Result<(BlockNumber, Vec), DatabaseError> { + const NUM_TXS_PER_CHUNK: i64 = 1000; // Read 1000 transactions at a time + + QueryParamAccountIdLimit::check(account_ids.len())?; + + let max_payload_bytes = + i64::try_from(MAX_RESPONSE_PAYLOAD_BYTES).expect("payload limit fits within i64"); + + if block_range.is_empty() { + return Err(DatabaseError::InvalidBlockRange { + from: *block_range.start(), + to: *block_range.end(), + }); + } + + let account_id_bytes = Vec::from_iter(account_ids.iter().map(Serializable::to_bytes)); + let desired_account_ids = InList::from_blobs(account_id_bytes.iter().map(Vec::as_slice)); + + // Read transactions in chunks to prevent loading excessive data and to stop as soon as we + // approach the size limit + let mut transactions = Vec::new(); + let mut total_size = 0i64; + let mut cursor: Option<(i64, Vec)> = None; + // Track the block number of the first transaction that did not fit within the payload cap. This + // is the explicit "we truncated" signal; the accumulated byte total cannot be used as a proxy, + // since a transaction can fail to fit while `total_size` is still below the cap. + let mut truncated_at_block: Option = None; + + loop { + // Apply cursor-based pagination using the last seen (block_num, transaction_id) + let chunk = match &cursor { + Some((last_block, last_tx_id)) => tx.query( + SQL_AFTER_CURSOR, + &[ + block_range.start(), + block_range.end(), + &desired_account_ids, + &NUM_TXS_PER_CHUNK, + last_block, + last_tx_id, + ], + transaction_row_from_row, + )?, + None => tx.query( + SQL_FIRST_CHUNK, + &[block_range.start(), block_range.end(), &desired_account_ids, &NUM_TXS_PER_CHUNK], + transaction_row_from_row, + )?, + }; + + // Add transactions from this chunk one by one until we hit the limit + let mut added_from_chunk = 0; + + for row in chunk { + if total_size + row.size_in_bytes <= max_payload_bytes { + total_size += row.size_in_bytes; + cursor = Some((row.block_num, row.transaction_id.to_bytes())); + transactions.push(row); + added_from_chunk += 1; + } else { + // This transaction does not fit, so the response is truncated at its block. + truncated_at_block = Some(row.block_num); + break; + } + } + + // Break if we truncated due to the payload cap, or the chunk was incomplete (i.e. the + // matching transactions are exhausted). + if truncated_at_block.is_some() || added_from_chunk < NUM_TXS_PER_CHUNK { + break; + } + } + + let Some(truncation_block) = truncated_at_block else { + // Every matching transaction in the range fit within the payload cap. + return Ok((*block_range.end(), with_output_note_proofs(tx, transactions)?)); + }; + + // We stopped within `truncation_block`, so that block may be partial. Block-based pagination + // can only report fully-included blocks, so drop every transaction belonging to the truncation + // block and report the previous block as the cursor. Transactions are ordered ascending by + // block number, so the truncation block's transactions form a contiguous suffix: + // `partition_point` locates the boundary and `truncate` drops the suffix in place, without + // allocating a new vector, with O(log n) complexity. + let complete_len = transactions.partition_point(|row| row.block_num < truncation_block); + transactions.truncate(complete_len); + + if transactions.is_empty() { + // A single block's transactions exceed the payload cap. Reporting `truncation_block - 1` + // here would tell the client to resume from `truncation_block`, which can never fit, so + // pagination would loop forever. Surface the condition instead of silently looping. + return Err(DatabaseError::TransactionPageExceedsPayloadLimit { + block_num: BlockNumber::from_raw_sql(truncation_block)?, + }); + } + + // SAFETY: block_num came from the database and was previously validated. Subtraction is safe + // under the assumption that genesis block (where it could fail) does not have any transactions. + let last_included_block = BlockNumber::from_raw_sql(truncation_block.saturating_sub(1))?; + Ok((last_included_block, with_output_note_proofs(tx, transactions)?)) +} + +/// Maps a transaction row, leaving the note blobs to be deserialized in bulk afterwards. +fn transaction_row_from_row( + row: &miden_node_db::sqlite::Row<'_>, +) -> Result { + Ok(TransactionRow { + account_id: row.get::(0)?, + block_num: row.get::(1)?, + transaction_id: row.get::(2)?, + initial_state_commitment: row.get::(3)?, + final_state_commitment: row.get::(4)?, + input_notes: row.get::>(5)?, + output_notes: row.get::>(6)?, + size_in_bytes: row.get::(7)?, + }) +} + +/// Resolves each transaction's committed output notes and consumed note references. +fn with_output_note_proofs( + tx: &ReadTx<'_>, + raw_transactions: Vec, +) -> Result, DatabaseError> { + // Pre-deserialize output notes to collect IDs for the batch lookup. + let mut tx_output_notes = Vec::with_capacity(raw_transactions.len()); + let mut all_note_ids: Vec = Vec::new(); + for raw in &raw_transactions { + let notes: Vec = Deserializable::read_from_bytes(&raw.output_notes)?; + all_note_ids.extend(notes.iter().map(NoteHeader::id)); + tx_output_notes.push(notes); + } + + let mut output_notes_by_id = BTreeMap::new(); + for chunk in all_note_ids.chunks(QueryParamNoteCommitmentLimit::LIMIT) { + output_notes_by_id.extend(select_note_sync_records(tx, chunk)?); + } + + // Deserialize each transaction's input notes once and reuse them below. Authenticated inputs + // have no header and carry only a nullifier, so gather those nullifiers to look their note IDs + // up in one batch. + let mut tx_input_notes: Vec> = + Vec::with_capacity(raw_transactions.len()); + let mut authenticated_nullifiers: Vec = Vec::new(); + for raw in &raw_transactions { + let commitments: Vec = + Deserializable::read_from_bytes(&raw.input_notes)?; + for commitment in &commitments { + if commitment.header().is_none() { + authenticated_nullifiers.push(commitment.nullifier()); + } + } + tx_input_notes.push(commitments); + } + + let mut note_ids_by_nullifier = BTreeMap::new(); + for chunk in authenticated_nullifiers.chunks(QueryParamNoteCommitmentLimit::LIMIT) { + note_ids_by_nullifier.extend(select_note_ids_by_nullifier(tx, chunk)?); + } + + // Assemble the final records. + raw_transactions + .into_iter() + .zip(tx_output_notes) + .zip(tx_input_notes) + .map(|((raw, output_notes), input_notes)| { + // Collect inclusion proofs for committed output notes. Notes not found in the `notes` + // table were erased (created and consumed in the same batch). + let output_note_proofs = output_notes + .iter() + .filter_map(|note| output_notes_by_id.get(¬e.id()).cloned()) + .collect(); + + // Build the side-channel refs. The input note commitments are left untouched, so the + // header and its commitment stay exactly as the transaction submitted them. + let consumed_note_refs = input_notes + .iter() + .filter(|commitment| commitment.header().is_none()) + .filter_map(|commitment| { + let nullifier = commitment.nullifier(); + note_ids_by_nullifier.get(&nullifier).map(|note_id| (nullifier, *note_id)) + }) + .collect(); + + let header = TransactionHeader::new_unchecked( + raw.transaction_id, + raw.account_id, + raw.initial_state_commitment, + raw.final_state_commitment, + InputNotes::new_unchecked(input_notes), + output_notes, + ); + + Ok(TransactionRecord { + block_num: BlockNumber::from_raw_sql(raw.block_num)?, + header, + output_note_proofs, + consumed_note_refs, + }) + }) + .collect() +} diff --git a/crates/store/src/db/queries/select_transactions_records/select_transactions_records_chunk.sql b/crates/store/src/db/queries/select_transactions_records/select_transactions_records_chunk.sql new file mode 100644 index 0000000000..d2ff19ea9a --- /dev/null +++ b/crates/store/src/db/queries/select_transactions_records/select_transactions_records_chunk.sql @@ -0,0 +1,12 @@ +-- Returns a chunk of transactions for the given accounts within a block range, in a stable order. +-- +-- Account ids are bound as a single array parameter so the statement text stays constant regardless +-- of how many are requested; see `miden_node_db::sqlite::InList`. +SELECT account_id, block_num, transaction_id, initial_state_commitment, final_state_commitment, + input_notes, output_notes, size_in_bytes +FROM transactions +WHERE block_num >= ?1 + AND block_num <= ?2 + AND account_id IN (SELECT value FROM rarray(?3)) +ORDER BY block_num ASC, transaction_id ASC +LIMIT ?4; diff --git a/crates/store/src/db/queries/select_transactions_records/select_transactions_records_chunk_after.sql b/crates/store/src/db/queries/select_transactions_records/select_transactions_records_chunk_after.sql new file mode 100644 index 0000000000..20da5622ab --- /dev/null +++ b/crates/store/src/db/queries/select_transactions_records/select_transactions_records_chunk_after.sql @@ -0,0 +1,13 @@ +-- Returns the chunk of transactions following the `(block_num, transaction_id)` cursor. +-- +-- The cursor comparison is spelled out rather than using nullable parameters so the range scan can +-- use the index on `(block_num, transaction_id)`. +SELECT account_id, block_num, transaction_id, initial_state_commitment, final_state_commitment, + input_notes, output_notes, size_in_bytes +FROM transactions +WHERE block_num >= ?1 + AND block_num <= ?2 + AND account_id IN (SELECT value FROM rarray(?3)) + AND (block_num > ?5 OR (block_num = ?5 AND transaction_id > ?6)) +ORDER BY block_num ASC, transaction_id ASC +LIMIT ?4; diff --git a/crates/store/src/db/queries/upsert_accounts/close_account_validity.sql b/crates/store/src/db/queries/upsert_accounts/close_account_validity.sql new file mode 100644 index 0000000000..9026cf5159 --- /dev/null +++ b/crates/store/src/db/queries/upsert_accounts/close_account_validity.sql @@ -0,0 +1,8 @@ +-- Closes an account's current row at the block that supersedes it. +-- +-- Only the open-ended row (`valid_until` at the sentinel) can be the previous version, so matching +-- on it both selects that row and makes the update idempotent. +UPDATE accounts +SET valid_until = ?1 +WHERE account_id = ?2 + AND valid_until = ?3 diff --git a/crates/store/src/db/models/queries/accounts/delta.rs b/crates/store/src/db/queries/upsert_accounts/delta.rs similarity index 70% rename from crates/store/src/db/models/queries/accounts/delta.rs rename to crates/store/src/db/queries/upsert_accounts/delta.rs index 15e2956b98..79e3ea3427 100644 --- a/crates/store/src/db/models/queries/accounts/delta.rs +++ b/crates/store/src/db/queries/upsert_accounts/delta.rs @@ -10,8 +10,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; -use diesel::query_dsl::methods::SelectDsl; -use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl, SqliteConnection}; +use miden_node_db::sqlite::ReadTx; #[cfg(test)] use miden_protocol::EMPTY_WORD; use miden_protocol::account::{ @@ -28,60 +27,53 @@ use miden_protocol::account::{ #[cfg(test)] use miden_protocol::account::{StorageMap, StorageMapKey}; use miden_protocol::block::BlockNumber; -use miden_protocol::utils::serde::{Deserializable, Serializable}; use miden_protocol::{Felt, Word}; -use super::{NetworkAccountType, VALID_FOREVER}; -use crate::db::models::conv::{SqlTypeConvert, raw_sql_to_nonce}; -use crate::db::schema; +use crate::db::queries::{NetworkAccountType, VALID_FOREVER}; use crate::errors::DatabaseError; #[cfg(test)] mod tests; +const SQL_LATEST_ACCOUNT_STATE: &str = include_str!("select_latest_account_state.sql"); + // TYPES // ================================================================================================ /// Latest account row fields needed by account update preparation. -#[derive(diesel::prelude::Queryable)] pub(super) struct LatestAccountStateRow { - created_at_block: i64, - network_account_type: i32, - nonce: Option, - code_commitment: Option>, - storage_header: Option>, + created_at_block: BlockNumber, + network_account_type: NetworkAccountType, + nonce: Option, + code_commitment: Option, + storage_header: Option, } impl LatestAccountStateRow { - pub(super) fn created_at_block(&self) -> Result { - Ok(BlockNumber::from_raw_sql(self.created_at_block)?) + pub(super) fn created_at_block(&self) -> BlockNumber { + self.created_at_block } - pub(super) fn network_account_type(&self) -> Result { - Ok(NetworkAccountType::from_raw_sql(self.network_account_type)?) + pub(super) fn network_account_type(&self) -> NetworkAccountType { + self.network_account_type } pub(super) fn state_headers( &self, account_id: AccountId, ) -> Result { - let nonce = raw_sql_to_nonce(self.nonce.ok_or_else(|| { + let nonce = self.nonce.ok_or_else(|| { DatabaseError::DataCorrupted(format!("No nonce found for account {account_id}")) - })?); - - let code_commitment = self - .code_commitment - .as_deref() - .map(Word::read_from_bytes) - .transpose()? - .ok_or_else(|| { - DatabaseError::DataCorrupted(format!( - "No code_commitment found for account {account_id}" - )) - })?; - - let storage_header = match self.storage_header.as_deref() { - Some(bytes) => AccountStorageHeader::read_from_bytes(bytes)?, + })?; + + let code_commitment = self.code_commitment.ok_or_else(|| { + DatabaseError::DataCorrupted(format!( + "No code_commitment found for account {account_id}" + )) + })?; + + let storage_header = match self.storage_header.clone() { + Some(header) => header, None => AccountStorageHeader::new(Vec::new())?, }; @@ -135,40 +127,22 @@ pub(super) enum AccountStateForInsert { // ================================================================================================ /// Selects the latest account state needed to prepare any account update. -/// -/// The query fetches: -/// - `created_at_block` and `network_account_type` for every update -/// - `nonce` (preserved when a partial patch omits its final nonce) -/// - `code_commitment` (unchanged in partial deltas) -/// - `storage_header` (to apply storage delta) -/// -/// # Raw SQL -/// -/// ```sql -/// SELECT created_at_block, network_account_type, nonce, code_commitment, storage_header -/// FROM accounts -/// WHERE account_id = ?1 AND valid_until = {VALID_FOREVER} -/// ``` pub(super) fn select_latest_account_state( - conn: &mut SqliteConnection, + tx: &ReadTx<'_>, account_id: AccountId, ) -> Result, DatabaseError> { - let row = SelectDsl::select( - schema::accounts::table, - ( - schema::accounts::created_at_block, - schema::accounts::network_account_type, - schema::accounts::nonce, - schema::accounts::code_commitment, - schema::accounts::storage_header, - ), - ) - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::valid_until.eq(VALID_FOREVER)) - .get_result(conn) - .optional()?; - - Ok(row) + Ok(tx + .query(SQL_LATEST_ACCOUNT_STATE, &[&account_id, &VALID_FOREVER], |row| { + Ok(LatestAccountStateRow { + created_at_block: row.get::(0)?, + network_account_type: row.get::(1)?, + nonce: row.get::>(2)?, + code_commitment: row.get::>(3)?, + storage_header: row.get::>(4)?, + }) + })? + .into_iter() + .next()) } // HELPER FUNCTIONS @@ -226,31 +200,7 @@ pub(super) fn apply_storage_patch( map_updates.insert(slot_name, storage_map.root()); } - let mut slots = - Vec::from_iter(header.slots().filter(|slot| !removed.contains(slot.name())).map(|slot| { - let slot_name = slot.name(); - if let Some(new_value) = value_updates.remove(slot_name) { - StorageSlotHeader::new(slot_name.clone(), slot.slot_type(), new_value) - } else if let Some(new_root) = map_updates.remove(slot_name) { - StorageSlotHeader::new(slot_name.clone(), slot.slot_type(), new_root) - } else { - slot.clone() - } - })); - - // Any updates left over belong to slots created by the patch. - for (slot_name, value) in value_updates { - slots.push(StorageSlotHeader::new(slot_name.clone(), StorageSlotType::Value, value)); - } - for (slot_name, root) in map_updates { - slots.push(StorageSlotHeader::new(slot_name.clone(), StorageSlotType::Map, root)); - } - - slots.sort_by_key(StorageSlotHeader::id); - - AccountStorageHeader::new(slots).map_err(|e| { - DatabaseError::DataCorrupted(format!("Failed to create storage header: {e:?}")) - }) + build_patched_header(header, value_updates, map_updates, &removed) } /// Applies a storage patch to an existing storage header using precomputed map roots. @@ -297,6 +247,16 @@ pub(super) fn apply_storage_patch_with_roots( map_updates.insert(slot_name, root); } + build_patched_header(header, value_updates, map_updates, &removed) +} + +/// Rebuilds a storage header from the patch's value updates, map roots, and removals. +fn build_patched_header( + header: &AccountStorageHeader, + mut value_updates: HashMap<&StorageSlotName, Word>, + mut map_updates: HashMap<&StorageSlotName, Word>, + removed: &HashSet<&StorageSlotName>, +) -> Result { let mut slots = Vec::from_iter(header.slots().filter(|slot| !removed.contains(slot.name())).map(|slot| { let slot_name = slot.name(); diff --git a/crates/store/src/db/models/queries/accounts/delta/tests.rs b/crates/store/src/db/queries/upsert_accounts/delta/tests.rs similarity index 86% rename from crates/store/src/db/models/queries/accounts/delta/tests.rs rename to crates/store/src/db/queries/upsert_accounts/delta/tests.rs index 8eb735fa21..420096e8ba 100644 --- a/crates/store/src/db/models/queries/accounts/delta/tests.rs +++ b/crates/store/src/db/queries/upsert_accounts/delta/tests.rs @@ -4,7 +4,6 @@ use std::collections::BTreeMap; use assert_matches::assert_matches; -use diesel::{ExpressionMethods, QueryDsl, RunQueryDsl, SqliteConnection}; use miden_node_utils::fee::test_fee_params; use miden_protocol::account::auth::{AuthScheme, PublicKeyCommitment}; use miden_protocol::account::component::AccountComponentMetadata; @@ -12,9 +11,11 @@ use miden_protocol::account::{ Account, AccountBuilder, AccountComponent, + AccountHeader, AccountId, AccountIdVersion, AccountPatch, + AccountStorageHeader, AccountStoragePatch, AccountType, AccountUpdateDetails, @@ -29,36 +30,73 @@ use miden_protocol::account::{ StorageValuePatch, }; use miden_protocol::asset::{Asset, FungibleAsset}; -use miden_protocol::block::{BlockAccountUpdate, BlockHeader, BlockNumber, ValidatorKeys}; +use miden_protocol::block::{ + BlockAccountUpdate, + BlockHeader, + BlockNumber, + BlockSignatures, + ValidatorKeys, +}; use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SigningKey; use miden_protocol::testing::account_id::{ ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET, ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET_1, }; -use miden_protocol::utils::serde::Serializable; use miden_protocol::{EMPTY_WORD, Felt, Word}; use miden_standards::account::auth::{Approver, AuthSingleSig}; use miden_standards::code_builder::CodeBuilder; -use crate::db::models::queries::accounts::{ +use crate::db::queries::{ + self, PrecomputedPublicAccountState, PrecomputedPublicAccountStates, VALID_FOREVER, - select_account_header_with_storage_header_at_block, - select_account_vault_at_block, - select_full_account, - upsert_accounts, }; -use crate::db::schema::accounts; +use crate::db::{Result, TestDb}; use crate::errors::DatabaseError; -fn setup_test_db() -> SqliteConnection { - crate::db::migrations::test_connection() +// QUERY DRIVERS +// ================================================================================================ +// +// Each driver runs one query function on the test database, so a test body reads the same as the +// production call site with the transaction handle replaced by the test handle. + +fn upsert_accounts( + db: &TestDb, + accounts: &[BlockAccountUpdate], + block_num: BlockNumber, + precomputed_public_states: &PrecomputedPublicAccountStates, +) -> Result { + let accounts = accounts.to_vec(); + let precomputed_public_states = precomputed_public_states.clone(); + db.write(move |tx| { + queries::upsert_accounts(tx, &accounts, block_num, &precomputed_public_states) + }) } -fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { - use crate::db::schema::block_headers; +fn select_full_account(db: &TestDb, account_id: AccountId) -> Result { + db.read(move |tx| queries::select_full_account(tx, account_id)) +} +fn select_account_vault_at_block( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, +) -> Result> { + db.read(move |tx| queries::select_account_vault_at_block(tx, account_id, block_num)) +} + +fn select_account_header_with_storage_header_at_block( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, +) -> Result> { + db.read(move |tx| { + queries::select_account_header_with_storage_header_at_block(tx, account_id, block_num) + }) +} + +fn insert_block_header(db: &TestDb, block_num: BlockNumber) { let secret_key = SigningKey::new(); let block_header = BlockHeader::new( 1_u8.into(), @@ -74,17 +112,30 @@ fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { test_fee_params(), 0_u8.into(), ); - let signature = secret_key.sign(block_header.commitment()); - - diesel::insert_into(block_headers::table) - .values(( - block_headers::block_num.eq(i64::from(block_num.as_u32())), - block_headers::block_header.eq(block_header.to_bytes()), - block_headers::signature.eq(signature.to_bytes()), - block_headers::commitment.eq(block_header.commitment().to_bytes()), - )) - .execute(conn) - .expect("Failed to insert block header"); + let signatures = + BlockSignatures::new(vec![secret_key.sign(block_header.commitment())]).unwrap(); + + db.write::<_, DatabaseError, _>(move |tx| { + queries::insert_block_header(tx, &block_header, &signatures) + }) + .expect("Failed to insert block header"); +} + +/// Returns the current `accounts` row's commitment, nonce, and code commitment. +fn latest_account_row(db: &TestDb, account_id: AccountId) -> (Word, Option, Option) { + const SQL: &str = "SELECT account_commitment, nonce, code_commitment \ + FROM accounts WHERE account_id = ?1 AND valid_until = ?2"; + + db.read::<_, DatabaseError, _>(move |tx| { + Ok(tx + .query(SQL, &[&account_id, &VALID_FOREVER], |row| { + Ok((row.get::(0)?, row.get::>(1)?, row.get::>(2)?)) + })? + .into_iter() + .next()) + }) + .expect("query should succeed") + .expect("Account should exist in DB") } fn precomputed_state_from_account(account: &Account) -> PrecomputedPublicAccountState { @@ -144,10 +195,10 @@ fn callback_delta_test_account(seed: [u8; 32], slot_index: usize) -> Account { .unwrap() } -fn insert_public_account(conn: &mut SqliteConnection, block_num: BlockNumber, account: &Account) { +fn insert_public_account(db: &TestDb, block_num: BlockNumber, account: &Account) { let patch_initial = AccountPatch::try_from(account.clone()).unwrap(); upsert_accounts( - conn, + db, &[BlockAccountUpdate::new( account.id(), account.to_commitment(), @@ -160,14 +211,14 @@ fn insert_public_account(conn: &mut SqliteConnection, block_num: BlockNumber, ac } fn apply_callback_delta( - conn: &mut SqliteConnection, + db: &TestDb, account_id: AccountId, faucet_id: AccountId, block: BlockNumber, amount: u64, nonce_delta: u64, ) -> Account { - let prev = select_full_account(conn, account_id).expect("load account"); + let prev = select_full_account(db, account_id).expect("load account"); let callback_template = FungibleAsset::new(faucet_id, amount).unwrap(); let prev_amount = match prev.vault().get(callback_template.id()) { Some(Asset::Fungible(f)) => f.amount().as_u64(), @@ -193,7 +244,7 @@ fn apply_callback_delta( let precomputed_public_states = precomputed_states_from_account(&expected); upsert_accounts( - conn, + db, &[BlockAccountUpdate::new( account_id, expected.to_commitment(), @@ -204,7 +255,7 @@ fn apply_callback_delta( ) .expect("partial delta upsert failed"); - let after = select_full_account(conn, account_id).expect("load account after"); + let after = select_full_account(db, account_id).expect("load account after"); assert_eq!(after.vault().root(), expected.vault().root(), "vault root mismatch"); assert_eq!(after.to_commitment(), expected.to_commitment(), "commitment mismatch"); after @@ -241,7 +292,7 @@ fn optimized_delta_matches_full_account_method() { const NONCE_DELTA: u64 = 5; const VAULT_AMOUNT: u64 = 500; - let mut conn = setup_test_db(); + let db = TestDb::new(); // Create an account with value slots only (no map slots to avoid SmtForest complexity) let slot_value_initial = Word::from([ @@ -279,8 +330,8 @@ fn optimized_delta_matches_full_account_method() { let block_1 = BlockNumber::from(BLOCK_NUM_1); let block_2 = BlockNumber::from(BLOCK_NUM_2); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); // Insert the initial account at block 1 (full state) - no vault assets let patch_initial = AccountPatch::try_from(account.clone()).unwrap(); @@ -290,7 +341,7 @@ fn optimized_delta_matches_full_account_method() { AccountUpdateDetails::Public(patch_initial), ); upsert_accounts( - &mut conn, + &db, &[account_update_initial], block_1, &precomputed_states_from_account(&account), @@ -299,7 +350,7 @@ fn optimized_delta_matches_full_account_method() { // Verify initial state let full_account_before = - select_full_account(&mut conn, account.id()).expect("Failed to load full account"); + select_full_account(&db, account.id()).expect("Failed to load full account"); assert_eq!(full_account_before.nonce(), account.nonce()); assert!( full_account_before.vault().assets().next().is_none(), @@ -371,13 +422,13 @@ fn optimized_delta_matches_full_account_method() { final_commitment, AccountUpdateDetails::Public(partial_patch), ); - upsert_accounts(&mut conn, &[account_update], block_2, &precomputed_public_states) + upsert_accounts(&db, &[account_update], block_2, &precomputed_public_states) .expect("Partial delta upsert failed"); // ----- VERIFY: Query the DB and check that optimized path produced correct results ----- let (header_after, storage_header_after) = - select_account_header_with_storage_header_at_block(&mut conn, account.id(), block_2) + select_account_header_with_storage_header_at_block(&db, account.id(), block_2) .expect("Query should succeed") .expect("Account should exist"); @@ -405,7 +456,7 @@ fn optimized_delta_matches_full_account_method() { ); // Verify vault assets - let vault_assets_after = select_account_vault_at_block(&mut conn, account.id(), block_2) + let vault_assets_after = select_account_vault_at_block(&db, account.id(), block_2) .expect("Query vault should succeed"); assert_eq!(vault_assets_after.len(), 1, "Should have 1 vault asset"); @@ -422,8 +473,8 @@ fn optimized_delta_matches_full_account_method() { ); // Also verify we can load the full account and it has correct state - let full_account_after = select_full_account(&mut conn, account.id()) - .expect("Failed to load full account after update"); + let full_account_after = + select_full_account(&db, account.id()).expect("Failed to load full account after update"); assert_eq!(full_account_after.nonce(), expected_nonce, "Full account nonce mismatch"); assert_eq!( @@ -454,7 +505,7 @@ fn optimized_delta_updates_non_empty_vault() { const ADDED_AMOUNT_BLOCK_3: u64 = 150; const SLOT_INDEX: usize = 0; - let mut conn = setup_test_db(); + let db = TestDb::new(); let faucet_id = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap(); let faucet_id_1 = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET_1).unwrap(); @@ -488,9 +539,9 @@ fn optimized_delta_updates_non_empty_vault() { let block_1 = BlockNumber::from(BLOCK_NUM_1); let block_2 = BlockNumber::from(BLOCK_NUM_2); let block_3 = BlockNumber::from(BLOCK_NUM_3); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); - insert_block_header(&mut conn, block_3); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); + insert_block_header(&db, block_3); // Block 1: insert full-state patch (initial account with 700 tokens of faucet_id) let patch_initial = AccountPatch::try_from(account.clone()).unwrap(); @@ -500,7 +551,7 @@ fn optimized_delta_updates_non_empty_vault() { AccountUpdateDetails::Public(patch_initial), ); upsert_accounts( - &mut conn, + &db, &[account_update_initial], block_1, &precomputed_states_from_account(&account), @@ -508,7 +559,7 @@ fn optimized_delta_updates_non_empty_vault() { .expect("Initial upsert failed"); let full_account_before = - select_full_account(&mut conn, account.id()).expect("Failed to load full account"); + select_full_account(&db, account.id()).expect("Failed to load full account"); // Block 2: partial patch — remove faucet_id (700), add faucet_id_1 (250) let removed_asset = Asset::Fungible(FungibleAsset::new(faucet_id, INITIAL_AMOUNT).unwrap()); @@ -540,10 +591,10 @@ fn optimized_delta_updates_non_empty_vault() { expected_commitment, AccountUpdateDetails::Public(partial_patch), ); - upsert_accounts(&mut conn, &[account_update], block_2, &precomputed_public_states) + upsert_accounts(&db, &[account_update], block_2, &precomputed_public_states) .expect("Partial delta upsert failed"); - let vault_assets_after = select_account_vault_at_block(&mut conn, account.id(), block_2) + let vault_assets_after = select_account_vault_at_block(&db, account.id(), block_2) .expect("Query vault should succeed"); assert_eq!(vault_assets_after.len(), 1, "Should have 1 vault asset"); @@ -552,8 +603,8 @@ fn optimized_delta_updates_non_empty_vault() { assert_eq!(f.amount().as_u64(), ADDED_AMOUNT_BLOCK_2, "Amount should match"); }); - let full_account_after = select_full_account(&mut conn, account.id()) - .expect("Failed to load full account after update"); + let full_account_after = + select_full_account(&db, account.id()).expect("Failed to load full account after update"); assert_eq!(full_account_after.vault().root(), expected_vault_root); assert_eq!(full_account_after.to_commitment(), expected_commitment); @@ -586,11 +637,11 @@ fn optimized_delta_updates_non_empty_vault() { commitment_3, AccountUpdateDetails::Public(partial_patch_3), ); - upsert_accounts(&mut conn, &[account_update_3], block_3, &precomputed_public_states_3) + upsert_accounts(&db, &[account_update_3], block_3, &precomputed_public_states_3) .expect("Block 3 upsert failed"); let full_account_final = - select_full_account(&mut conn, account.id()).expect("Failed to load after block 3"); + select_full_account(&db, account.id()).expect("Failed to load after block 3"); let final_assets: Vec = full_account_final.vault().assets().collect(); assert_eq!(final_assets.len(), 1, "Should have exactly 1 vault asset"); @@ -614,29 +665,22 @@ fn optimized_delta_updates_preserve_callback_flag() { const ADDED_AMOUNT_BLOCK_3: u64 = 150; const SLOT_INDEX: usize = 0; - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_1 = BlockNumber::from(1u32); let block_2 = BlockNumber::from(2u32); let block_3 = BlockNumber::from(3u32); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); - insert_block_header(&mut conn, block_3); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); + insert_block_header(&db, block_3); let faucet_id = callback_enabled_faucet_id(); let account = callback_delta_test_account(ACCOUNT_SEED, SLOT_INDEX); - insert_public_account(&mut conn, block_1, &account); + insert_public_account(&db, block_1, &account); - apply_callback_delta( - &mut conn, - account.id(), - faucet_id, - block_2, - ADDED_AMOUNT_BLOCK_2, - NONCE_DELTA, - ); + apply_callback_delta(&db, account.id(), faucet_id, block_2, ADDED_AMOUNT_BLOCK_2, NONCE_DELTA); let final_account = apply_callback_delta( - &mut conn, + &db, account.id(), faucet_id, block_3, @@ -674,7 +718,7 @@ fn optimized_delta_updates_storage_map_header() { // Use nonzero nonce delta (required when storage/vault changes). const NONCE_DELTA: u64 = 1; - let mut conn = setup_test_db(); + let db = TestDb::new(); let map_key = StorageMapKey::new(Word::from([ Felt::new_unchecked(MAP_KEY_VALUES[0]), @@ -722,8 +766,8 @@ fn optimized_delta_updates_storage_map_header() { let block_1 = BlockNumber::from(BLOCK_NUM_1); let block_2 = BlockNumber::from(BLOCK_NUM_2); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); let patch_initial = AccountPatch::try_from(account.clone()).unwrap(); let account_update_initial = BlockAccountUpdate::new( @@ -732,7 +776,7 @@ fn optimized_delta_updates_storage_map_header() { AccountUpdateDetails::Public(patch_initial), ); upsert_accounts( - &mut conn, + &db, &[account_update_initial], block_1, &precomputed_states_from_account(&account), @@ -740,7 +784,7 @@ fn optimized_delta_updates_storage_map_header() { .expect("Initial upsert failed"); let full_account_before = - select_full_account(&mut conn, account.id()).expect("Failed to load full account"); + select_full_account(&db, account.id()).expect("Failed to load full account"); let map_patch = StorageMapPatch::from_iters([], [(map_key, map_value_updated)]); let storage_patch = AccountStoragePatch::from_raw(BTreeMap::from_iter([( @@ -771,11 +815,11 @@ fn optimized_delta_updates_storage_map_header() { expected_commitment, AccountUpdateDetails::Public(partial_patch), ); - upsert_accounts(&mut conn, &[account_update], block_2, &precomputed_public_states) + upsert_accounts(&db, &[account_update], block_2, &precomputed_public_states) .expect("Partial delta upsert failed"); let (header_after, storage_header_after) = - select_account_header_with_storage_header_at_block(&mut conn, account.id(), block_2) + select_account_header_with_storage_header_at_block(&db, account.id(), block_2) .expect("Query should succeed") .expect("Account should exist"); @@ -828,11 +872,11 @@ fn partial_public_upsert_requires_precomputed_state() { const ACCOUNT_SEED: [u8; 32] = [80u8; 32]; const SLOT_INDEX: usize = 0; - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_1 = BlockNumber::from(1u32); let block_2 = BlockNumber::from(2u32); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); let component_storage = vec![StorageSlot::with_value(StorageSlotName::mock(SLOT_INDEX), EMPTY_WORD)]; @@ -860,7 +904,7 @@ fn partial_public_upsert_requires_precomputed_state() { let patch_initial = AccountPatch::try_from(account.clone()).unwrap(); upsert_accounts( - &mut conn, + &db, &[BlockAccountUpdate::new( account.id(), account.to_commitment(), @@ -871,7 +915,7 @@ fn partial_public_upsert_requires_precomputed_state() { ) .expect("initial full-state upsert failed"); - let mut current_account = select_full_account(&mut conn, account.id()).unwrap(); + let mut current_account = select_full_account(&db, account.id()).unwrap(); let patch = AccountPatch::new( account.id(), AccountStoragePatch::new(), @@ -883,7 +927,7 @@ fn partial_public_upsert_requires_precomputed_state() { current_account.apply_patch(&patch).unwrap(); let err = upsert_accounts( - &mut conn, + &db, &[BlockAccountUpdate::new( account.id(), current_account.to_commitment(), @@ -902,11 +946,11 @@ fn partial_public_upsert_rejects_bad_precomputed_root() { const ACCOUNT_SEED: [u8; 32] = [81u8; 32]; const SLOT_INDEX: usize = 0; - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_1 = BlockNumber::from(1u32); let block_2 = BlockNumber::from(2u32); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); let component_storage = vec![StorageSlot::with_value(StorageSlotName::mock(SLOT_INDEX), EMPTY_WORD)]; @@ -931,7 +975,7 @@ fn partial_public_upsert_rejects_bad_precomputed_root() { let patch_initial = AccountPatch::try_from(account.clone()).unwrap(); upsert_accounts( - &mut conn, + &db, &[BlockAccountUpdate::new( account.id(), account.to_commitment(), @@ -942,7 +986,7 @@ fn partial_public_upsert_rejects_bad_precomputed_root() { ) .expect("initial full-state upsert failed"); - let mut expected_account = select_full_account(&mut conn, account.id()).unwrap(); + let mut expected_account = select_full_account(&db, account.id()).unwrap(); let patch = AccountPatch::new( account.id(), AccountStoragePatch::new(), @@ -958,7 +1002,7 @@ fn partial_public_upsert_rejects_bad_precomputed_root() { Word::from([Felt::new_unchecked(999); 4]); let err = upsert_accounts( - &mut conn, + &db, &[BlockAccountUpdate::new( account.id(), expected_account.to_commitment(), @@ -986,10 +1030,10 @@ fn upsert_private_account() { // Use fixed commitment values to validate storage behavior. const COMMITMENT_WORDS: [u64; 4] = [1, 2, 3, 4]; - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_num = BlockNumber::from(BLOCK_NUM); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); // Create a private account ID let account_id = AccountId::dummy( @@ -1010,29 +1054,14 @@ fn upsert_private_account() { let account_update = BlockAccountUpdate::new(account_id, account_commitment, AccountUpdateDetails::Private); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ) - .expect("Private account upsert failed"); + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()) + .expect("Private account upsert failed"); // Verify the account exists and commitment matches - let (stored_commitment, stored_nonce, stored_code): (Vec, Option, Option>) = - accounts::table - .filter(accounts::account_id.eq(account_id.to_bytes())) - .filter(accounts::valid_until.eq(VALID_FOREVER)) - .select((accounts::account_commitment, accounts::nonce, accounts::code_commitment)) - .first(&mut conn) - .expect("Account should exist in DB"); + let (stored_commitment, stored_nonce, stored_code) = latest_account_row(&db, account_id); - assert_eq!( - stored_commitment, - account_commitment.to_bytes(), - "Stored commitment should match" - ); + assert_eq!(stored_commitment, account_commitment, "Stored commitment should match"); // Private accounts have NULL for nonce, code_commitment, storage_header, vault_root assert!(stored_nonce.is_none(), "Private account should have NULL nonce"); @@ -1053,10 +1082,10 @@ fn upsert_full_state_delta() { // Use explicit slot index to avoid magic numbers. const SLOT_INDEX: usize = 0; - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_num = BlockNumber::from(BLOCK_NUM); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); // Create an account with storage let slot_value = Word::from([ @@ -1099,17 +1128,12 @@ fn upsert_full_state_delta() { AccountUpdateDetails::Public(patch), ); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &precomputed_states_from_account(&account), - ) - .expect("Full-state delta upsert failed"); + upsert_accounts(&db, &[account_update], block_num, &precomputed_states_from_account(&account)) + .expect("Full-state delta upsert failed"); // Verify the account state was stored correctly let (header, storage_header) = - select_account_header_with_storage_header_at_block(&mut conn, account.id(), block_num) + select_account_header_with_storage_header_at_block(&db, account.id(), block_num) .expect("Query should succeed") .expect("Account should exist"); @@ -1126,8 +1150,7 @@ fn upsert_full_state_delta() { ); // Verify we can load the full account back - let loaded_account = - select_full_account(&mut conn, account.id()).expect("Should load full account"); + let loaded_account = select_full_account(&db, account.id()).expect("Should load full account"); assert_eq!(loaded_account.nonce(), account.nonce()); assert_eq!(loaded_account.code().commitment(), account.code().commitment()); diff --git a/crates/store/src/db/queries/upsert_accounts/insert_account_code.sql b/crates/store/src/db/queries/upsert_accounts/insert_account_code.sql new file mode 100644 index 0000000000..6e4758dca8 --- /dev/null +++ b/crates/store/src/db/queries/upsert_accounts/insert_account_code.sql @@ -0,0 +1,5 @@ +-- Stores an account's code, keyed by its commitment. Code is shared across accounts and across an +-- account's versions, so a commitment already present is left untouched. +INSERT INTO account_codes (code_commitment, code) +VALUES (?1, ?2) +ON CONFLICT(code_commitment) DO NOTHING diff --git a/crates/store/src/db/queries/upsert_accounts/mod.rs b/crates/store/src/db/queries/upsert_accounts/mod.rs new file mode 100644 index 0000000000..8eebe09752 --- /dev/null +++ b/crates/store/src/db/queries/upsert_accounts/mod.rs @@ -0,0 +1,541 @@ +//! Writes the account state produced by a block. +//! +//! Every account table is versioned: a row is applicable for blocks in `[block_num, valid_until)`, +//! and writing a new version closes the previous one. This module owns that bookkeeping for the +//! `accounts` row itself and drives the per-key writes in +//! [`insert_account_vault_asset`](super::insert_account_vault_asset) and +//! [`insert_account_storage_map_value`](super::insert_account_storage_map_value). + +use std::collections::BTreeMap; + +use miden_node_db::sqlite::WriteTx; +use miden_node_utils::tracing::miden_instrument; +use miden_protocol::account::{ + Account, + AccountCode, + AccountHeader, + AccountId, + AccountPatch, + AccountStorageHeader, + AccountUpdateDetails, + StorageMapKey, + StorageMapPatchEntries, + StorageSlotContent, + StorageSlotName, +}; +use miden_protocol::asset::{Asset, AssetId}; +use miden_protocol::block::{BlockAccountUpdate, BlockNumber}; +use miden_protocol::{Felt, Word}; +use miden_standards::account::auth::NetworkAccount; + +use crate::COMPONENT; +use crate::db::queries::insert_account_storage_map_value::insert_account_storage_map_value_inner; +use crate::db::queries::{ + NetworkAccountType, + VALID_FOREVER, + insert_account_storage_map_value, + insert_account_vault_asset, +}; +use crate::errors::DatabaseError; + +mod delta; +use delta::{ + AccountStateForInsert, + LatestAccountStateRow, + PartialAccountState, + PrecomputedFullAccountState, + apply_storage_patch_with_roots, + select_latest_account_state, +}; + +#[cfg(test)] +mod tests; + +const SQL_INSERT_ACCOUNT_CODE: &str = include_str!("insert_account_code.sql"); +const SQL_CLOSE_ACCOUNT_VALIDITY: &str = include_str!("close_account_validity.sql"); +const SQL_UPSERT_ACCOUNT: &str = include_str!("upsert_account.sql"); + +// PRECOMPUTED PUBLIC ACCOUNT STATE +// ================================================================================================ + +/// Public account state commitments computed by the account state forest before SQLite writes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PrecomputedPublicAccountState { + pub vault_root: Word, + pub storage_map_roots: BTreeMap, +} + +pub type PrecomputedPublicAccountStates = BTreeMap; + +// QUERY +// ================================================================================================ + +type PendingStorageInserts = Vec<(AccountId, StorageSlotName, StorageMapKey, Word)>; +type PendingAssetInserts = Vec<(AccountId, AssetId, Option)>; + +/// Writes the state of every account a block updated. +/// +/// Attention: Assumes the account details are NOT null! The schema explicitly allows this though! +#[miden_instrument( + target = COMPONENT, + err, +)] +pub(crate) fn upsert_accounts( + tx: &WriteTx<'_>, + accounts: &[BlockAccountUpdate], + block_num: BlockNumber, + precomputed_public_states: &PrecomputedPublicAccountStates, +) -> Result { + let mut count = 0; + for update in accounts { + upsert_account(tx, update, block_num, precomputed_public_states)?; + count += 1; + } + + Ok(count) +} + +/// Writes a single account's new state, closing its previous version's validity interval. +fn upsert_account( + tx: &WriteTx<'_>, + update: &BlockAccountUpdate, + block_num: BlockNumber, + precomputed_public_states: &PrecomputedPublicAccountStates, +) -> Result<(), DatabaseError> { + let account_id = update.account_id(); + + // Pull the latest row once. Partial updates consume the state headers below, while every update + // carries forward creation metadata. + let existing = select_latest_account_state(tx, account_id)?; + let account_is_new = existing.is_none(); + + let created_at_block = + existing.as_ref().map_or(block_num, LatestAccountStateRow::created_at_block); + + // NOTE: we collect storage / asset inserts to apply them only after the account row is written. + // The storage and vault tables have FKs pointing to accounts `(account_id, block_num)`, so + // inserting them earlier would violate those constraints when inserting a brand-new account. + let (account_state, pending_storage_inserts, pending_asset_inserts) = + prepare_account_update(update, block_num, precomputed_public_states, existing.as_ref())?; + + // Inherit the classification when the account already exists; otherwise classify it once at + // creation based on the new state. + let network_account_type = match &existing { + Some(row) => row.network_account_type(), + None => match &account_state { + AccountStateForInsert::FullAccount(account) + if NetworkAccount::new(account.clone()).is_ok() => + { + NetworkAccountType::Network + }, + AccountStateForInsert::PrecomputedFullState(state) if state.is_network_account => { + NetworkAccountType::Network + }, + _ => NetworkAccountType::None, + }, + }; + + // Insert account _code_ for full accounts (new account creation). + match &account_state { + AccountStateForInsert::FullAccount(account) => insert_account_code(tx, account.code())?, + AccountStateForInsert::PrecomputedFullState(state) => insert_account_code(tx, &state.code)?, + AccountStateForInsert::Private | AccountStateForInsert::PartialState(_) => {}, + } + + // Close the previous row's validity interval and insert the NEW account row. + tx.execute(SQL_CLOSE_ACCOUNT_VALIDITY, &[&block_num, &account_id, &VALID_FOREVER])?; + + let row = AccountRow::new( + account_id, + network_account_type, + update.final_state_commitment(), + block_num, + created_at_block, + &account_state, + ); + row.upsert(tx)?; + + // Insert pending storage map entries. TODO consider batching + for (acc_id, slot_name, key, value) in pending_storage_inserts { + if account_is_new { + // A brand-new account cannot have a previous open row to invalidate. + insert_account_storage_map_value_inner( + tx, acc_id, block_num, &slot_name, key, value, false, + )?; + } else { + insert_account_storage_map_value(tx, acc_id, block_num, &slot_name, key, value)?; + } + } + + for (acc_id, vault_key, asset) in pending_asset_inserts { + insert_account_vault_asset(tx, acc_id, block_num, vault_key, asset)?; + } + + Ok(()) +} + +/// Stores an account's code, keyed by its commitment; a commitment already present is left as is. +fn insert_account_code(tx: &WriteTx<'_>, code: &AccountCode) -> Result<(), DatabaseError> { + tx.execute(SQL_INSERT_ACCOUNT_CODE, &[&code.commitment(), code])?; + Ok(()) +} + +// UPDATE PREPARATION +// ================================================================================================ + +/// Turns a block's account update into the row state to write, plus the storage-map and vault +/// writes that follow it. +fn prepare_account_update( + update: &BlockAccountUpdate, + block_num: BlockNumber, + precomputed_public_states: &PrecomputedPublicAccountStates, + existing: Option<&LatestAccountStateRow>, +) -> Result<(AccountStateForInsert, PendingStorageInserts, PendingAssetInserts), DatabaseError> { + let account_id = update.account_id(); + + match update.details() { + AccountUpdateDetails::Private => Ok((AccountStateForInsert::Private, vec![], vec![])), + + // New account is always a full account, but also comes as an update + AccountUpdateDetails::Public(patch) if patch.is_full_state() => { + if block_num == BlockNumber::GENESIS { + let account = Account::try_from(patch) + .expect("Patch to full account always works for full state patches"); + debug_assert_eq!(account_id, account.id()); + prepare_full_account_update(update, account) + } else { + let precomputed = precomputed_state(precomputed_public_states, account_id)?; + prepare_precomputed_full_account_update(update, patch, precomputed) + } + }, + + // Update of an existing account + AccountUpdateDetails::Public(patch) => { + let precomputed = precomputed_state(precomputed_public_states, account_id)?; + let existing = existing.ok_or(DatabaseError::AccountNotFoundInDb(account_id))?; + prepare_partial_account_update(update, account_id, patch, precomputed, existing) + }, + } +} + +/// Looks up the forest-computed state for a public account, which every non-genesis public update +/// requires. +fn precomputed_state( + precomputed_public_states: &PrecomputedPublicAccountStates, + account_id: AccountId, +) -> Result<&PrecomputedPublicAccountState, DatabaseError> { + precomputed_public_states.get(&account_id).ok_or_else(|| { + DatabaseError::DataCorrupted(format!( + "missing precomputed public account state for account {account_id}" + )) + }) +} + +fn prepare_full_account_update( + update: &BlockAccountUpdate, + account: Account, +) -> Result<(AccountStateForInsert, PendingStorageInserts, PendingAssetInserts), DatabaseError> { + let account_id = account.id(); + + // sanity check the commitment of account matches the final state commitment + if account.to_commitment() != update.final_state_commitment() { + return Err(DatabaseError::AccountCommitmentsMismatch { + calculated: account.to_commitment(), + expected: update.final_state_commitment(), + }); + } + + // collect storage-map inserts to apply after account upsert + let mut storage = Vec::new(); + for slot in account.storage().slots() { + if let StorageSlotContent::Map(storage_map) = slot.content() { + for (key, value) in storage_map.entries() { + storage.push((account_id, slot.name().clone(), *key, *value)); + } + } + } + + // collect vault-asset inserts to apply after account upsert + let mut assets = Vec::new(); + for asset in account.vault().assets() { + // Only insert assets with non-zero values for fungible assets + let should_insert = match asset { + Asset::Fungible(fungible) => fungible.amount().as_u64() > 0, + Asset::NonFungible(_) => true, + }; + if should_insert { + assets.push((account_id, asset.id(), Some(asset))); + } + } + + Ok((AccountStateForInsert::FullAccount(account), storage, assets)) +} + +/// Prepares a full public-account insertion using roots computed by the account-state forest. +/// +/// This avoids reconstructing the account's vault and storage maps in SQLite. The returned state +/// contains the account-row fields, while storage-map entries and vault assets are returned +/// separately for insertion after the account row has satisfied their foreign-key dependency. +/// Empty-word map entries and assets are omitted from the pending inserts. +/// +/// # Errors +/// +/// Returns an error if the full-state patch is missing its code or nonce, a required precomputed +/// storage root is absent, an asset is invalid, or the reconstructed account header does not match +/// the update's final state commitment. +fn prepare_precomputed_full_account_update( + update: &BlockAccountUpdate, + patch: &AccountPatch, + precomputed: &PrecomputedPublicAccountState, +) -> Result<(AccountStateForInsert, PendingStorageInserts, PendingAssetInserts), DatabaseError> { + let account_id = patch.id(); + let code = patch.code().cloned().ok_or_else(|| { + DatabaseError::DataCorrupted(format!( + "full-state patch for account {account_id} is missing account code" + )) + })?; + let nonce = patch.final_nonce().ok_or_else(|| { + DatabaseError::DataCorrupted(format!( + "full-state patch for account {account_id} is missing final nonce" + )) + })?; + + let storage_header = apply_storage_patch_with_roots( + &AccountStorageHeader::new(Vec::new())?, + patch.storage(), + &precomputed.storage_map_roots, + )?; + let account_header = AccountHeader::new( + account_id, + nonce, + precomputed.vault_root, + storage_header.to_commitment(), + code.commitment(), + ); + if account_header.to_commitment() != update.final_state_commitment() { + return Err(DatabaseError::AccountCommitmentsMismatch { + calculated: account_header.to_commitment(), + expected: update.final_state_commitment(), + }); + } + + let storage = patch + .storage() + .maps() + .flat_map(|(slot_name, map_patch)| { + map_patch.entries().into_iter().flat_map(move |entries| { + entries + .as_map() + .iter() + .filter(|(_key, value)| **value != Word::empty()) + .map(move |(key, value)| (account_id, slot_name.clone(), *key, *value)) + }) + }) + .collect(); + let assets = patch + .vault() + .iter() + .filter(|(_asset_id, value)| **value != Word::empty()) + .map(|(asset_id, value)| { + Asset::from_id_and_value(*asset_id, *value) + .map(|asset| (account_id, *asset_id, Some(asset))) + }) + .collect::, _>>()?; + + // The patch carries full state, so it can be turned back into an account and classified with + // the canonical check. + let is_network_account = NetworkAccount::new(Account::try_from(patch)?).is_ok(); + let state = PrecomputedFullAccountState { + nonce, + code, + storage_header, + vault_root: precomputed.vault_root, + is_network_account, + }; + + Ok((AccountStateForInsert::PrecomputedFullState(state), storage, assets)) +} + +/// Prepares a partial public-account update using the latest row and precomputed forest roots. +/// +/// Unchanged header fields are carried forward from `existing`. The returned partial state is used +/// for the next account row, while storage-map values and vault asset updates are returned +/// separately for insertion after that row. Empty vault values are represented as removals. +/// +/// # Errors +/// +/// Returns an error if the existing row is invalid, a required precomputed storage root is absent, +/// a patched asset is invalid, or the reconstructed account header does not match the update's +/// final state commitment. +fn prepare_partial_account_update( + update: &BlockAccountUpdate, + account_id: AccountId, + patch: &AccountPatch, + precomputed: &PrecomputedPublicAccountState, + existing: &LatestAccountStateRow, +) -> Result<(AccountStateForInsert, PendingStorageInserts, PendingAssetInserts), DatabaseError> { + // Build the minimal account state needed for partial patch application from the latest row that + // was loaded with the account's creation metadata. + let state_headers = existing.state_headers(account_id)?; + + // --- Process asset updates. --------------------------------- The patch carries absolute final + // values, so encode `Some` as update and `None` (an empty value word) as removal. + let mut assets = Vec::new(); + for (vault_key, value) in patch.vault().iter() { + let update_or_remove = if *value == Word::empty() { + None + } else { + Some(Asset::from_id_and_value(*vault_key, *value)?) + }; + assets.push((account_id, *vault_key, update_or_remove)); + } + + // --- Collect storage map updates. --------------------------- + + let mut storage = Vec::new(); + for (slot_name, map_patch) in patch.storage().maps() { + for (key, value) in map_patch.entries().into_iter().flat_map(StorageMapPatchEntries::as_map) + { + storage.push((account_id, slot_name.clone(), *key, *value)); + } + } + + // Apply the patch storage to the given storage header. + let new_storage_header = apply_storage_patch_with_roots( + &state_headers.storage_header, + patch.storage(), + &precomputed.storage_map_roots, + )?; + + let new_vault_root = precomputed.vault_root; + + // --- Compute updated account state for the accounts row. --- Use the absolute final nonce. + let new_nonce = patch.final_nonce().unwrap_or(state_headers.nonce); + + // Create minimal account state data for the row insert. + let account_state = PartialAccountState { + nonce: new_nonce, + code_commitment: state_headers.code_commitment, + storage_header: new_storage_header, + vault_root: new_vault_root, + }; + + let account_header = AccountHeader::new( + account_id, + account_state.nonce, + account_state.vault_root, + account_state.storage_header.to_commitment(), + account_state.code_commitment, + ); + + if account_header.to_commitment() != update.final_state_commitment() { + return Err(DatabaseError::AccountCommitmentsMismatch { + calculated: account_header.to_commitment(), + expected: update.final_state_commitment(), + }); + } + + Ok((AccountStateForInsert::PartialState(account_state), storage, assets)) +} + +// ACCOUNT ROW +// ================================================================================================ + +/// The `accounts` row written for an account's new state. +/// +/// Private accounts carry no public state, so every optional column is `None` for them. +pub(crate) struct AccountRow { + account_id: AccountId, + network_account_type: NetworkAccountType, + block_num: BlockNumber, + account_commitment: Word, + code_commitment: Option, + nonce: Option, + storage_header: Option, + vault_root: Option, + created_at_block: BlockNumber, +} + +impl AccountRow { + /// Builds the row for the given prepared account state. + fn new( + account_id: AccountId, + network_account_type: NetworkAccountType, + account_commitment: Word, + block_num: BlockNumber, + created_at_block: BlockNumber, + state: &AccountStateForInsert, + ) -> Self { + let mut row = Self::new_private( + account_id, + network_account_type, + account_commitment, + block_num, + created_at_block, + ); + + match state { + AccountStateForInsert::Private => {}, + AccountStateForInsert::FullAccount(account) => { + row.code_commitment = Some(account.code().commitment()); + row.nonce = Some(account.nonce()); + row.storage_header = Some(account.storage().to_header()); + row.vault_root = Some(account.vault().root()); + }, + AccountStateForInsert::PrecomputedFullState(state) => { + row.code_commitment = Some(state.code.commitment()); + row.nonce = Some(state.nonce); + row.storage_header = Some(state.storage_header.clone()); + row.vault_root = Some(state.vault_root); + }, + AccountStateForInsert::PartialState(state) => { + row.code_commitment = Some(state.code_commitment); + row.nonce = Some(state.nonce); + row.storage_header = Some(state.storage_header.clone()); + row.vault_root = Some(state.vault_root); + }, + } + + row + } + + /// Builds the row for a private account, which has no public state. + pub(crate) fn new_private( + account_id: AccountId, + network_account_type: NetworkAccountType, + account_commitment: Word, + block_num: BlockNumber, + created_at_block: BlockNumber, + ) -> Self { + Self { + account_id, + network_account_type, + block_num, + account_commitment, + code_commitment: None, + nonce: None, + storage_header: None, + vault_root: None, + created_at_block, + } + } + + /// Writes the row as the account's current, open-ended version. + pub(crate) fn upsert(&self, tx: &WriteTx<'_>) -> Result { + Ok(tx.execute( + SQL_UPSERT_ACCOUNT, + &[ + &self.account_id, + &self.network_account_type, + &self.block_num, + &self.account_commitment, + &self.code_commitment, + &self.nonce, + &self.storage_header, + &self.vault_root, + &self.created_at_block, + &VALID_FOREVER, + ], + )?) + } +} diff --git a/crates/store/src/db/queries/upsert_accounts/select_latest_account_state.sql b/crates/store/src/db/queries/upsert_accounts/select_latest_account_state.sql new file mode 100644 index 0000000000..f15e012e63 --- /dev/null +++ b/crates/store/src/db/queries/upsert_accounts/select_latest_account_state.sql @@ -0,0 +1,10 @@ +-- Returns the fields of an account's current row that preparing its next update depends on: +-- +-- * `created_at_block` and `network_account_type` are carried forward by every update +-- * `nonce` is preserved when a partial patch omits its final nonce +-- * `code_commitment` is unchanged by partial patches +-- * `storage_header` is the header the storage patch is applied to +SELECT created_at_block, network_account_type, nonce, code_commitment, storage_header +FROM accounts +WHERE account_id = ?1 + AND valid_until = ?2 diff --git a/crates/store/src/db/models/queries/accounts/tests.rs b/crates/store/src/db/queries/upsert_accounts/tests.rs similarity index 75% rename from crates/store/src/db/models/queries/accounts/tests.rs rename to crates/store/src/db/queries/upsert_accounts/tests.rs index cecdf0537f..907f5f7638 100644 --- a/crates/store/src/db/models/queries/accounts/tests.rs +++ b/crates/store/src/db/queries/upsert_accounts/tests.rs @@ -2,8 +2,7 @@ use std::collections::BTreeMap; -use diesel::query_dsl::methods::SelectDsl; -use diesel::{BoolExpressionMethods, ExpressionMethods, OptionalExtension, QueryDsl, RunQueryDsl}; +use miden_node_proto::domain::account::AccountVaultDetails; use miden_node_utils::fee::test_fee_params; use miden_protocol::account::auth::{AuthScheme, PublicKeyCommitment}; use miden_protocol::account::component::AccountComponentMetadata; @@ -31,69 +30,135 @@ use miden_protocol::account::{ StorageSlotType, }; use miden_protocol::asset::{NonFungibleAsset, NonFungibleAssetDetails}; -use miden_protocol::block::{BlockAccountUpdate, BlockHeader, BlockNumber, ValidatorKeys}; +use miden_protocol::block::{ + BlockAccountUpdate, + BlockHeader, + BlockNumber, + BlockSignatures, + ValidatorKeys, +}; use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SigningKey; use miden_protocol::testing::account_id::AccountIdBuilder; -use miden_protocol::utils::serde::{Deserializable, Serializable}; use miden_protocol::{EMPTY_WORD, Felt, Word}; use miden_standards::account::auth::{Approver, AuthSingleSig}; use miden_standards::code_builder::CodeBuilder; use super::*; -use crate::db::models::conv::SqlTypeConvert; -use crate::db::schema; +use crate::db::queries::{self, HISTORICAL_BLOCK_RETENTION, VALID_FOREVER}; +use crate::db::{Result, TestDb}; use crate::errors::DatabaseError; -fn setup_test_db() -> SqliteConnection { - crate::db::migrations::test_connection() +// QUERY DRIVERS +// ================================================================================================ +// +// Each driver runs one query function on the test database, so a test body reads the same as the +// production call site with the transaction handle replaced by the test handle. + +fn upsert_accounts( + db: &TestDb, + accounts: &[BlockAccountUpdate], + block_num: BlockNumber, + precomputed_public_states: &PrecomputedPublicAccountStates, +) -> Result { + let accounts = accounts.to_vec(); + let precomputed_public_states = precomputed_public_states.clone(); + db.write(move |tx| { + queries::upsert_accounts(tx, &accounts, block_num, &precomputed_public_states) + }) +} + +fn insert_account_vault_asset( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, + vault_key: AssetId, + asset: Option, +) -> Result { + db.write(move |tx| { + queries::insert_account_vault_asset(tx, account_id, block_num, vault_key, asset) + }) +} + +fn prune_history(db: &TestDb, chain_tip: BlockNumber) -> Result<(usize, usize, usize)> { + db.write(move |tx| queries::prune_history(tx, chain_tip)) +} + +fn select_latest_account_storage(db: &TestDb, account_id: AccountId) -> Result { + db.read(move |tx| queries::select_latest_account_storage(tx, account_id)) +} + +fn select_account_vault_at_block( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, +) -> Result> { + db.read(move |tx| queries::select_account_vault_at_block(tx, account_id, block_num)) +} + +fn select_account_header_with_storage_header_at_block( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, +) -> Result> { + db.read(move |tx| { + queries::select_account_header_with_storage_header_at_block(tx, account_id, block_num) + }) } +fn select_network_accounts_subset( + db: &TestDb, + account_ids: &[AccountId], +) -> Result> { + let account_ids = account_ids.to_vec(); + db.read(move |tx| queries::select_network_accounts_subset(tx, &account_ids)) +} + +// TEST HELPERS +// ================================================================================================ + /// Test helper: reconstructs account storage at a given block from DB. /// /// Reads `accounts.storage_header` and `account_storage_map_values` to reconstruct /// the full `AccountStorage` at the specified block. fn reconstruct_account_storage_at_block( - conn: &mut SqliteConnection, + db: &TestDb, account_id: AccountId, block_num: BlockNumber, -) -> Result { - use schema::account_storage_map_values as t; - - let account_id_bytes = account_id.to_bytes(); - let block_num_sql = block_num.to_raw_sql(); - - // Query storage header blob for this account at or before this block - let storage_blob: Option> = - SelectDsl::select(schema::accounts::table, schema::accounts::storage_header) - .filter(schema::accounts::account_id.eq(&account_id_bytes)) - .filter(schema::accounts::block_num.le(block_num_sql)) - .order(schema::accounts::block_num.desc()) - .limit(1) - .first(conn) - .optional()? - .flatten(); - - let Some(blob) = storage_blob else { +) -> Result { + const SQL_STORAGE_HEADER: &str = "SELECT storage_header FROM accounts \ + WHERE account_id = ?1 AND block_num <= ?2 \ + ORDER BY block_num DESC LIMIT 1"; + const SQL_MAP_VALUES: &str = "SELECT slot_name, key, value FROM account_storage_map_values \ + WHERE account_id = ?1 AND block_num <= ?2 \ + ORDER BY slot_name ASC, key ASC, block_num DESC"; + + let header = db.read::<_, DatabaseError, _>(move |tx| { + Ok(tx + .query(SQL_STORAGE_HEADER, &[&account_id, &block_num], |row| { + row.get::>(0) + })? + .into_iter() + .next() + .flatten()) + })?; + + let Some(header) = header else { return Ok(AccountStorage::new(Vec::new())?); }; - let header = AccountStorageHeader::read_from_bytes(&blob)?; - - // Query all map values for this account up to and including this block. - let map_values: Vec<(i64, String, Vec, Vec)> = - SelectDsl::select(t::table, (t::block_num, t::slot_name, t::key, t::value)) - .filter(t::account_id.eq(&account_id_bytes).and(t::block_num.le(block_num_sql))) - .order((t::slot_name.asc(), t::key.asc(), t::block_num.desc())) - .load(conn)?; + // Rows arrive newest-first per key, so the first one seen for a key is the latest. + let map_values = db.read::<_, DatabaseError, _>(move |tx| { + Ok(tx.query(SQL_MAP_VALUES, &[&account_id, &block_num], |row| { + Ok(( + row.get::(0)?, + row.get::(1)?, + row.get::(2)?, + )) + })?) + })?; - // For each (slot_name, key) pair, keep only the latest entry let mut latest_map_entries: BTreeMap<(StorageSlotName, StorageMapKey), Word> = BTreeMap::new(); - for (_, slot_name_str, key_bytes, value_bytes) in map_values { - let slot_name: StorageSlotName = slot_name_str.parse().map_err(|_| { - DatabaseError::DataCorrupted(format!("Invalid slot name: {slot_name_str}")) - })?; - let key = StorageMapKey::read_from_bytes(&key_bytes)?; - let value = Word::read_from_bytes(&value_bytes)?; + for (slot_name, key, value) in map_values { latest_map_entries.entry((slot_name, key)).or_insert(value); } @@ -164,9 +229,7 @@ fn create_test_account_with_storage() -> (Account, AccountId) { (account, account_id) } -fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { - use crate::db::schema::block_headers; - +fn insert_block_header(db: &TestDb, block_num: BlockNumber) { let secret_key = SigningKey::new(); let block_header = BlockHeader::new( 1_u8.into(), @@ -182,17 +245,44 @@ fn insert_block_header(conn: &mut SqliteConnection, block_num: BlockNumber) { test_fee_params(), 0_u8.into(), ); - let signature = secret_key.sign(block_header.commitment()); - - diesel::insert_into(block_headers::table) - .values(( - block_headers::block_num.eq(i64::from(block_num.as_u32())), - block_headers::block_header.eq(block_header.to_bytes()), - block_headers::signature.eq(signature.to_bytes()), - block_headers::commitment.eq(block_header.commitment().to_bytes()), - )) - .execute(conn) - .expect("Failed to insert block header"); + let signatures = + BlockSignatures::new(vec![secret_key.sign(block_header.commitment())]).unwrap(); + + db.write::<_, DatabaseError, _>(move |tx| { + queries::insert_block_header(tx, &block_header, &signatures) + }) + .expect("Failed to insert block header"); +} + +/// Counts the rows of `accounts` for the given account, and how many of them are current. +fn count_account_rows(db: &TestDb, account_id: AccountId) -> (i64, i64) { + const SQL: &str = "SELECT COUNT(*), COUNT(*) FILTER (WHERE valid_until = ?2) \ + FROM accounts WHERE account_id = ?1"; + + db.read::<_, DatabaseError, _>(move |tx| { + Ok(tx + .query(SQL, &[&account_id, &VALID_FOREVER], |row| { + Ok((row.get::(0)?, row.get::(1)?)) + })? + .into_iter() + .next() + .unwrap_or((0, 0))) + }) + .expect("Failed to count account rows") +} + +/// Returns whether the account's current row stores a storage header. +fn latest_account_has_storage_header(db: &TestDb, account_id: AccountId) -> Option { + const SQL: &str = "SELECT storage_header IS NOT NULL FROM accounts \ + WHERE account_id = ?1 AND valid_until = ?2"; + + db.read::<_, DatabaseError, _>(move |tx| { + Ok(tx + .query(SQL, &[&account_id, &VALID_FOREVER], |row| row.get::(0))? + .into_iter() + .next()) + }) + .expect("Failed to query storage header presence") } fn precomputed_state_from_account(account: &Account) -> PrecomputedPublicAccountState { @@ -270,9 +360,9 @@ fn assert_storage_map_slot_entries( #[test] fn select_account_header_at_block_returns_none_for_nonexistent() { - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); let account_id = AccountId::dummy( [99u8; 15], @@ -282,21 +372,20 @@ fn select_account_header_at_block_returns_none_for_nonexistent() { ); // Query for a non-existent account - let result = - select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num) - .expect("Query should succeed"); + let result = select_account_header_with_storage_header_at_block(&db, account_id, block_num) + .expect("Query should succeed"); assert!(result.is_none(), "Should return None for non-existent account"); } #[test] fn select_account_header_at_block_returns_correct_header() { - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, _) = create_test_account_with_storage(); let account_id = account.id(); let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); // Insert the account let patch = AccountPatch::try_from(account.clone()).unwrap(); @@ -306,17 +395,12 @@ fn select_account_header_at_block_returns_correct_header() { AccountUpdateDetails::Public(patch), ); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ) - .expect("upsert_accounts failed"); + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()) + .expect("upsert_accounts failed"); // Query the account header let (header, _storage_header) = - select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num) + select_account_header_with_storage_header_at_block(&db, account_id, block_num) .expect("Query should succeed") .expect("Header should exist"); @@ -331,14 +415,14 @@ fn select_account_header_at_block_returns_correct_header() { #[test] fn select_account_header_at_block_historical_query() { - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, _) = create_test_account_with_storage(); let account_id = account.id(); let block_num_1 = BlockNumber::from_epoch(0); let block_num_2 = BlockNumber::from_epoch(1); - insert_block_header(&mut conn, block_num_1); - insert_block_header(&mut conn, block_num_2); + insert_block_header(&db, block_num_1); + insert_block_header(&db, block_num_2); // Insert the account at block 1 let nonce_1 = account.nonce(); @@ -349,17 +433,12 @@ fn select_account_header_at_block_historical_query() { AccountUpdateDetails::Public(patch_1), ); - upsert_accounts( - &mut conn, - &[account_update_1], - block_num_1, - &PrecomputedPublicAccountStates::new(), - ) - .expect("First upsert failed"); + upsert_accounts(&db, &[account_update_1], block_num_1, &PrecomputedPublicAccountStates::new()) + .expect("First upsert failed"); // Query at block 1 - should return the account let (header_1, _) = - select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num_1) + select_account_header_with_storage_header_at_block(&db, account_id, block_num_1) .expect("Query should succeed") .expect("Header should exist at block 1"); @@ -367,7 +446,7 @@ fn select_account_header_at_block_historical_query() { // Query at block 2 - should return the same account (most recent before block 2) let (header_2, _) = - select_account_header_with_storage_header_at_block(&mut conn, account_id, block_num_2) + select_account_header_with_storage_header_at_block(&db, account_id, block_num_2) .expect("Query should succeed") .expect("Header should exist at block 2"); @@ -379,12 +458,12 @@ fn select_account_header_at_block_historical_query() { #[test] fn select_account_vault_at_block_empty() { - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, _) = create_test_account_with_storage(); let account_id = account.id(); let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); // Insert account without vault assets let patch = AccountPatch::try_from(account.clone()).unwrap(); @@ -394,17 +473,12 @@ fn select_account_vault_at_block_empty() { AccountUpdateDetails::Public(patch), ); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ) - .expect("upsert_accounts failed"); + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()) + .expect("upsert_accounts failed"); // Query vault - should return empty (the test account has no assets) - let assets = select_account_vault_at_block(&mut conn, account_id, block_num) - .expect("Query should succeed"); + let assets = + select_account_vault_at_block(&db, account_id, block_num).expect("Query should succeed"); assert!(assets.is_empty(), "Account should have no assets"); } @@ -414,12 +488,12 @@ fn select_account_vault_at_block_empty() { #[test] fn upsert_accounts_inserts_storage_header() { - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, account_id) = create_test_account_with_storage(); // Block 1 let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); let storage_commitment_original = account.storage().to_commitment(); let storage_slots_len = account.storage().slots().len(); @@ -436,18 +510,14 @@ fn upsert_accounts_inserts_storage_header() { ); // Upsert account - let result = upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ); + let result = + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()); assert!(result.is_ok(), "upsert_accounts failed: {:?}", result.err()); assert_eq!(result.unwrap(), 1, "Expected 1 account to be inserted"); // Query storage header back - let queried_storage = select_latest_account_storage(&mut conn, account_id) - .expect("Failed to query storage header"); + let queried_storage = + select_latest_account_storage(&db, account_id).expect("Failed to query storage header"); // Verify storage commitment matches assert_eq!( @@ -460,28 +530,26 @@ fn upsert_accounts_inserts_storage_header() { assert_eq!(queried_storage.slots().len(), storage_slots_len, "Storage slots count mismatch"); // Verify exactly 1 latest account with storage exists - let header_count: i64 = schema::accounts::table - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::valid_until.eq(VALID_FOREVER)) - .filter(schema::accounts::storage_header.is_not_null()) - .count() - .get_result(&mut conn) - .expect("Failed to count accounts with storage"); - - assert_eq!(header_count, 1, "Expected exactly 1 latest account with storage"); + let (_, latest_accounts) = count_account_rows(&db, account_id); + assert_eq!(latest_accounts, 1, "Expected exactly 1 latest account"); + assert_eq!( + latest_account_has_storage_header(&db, account_id), + Some(true), + "the latest account row must store a storage header" + ); } #[test] fn upsert_accounts_closes_previous_validity_interval() { - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, account_id) = create_test_account_with_storage(); // Block 1 and 2 let block_num_1 = BlockNumber::from_epoch(0); let block_num_2 = BlockNumber::from_epoch(1); - insert_block_header(&mut conn, block_num_1); - insert_block_header(&mut conn, block_num_2); + insert_block_header(&db, block_num_1); + insert_block_header(&db, block_num_2); // Save storage commitment before moving account let storage_commitment_1 = account.storage().to_commitment(); @@ -497,7 +565,7 @@ fn upsert_accounts_closes_previous_validity_interval() { AccountUpdateDetails::Public(patch_1), ); - upsert_accounts(&mut conn, &[account_update_1], block_num_1, &precomputed_1) + upsert_accounts(&db, &[account_update_1], block_num_1, &precomputed_1) .expect("First upsert failed"); // Create modified account with different storage value @@ -544,31 +612,17 @@ fn upsert_accounts_closes_previous_validity_interval() { AccountUpdateDetails::Public(patch_2), ); - upsert_accounts(&mut conn, &[account_update_2], block_num_2, &precomputed_2) + upsert_accounts(&db, &[account_update_2], block_num_2, &precomputed_2) .expect("Second upsert failed"); - // Verify 2 total account rows exist (both historical records) - let total_accounts: i64 = schema::accounts::table - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .count() - .get_result(&mut conn) - .expect("Failed to count total accounts"); - + // Both historical records must exist, but only one of them is open-ended (the latest). + let (total_accounts, latest_accounts) = count_account_rows(&db, account_id); assert_eq!(total_accounts, 2, "Expected 2 total account records"); - - // Verify only 1 is open-ended (latest) - let latest_accounts: i64 = schema::accounts::table - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::valid_until.eq(VALID_FOREVER)) - .count() - .get_result(&mut conn) - .expect("Failed to count latest accounts"); - assert_eq!(latest_accounts, 1, "Expected exactly 1 latest account"); // Verify latest storage matches second update - let latest_storage = select_latest_account_storage(&mut conn, account_id) - .expect("Failed to query latest storage"); + let latest_storage = + select_latest_account_storage(&db, account_id).expect("Failed to query latest storage"); assert_eq!( latest_storage.to_commitment(), @@ -577,9 +631,8 @@ fn upsert_accounts_closes_previous_validity_interval() { ); // Verify historical query returns first update - let storage_at_block_1 = - reconstruct_account_storage_at_block(&mut conn, account_id, block_num_1) - .expect("Failed to query storage at block 1"); + let storage_at_block_1 = reconstruct_account_storage_at_block(&db, account_id, block_num_1) + .expect("Failed to query storage at block 1"); assert_eq!( storage_at_block_1.to_commitment(), @@ -590,7 +643,7 @@ fn upsert_accounts_closes_previous_validity_interval() { #[test] fn upsert_accounts_with_multiple_storage_slots() { - let mut conn = setup_test_db(); + let db = TestDb::new(); // Create account with 3 storage slots let account_id = AccountId::dummy( @@ -632,7 +685,7 @@ fn upsert_accounts_with_multiple_storage_slots() { .unwrap(); let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); let storage_commitment = account.storage().to_commitment(); let account_commitment = account.to_commitment(); @@ -644,17 +697,12 @@ fn upsert_accounts_with_multiple_storage_slots() { AccountUpdateDetails::Public(patch), ); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ) - .expect("Upsert with multiple storage slots failed"); + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()) + .expect("Upsert with multiple storage slots failed"); // Query back and verify let queried_storage = - select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + select_latest_account_storage(&db, account_id).expect("Failed to query storage"); assert_eq!( queried_storage.to_commitment(), @@ -676,7 +724,7 @@ fn upsert_accounts_with_multiple_storage_slots() { #[test] fn upsert_accounts_with_empty_storage() { - let mut conn = setup_test_db(); + let db = TestDb::new(); // Create account with no component storage slots (only auth slot) let account_id = AccountId::dummy( @@ -708,7 +756,7 @@ fn upsert_accounts_with_empty_storage() { .unwrap(); let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); let storage_commitment = account.storage().to_commitment(); let account_commitment = account.to_commitment(); @@ -720,17 +768,12 @@ fn upsert_accounts_with_empty_storage() { AccountUpdateDetails::Public(patch), ); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ) - .expect("Upsert with empty storage failed"); + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()) + .expect("Upsert with empty storage failed"); // Query back and verify let queried_storage = - select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + select_latest_account_storage(&db, account_id).expect("Failed to query storage"); assert_eq!( queried_storage.to_commitment(), @@ -742,15 +785,7 @@ fn upsert_accounts_with_empty_storage() { assert_eq!(queried_storage.slots().len(), 2, "Expected 2 storage slots (auth component)"); // Verify the storage header blob exists in database - let storage_header_exists: Option = SelectDsl::select( - schema::accounts::table - .filter(schema::accounts::account_id.eq(account_id.to_bytes())) - .filter(schema::accounts::valid_until.eq(VALID_FOREVER)), - schema::accounts::storage_header.is_not_null(), - ) - .first(&mut conn) - .optional() - .expect("Failed to check storage header existence"); + let storage_header_exists = latest_account_has_storage_header(&db, account_id); assert_eq!( storage_header_exists, @@ -764,9 +799,9 @@ fn upsert_accounts_with_empty_storage() { #[test] fn select_latest_account_storage_ordering_semantics() { - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); let slot_name = StorageSlotName::mock(0); let key_1 = StorageMapKey::from_index(1); @@ -800,16 +835,10 @@ fn select_latest_account_storage_ordering_semantics() { AccountUpdateDetails::Public(patch), ); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ) - .expect("upsert_accounts failed"); + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()) + .expect("upsert_accounts failed"); - let storage = - select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + let storage = select_latest_account_storage(&db, account_id).expect("Failed to query storage"); let expected = BTreeMap::from_iter(entries); assert_storage_map_slot_entries(&storage, &slot_name, &expected); @@ -817,9 +846,9 @@ fn select_latest_account_storage_ordering_semantics() { #[test] fn select_latest_account_storage_multiple_slots() { - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_num = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); let slot_name_1 = StorageSlotName::mock(0); let slot_name_2 = StorageSlotName::mock(1); @@ -868,16 +897,10 @@ fn select_latest_account_storage_multiple_slots() { AccountUpdateDetails::Public(patch), ); - upsert_accounts( - &mut conn, - &[account_update], - block_num, - &PrecomputedPublicAccountStates::new(), - ) - .expect("upsert_accounts failed"); + upsert_accounts(&db, &[account_update], block_num, &PrecomputedPublicAccountStates::new()) + .expect("upsert_accounts failed"); - let storage = - select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + let storage = select_latest_account_storage(&db, account_id).expect("Failed to query storage"); let expected_slot_1 = BTreeMap::from_iter([(key_a, value_a)]); let expected_slot_2 = BTreeMap::from_iter([(key_b, value_b)]); @@ -888,11 +911,11 @@ fn select_latest_account_storage_multiple_slots() { #[test] fn select_latest_account_storage_slot_updates() { - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_1 = BlockNumber::from_epoch(0); let block_2 = BlockNumber::from_epoch(1); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); let slot_name = StorageSlotName::mock(0); let key_1 = StorageMapKey::from_index(1); @@ -913,7 +936,7 @@ fn select_latest_account_storage_slot_updates() { AccountUpdateDetails::Public(patch), ); - upsert_accounts(&mut conn, &[account_update], block_1, &PrecomputedPublicAccountStates::new()) + upsert_accounts(&db, &[account_update], block_1, &PrecomputedPublicAccountStates::new()) .expect("upsert_accounts failed"); let map_patch = StorageMapPatch::from_iters([], [(key_1, value_2), (key_2, value_3)]); @@ -944,11 +967,10 @@ fn select_latest_account_storage_slot_updates() { AccountUpdateDetails::Public(partial_patch), ); - upsert_accounts(&mut conn, &[account_update], block_2, &precomputed_public_states) + upsert_accounts(&db, &[account_update], block_2, &precomputed_public_states) .expect("upsert_accounts failed"); - let storage = - select_latest_account_storage(&mut conn, account_id).expect("Failed to query storage"); + let storage = select_latest_account_storage(&db, account_id).expect("Failed to query storage"); let expected = BTreeMap::from_iter([(key_1, value_2), (key_2, value_3)]); assert_storage_map_slot_entries(&storage, &slot_name, &expected); @@ -971,7 +993,7 @@ fn select_account_vault_at_block_historical_with_updates() { ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET_1, }; - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, _) = create_test_account_with_storage(); let account_id = account.id(); @@ -982,9 +1004,9 @@ fn select_account_vault_at_block_historical_with_updates() { let block_2 = BlockNumber::from_epoch(1); let block_3 = BlockNumber::from_epoch(2); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); - insert_block_header(&mut conn, block_3); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); + insert_block_header(&db, block_3); // Insert account at block 1 let patch = AccountPatch::try_from(account.clone()).unwrap(); @@ -996,7 +1018,7 @@ fn select_account_vault_at_block_historical_with_updates() { for block in [block_1, block_2, block_3] { upsert_accounts( - &mut conn, + &db, std::slice::from_ref(&account_update), block, &precomputed_states_from_account(&account), @@ -1008,35 +1030,35 @@ fn select_account_vault_at_block_historical_with_updates() { let asset_v1 = Asset::Fungible(FungibleAsset::new(faucet_id, 1000).unwrap()); let vault_key_1 = asset_v1.id(); - insert_account_vault_asset(&mut conn, account_id, block_1, vault_key_1, Some(asset_v1)) + insert_account_vault_asset(&db, account_id, block_1, vault_key_1, Some(asset_v1)) .expect("insert vault asset failed"); // Update vault asset at block 2: vault_key_1 = 2000 tokens (updated value) let asset_v2 = Asset::Fungible(FungibleAsset::new(faucet_id, 2000).unwrap()); - insert_account_vault_asset(&mut conn, account_id, block_2, vault_key_1, Some(asset_v2)) + insert_account_vault_asset(&db, account_id, block_2, vault_key_1, Some(asset_v2)) .expect("insert vault asset update failed"); // Add a second vault_key at block 2 (different faucet for different vault key) let faucet_id_2 = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET_1).unwrap(); let asset_key2 = Asset::Fungible(FungibleAsset::new(faucet_id_2, 500).unwrap()); let vault_key_2 = asset_key2.id(); - insert_account_vault_asset(&mut conn, account_id, block_2, vault_key_2, Some(asset_key2)) + insert_account_vault_asset(&db, account_id, block_2, vault_key_2, Some(asset_key2)) .expect("insert second vault asset failed"); // Update vault_key_1 again at block 3: vault_key_1 = 3000 tokens let asset_v3 = Asset::Fungible(FungibleAsset::new(faucet_id, 3000).unwrap()); - insert_account_vault_asset(&mut conn, account_id, block_3, vault_key_1, Some(asset_v3)) + insert_account_vault_asset(&db, account_id, block_3, vault_key_1, Some(asset_v3)) .expect("insert vault asset update 2 failed"); // Query at block 1: should only see vault_key_1 with 1000 tokens - let assets_at_block_1 = select_account_vault_at_block(&mut conn, account_id, block_1) + let assets_at_block_1 = select_account_vault_at_block(&db, account_id, block_1) .expect("Query at block 1 should succeed"); assert_eq!(assets_at_block_1.len(), 1, "Should have 1 asset at block 1"); assert_matches!(&assets_at_block_1[0], Asset::Fungible(f) if f.amount().as_u64() == 1000); // Query at block 2: should see vault_key_1 with 2000 tokens AND vault_key_2 with 500 tokens - let assets_at_block_2 = select_account_vault_at_block(&mut conn, account_id, block_2) + let assets_at_block_2 = select_account_vault_at_block(&db, account_id, block_2) .expect("Query at block 2 should succeed"); assert_eq!(assets_at_block_2.len(), 2, "Should have 2 assets at block 2"); @@ -1051,7 +1073,7 @@ fn select_account_vault_at_block_historical_with_updates() { assert!(amounts.contains(&500), "Block 2 should have vault_key_2 with 500 tokens"); // Query at block 3: should see vault_key_1 with 3000 tokens AND vault_key_2 with 500 tokens - let assets_at_block_3 = select_account_vault_at_block(&mut conn, account_id, block_3) + let assets_at_block_3 = select_account_vault_at_block(&db, account_id, block_3) .expect("Query at block 3 should succeed"); assert_eq!(assets_at_block_3.len(), 2, "Should have 2 assets at block 3"); @@ -1069,12 +1091,12 @@ fn select_account_vault_at_block_historical_with_updates() { /// without materializing the whole set. #[test] fn select_account_vault_at_block_bounds_read_to_limit() { - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, _) = create_test_account_with_storage(); let account_id = account.id(); let block_1 = BlockNumber::from_epoch(0); - insert_block_header(&mut conn, block_1); + insert_block_header(&db, block_1); let patch = AccountPatch::try_from(account.clone()).unwrap(); let account_update = BlockAccountUpdate::new( @@ -1083,7 +1105,7 @@ fn select_account_vault_at_block_bounds_read_to_limit() { AccountUpdateDetails::Public(patch), ); upsert_accounts( - &mut conn, + &db, std::slice::from_ref(&account_update), block_1, &PrecomputedPublicAccountStates::new(), @@ -1098,14 +1120,14 @@ fn select_account_vault_at_block_bounds_read_to_limit() { for i in 0..asset_count { let details = NonFungibleAssetDetails::new(faucet_id, vec![i as u8, (i >> 8) as u8]); let asset = Asset::NonFungible(NonFungibleAsset::new(&details)); - insert_account_vault_asset(&mut conn, account_id, block_1, asset.id(), Some(asset)) + insert_account_vault_asset(&db, account_id, block_1, asset.id(), Some(asset)) .expect("insert vault asset failed"); } // The query is capped at `MAX_RETURN_ENTRIES + 1` rows even though more assets exist, which is // enough for the caller to detect that the limit was exceeded. - let assets = select_account_vault_at_block(&mut conn, account_id, block_1) - .expect("query should succeed"); + let assets = + select_account_vault_at_block(&db, account_id, block_1).expect("query should succeed"); assert_eq!(assets.len(), AccountVaultDetails::MAX_RETURN_ENTRIES + 1); } @@ -1118,7 +1140,7 @@ fn select_account_vault_at_block_exponential_updates() { use miden_protocol::asset::{AssetId, FungibleAsset}; use miden_protocol::testing::account_id::ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET; - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, _) = create_test_account_with_storage(); let account_id = account.id(); @@ -1127,7 +1149,7 @@ fn select_account_vault_at_block_exponential_updates() { let blocks: Vec = (0..BLOCK_COUNT).map(BlockNumber::from).collect(); for block in &blocks { - insert_block_header(&mut conn, *block); + insert_block_header(&db, *block); } let patch = AccountPatch::try_from(account.clone()).unwrap(); @@ -1139,7 +1161,7 @@ fn select_account_vault_at_block_exponential_updates() { for block in &blocks { upsert_accounts( - &mut conn, + &db, std::slice::from_ref(&account_update), *block, &precomputed_states_from_account(&account), @@ -1152,12 +1174,12 @@ fn select_account_vault_at_block_exponential_updates() { for (index, block) in blocks.iter().enumerate() { let amount = 1u64 << index; let asset = Asset::Fungible(FungibleAsset::new(faucet_id, amount).unwrap()); - insert_account_vault_asset(&mut conn, account_id, *block, vault_key, Some(asset)) + insert_account_vault_asset(&db, account_id, *block, vault_key, Some(asset)) .expect("insert vault asset failed"); } for (index, block) in blocks.iter().enumerate() { - let assets_at_block = select_account_vault_at_block(&mut conn, account_id, *block) + let assets_at_block = select_account_vault_at_block(&db, account_id, *block) .expect("Query at block should succeed"); assert_eq!(assets_at_block.len(), 1, "Should have 1 asset at block"); @@ -1177,7 +1199,7 @@ fn select_account_vault_at_block_with_deletion() { use miden_protocol::asset::FungibleAsset; use miden_protocol::testing::account_id::ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET; - let mut conn = setup_test_db(); + let db = TestDb::new(); let (account, _) = create_test_account_with_storage(); let account_id = account.id(); @@ -1188,9 +1210,9 @@ fn select_account_vault_at_block_with_deletion() { let block_2 = BlockNumber::from_epoch(1); let block_3 = BlockNumber::from_epoch(2); - insert_block_header(&mut conn, block_1); - insert_block_header(&mut conn, block_2); - insert_block_header(&mut conn, block_3); + insert_block_header(&db, block_1); + insert_block_header(&db, block_2); + insert_block_header(&db, block_3); // Insert account at block 1 let patch = AccountPatch::try_from(account.clone()).unwrap(); @@ -1202,7 +1224,7 @@ fn select_account_vault_at_block_with_deletion() { for block in [block_1, block_2, block_3] { upsert_accounts( - &mut conn, + &db, std::slice::from_ref(&account_update), block, &precomputed_states_from_account(&account), @@ -1214,30 +1236,30 @@ fn select_account_vault_at_block_with_deletion() { let asset = Asset::Fungible(FungibleAsset::new(faucet_id, 1000).unwrap()); let vault_key = asset.id(); - insert_account_vault_asset(&mut conn, account_id, block_1, vault_key, Some(asset)) + insert_account_vault_asset(&db, account_id, block_1, vault_key, Some(asset)) .expect("insert vault asset failed"); // Delete the vault asset at block 2 (insert with asset = None) - insert_account_vault_asset(&mut conn, account_id, block_2, vault_key, None) + insert_account_vault_asset(&db, account_id, block_2, vault_key, None) .expect("delete vault asset failed"); // Re-add the vault asset at block 3 with different amount let asset_v3 = Asset::Fungible(FungibleAsset::new(faucet_id, 2000).unwrap()); - insert_account_vault_asset(&mut conn, account_id, block_3, vault_key, Some(asset_v3)) + insert_account_vault_asset(&db, account_id, block_3, vault_key, Some(asset_v3)) .expect("re-add vault asset failed"); // Query at block 1: should see the asset - let assets_at_block_1 = select_account_vault_at_block(&mut conn, account_id, block_1) + let assets_at_block_1 = select_account_vault_at_block(&db, account_id, block_1) .expect("Query at block 1 should succeed"); assert_eq!(assets_at_block_1.len(), 1, "Should have 1 asset at block 1"); // Query at block 2: should NOT see the asset (it was deleted) - let assets_at_block_2 = select_account_vault_at_block(&mut conn, account_id, block_2) + let assets_at_block_2 = select_account_vault_at_block(&db, account_id, block_2) .expect("Query at block 2 should succeed"); assert!(assets_at_block_2.is_empty(), "Should have no assets at block 2 (deleted)"); // Query at block 3: should see the re-added asset with new amount - let assets_at_block_3 = select_account_vault_at_block(&mut conn, account_id, block_3) + let assets_at_block_3 = select_account_vault_at_block(&db, account_id, block_3) .expect("Query at block 3 should succeed"); assert_eq!(assets_at_block_3.len(), 1, "Should have 1 asset at block 3"); assert_matches!(&assets_at_block_3[0], Asset::Fungible(f) if f.amount().as_u64() == 2000); @@ -1247,27 +1269,30 @@ fn select_account_vault_at_block_with_deletion() { // ================================================================================================ /// Counts the number of rows in `account_codes`. -fn count_account_codes(conn: &mut SqliteConnection) -> usize { - use schema::account_codes; - - let val = - SelectDsl::select(account_codes::table, diesel::dsl::count(account_codes::code_commitment)) - .get_result::(conn) - .expect("Failed to count account_codes"); - usize::try_from(u64::try_from(val).unwrap()).unwrap() -} +fn count_account_codes(db: &TestDb) -> usize { + const SQL: &str = "SELECT COUNT(*) FROM account_codes"; -/// Returns whether a specific code commitment exists in `account_codes`. -fn account_code_exists(conn: &mut SqliteConnection, code_commitment: Word) -> bool { - use schema::account_codes; + let count = db + .read::<_, DatabaseError, _>(|tx| { + Ok(tx.query(SQL, &[], |row| row.get::(0))?.into_iter().next().unwrap_or(0)) + }) + .expect("Failed to count account_codes"); - let n = - SelectDsl::select(account_codes::table, diesel::dsl::count(account_codes::code_commitment)) - .filter(account_codes::code_commitment.eq(code_commitment.to_bytes())) - .get_result::(conn) - .expect("Failed to query account_codes"); + usize::try_from(count).expect("row counts are non-negative") +} - n == 1 +/// Returns whether a specific code commitment exists in `account_codes`. +fn account_code_exists(db: &TestDb, code_commitment: Word) -> bool { + const SQL: &str = "SELECT EXISTS(SELECT 1 FROM account_codes WHERE code_commitment = ?1)"; + + db.read::<_, DatabaseError, _>(move |tx| { + Ok(tx + .query(SQL, &[&code_commitment], |row| row.get::(0))? + .into_iter() + .next() + .unwrap_or(false)) + }) + .expect("Failed to query account_codes") } /// Creates a full-state [`BlockAccountUpdate`] for the given account. @@ -1327,7 +1352,7 @@ fn build_account_with_code_seeded(push_value: u32, seed: [u8; 32]) -> Account { /// window, while the new (latest) code is retained. #[test] fn prune_account_code_retains_latest_after_code_change() { - let mut conn = setup_test_db(); + let db = TestDb::new(); // Block 0: account created with code A. // Block RETENTION+1 (=51): account updated to code B — within the retention window at prune @@ -1338,9 +1363,9 @@ fn prune_account_code_retains_latest_after_code_change() { let block_code_b = BlockNumber::from(HISTORICAL_BLOCK_RETENTION + 1); let block_prunable = BlockNumber::from(2 * HISTORICAL_BLOCK_RETENTION + 1); - insert_block_header(&mut conn, block_0); - insert_block_header(&mut conn, block_code_b); - insert_block_header(&mut conn, block_prunable); + insert_block_header(&db, block_0); + insert_block_header(&db, block_code_b); + insert_block_header(&db, block_prunable); let account_a = build_account_with_code(1); let account_b = build_account_with_code(2); @@ -1359,7 +1384,7 @@ fn prune_account_code_retains_latest_after_code_change() { // Block 0: insert account with code A. upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_a)], block_0, &precomputed_states_from_account(&account_a), @@ -1368,31 +1393,27 @@ fn prune_account_code_retains_latest_after_code_change() { // Block RETENTION+1: update the same account ID to code B via a full-state delta. upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_b)], block_code_b, &precomputed_states_from_account(&account_b), ) .expect("code-change upsert failed"); - assert_eq!(count_account_codes(&mut conn), 2, "both codes must exist before pruning"); + assert_eq!(count_account_codes(&db), 2, "both codes must exist before pruning"); // Advance past retention window and prune. cutoff = block_prunable - RETENTION = 2*RETENTION+1 // - RETENTION = RETENTION+1 = block_code_b - let (_, _, codes_deleted) = - prune_history(&mut conn, block_prunable).expect("prune_history failed"); + let (_, _, codes_deleted) = prune_history(&db, block_prunable).expect("prune_history failed"); // Only code A was dropped; code B is still referenced by the latest accounts row. assert_eq!(codes_deleted, 1, "exactly one code (A) must be pruned"); - assert!(!account_code_exists(&mut conn, code_commitment_a), "old code A must be pruned"); - assert!( - account_code_exists(&mut conn, code_commitment_b), - "current code B must be retained" - ); + assert!(!account_code_exists(&db, code_commitment_a), "old code A must be pruned"); + assert!(account_code_exists(&db, code_commitment_b), "current code B must be retained"); // Confirm the latest account row still points to code B. let (latest_header, _) = - select_account_header_with_storage_header_at_block(&mut conn, account_id, block_prunable) + select_account_header_with_storage_header_at_block(&db, account_id, block_prunable) .expect("query failed") .expect("account must still exist"); assert_eq!( @@ -1406,7 +1427,7 @@ fn prune_account_code_retains_latest_after_code_change() { /// code A must be retained because it is still the latest. #[test] fn prune_account_code_retains_revisited_code() { - let mut conn = setup_test_db(); + let db = TestDb::new(); // Block 0: code A. // Block RETENTION+1: code B (will be outside retention window at prune time). @@ -1420,10 +1441,10 @@ fn prune_account_code_retains_revisited_code() { let block_code_a_again = BlockNumber::from(HISTORICAL_BLOCK_RETENTION + 2); let block_prunable = BlockNumber::from(2 * HISTORICAL_BLOCK_RETENTION + 2); - insert_block_header(&mut conn, block_0); - insert_block_header(&mut conn, block_code_b); - insert_block_header(&mut conn, block_code_a_again); - insert_block_header(&mut conn, block_prunable); + insert_block_header(&db, block_0); + insert_block_header(&db, block_code_b); + insert_block_header(&db, block_code_a_again); + insert_block_header(&db, block_prunable); let account_a = build_account_with_code(1); let account_b = build_account_with_code(2); @@ -1441,7 +1462,7 @@ fn prune_account_code_retains_revisited_code() { // Block 0: code A. upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_a)], block_0, &precomputed_states_from_account(&account_a), @@ -1449,7 +1470,7 @@ fn prune_account_code_retains_revisited_code() { .expect("block 0 upsert failed"); // Block RETENTION+1: code B. upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_b)], block_code_b, &precomputed_states_from_account(&account_b), @@ -1457,7 +1478,7 @@ fn prune_account_code_retains_revisited_code() { .expect("block code_b upsert failed"); // Block RETENTION+2: back to code A. upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_a)], block_code_a_again, &precomputed_states_from_account(&account_a), @@ -1466,21 +1487,20 @@ fn prune_account_code_retains_revisited_code() { // Before pruning: both codes must be in account_codes (code A inserted once via ON CONFLICT DO // NOTHING, code B inserted once). - assert_eq!(count_account_codes(&mut conn), 2, "both codes must exist before pruning"); + assert_eq!(count_account_codes(&db), 2, "both codes must exist before pruning"); // Advance past retention window and prune. - let (_, _, codes_deleted) = - prune_history(&mut conn, block_prunable).expect("prune_history failed"); + let (_, _, codes_deleted) = prune_history(&db, block_prunable).expect("prune_history failed"); // Code B is no longer referenced by any account row within the retention window → pruned. Code // A is still referenced by the block_code_a_again accounts row (within cutoff) → retained. assert_eq!(codes_deleted, 1, "exactly one code (B) must be pruned"); - assert!(account_code_exists(&mut conn, code_commitment_a), "code A must be retained"); - assert!(!account_code_exists(&mut conn, code_commitment_b), "code B must be pruned"); + assert!(account_code_exists(&db, code_commitment_a), "code A must be retained"); + assert!(!account_code_exists(&db, code_commitment_b), "code B must be pruned"); // Confirm the latest account row still points to code A. let (latest_header, _) = - select_account_header_with_storage_header_at_block(&mut conn, account_id, block_prunable) + select_account_header_with_storage_header_at_block(&db, account_id, block_prunable) .expect("query failed") .expect("account must still exist"); assert_eq!( @@ -1496,7 +1516,7 @@ fn prune_account_code_retains_revisited_code() { /// code becomes prunable. #[test] fn prune_account_code_retains_baseline_code() { - let mut conn = setup_test_db(); + let db = TestDb::new(); // Block 0: code A. // Block 2*RETENTION: code B. @@ -1510,10 +1530,10 @@ fn prune_account_code_retains_baseline_code() { let block_first_prune = BlockNumber::from(2 * HISTORICAL_BLOCK_RETENTION + 1); let block_second_prune = BlockNumber::from(3 * HISTORICAL_BLOCK_RETENTION + 1); - insert_block_header(&mut conn, block_0); - insert_block_header(&mut conn, block_code_b); - insert_block_header(&mut conn, block_first_prune); - insert_block_header(&mut conn, block_second_prune); + insert_block_header(&db, block_0); + insert_block_header(&db, block_code_b); + insert_block_header(&db, block_first_prune); + insert_block_header(&db, block_second_prune); let account_a = build_account_with_code(1); let account_b = build_account_with_code(2); @@ -1525,7 +1545,7 @@ fn prune_account_code_retains_baseline_code() { // Block 0: code A. upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_a)], block_0, &precomputed_states_from_account(&account_a), @@ -1533,47 +1553,40 @@ fn prune_account_code_retains_baseline_code() { .expect("block 0 upsert failed"); // Block 2*RETENTION: code B. upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_b)], block_code_b, &precomputed_states_from_account(&account_b), ) .expect("code-change upsert failed"); - assert_eq!(count_account_codes(&mut conn), 2, "both codes must exist before pruning"); + assert_eq!(count_account_codes(&db), 2, "both codes must exist before pruning"); // First prune: the block-0 row is the baseline (its successor is above the cutoff), so code A // must survive. let (_, _, codes_deleted) = - prune_history(&mut conn, block_first_prune).expect("prune_history failed"); + prune_history(&db, block_first_prune).expect("prune_history failed"); assert_eq!(codes_deleted, 0, "no code may be pruned while code A backs the baseline row"); - assert!( - account_code_exists(&mut conn, code_commitment_a), - "baseline code A must be retained" - ); - assert!( - account_code_exists(&mut conn, code_commitment_b), - "current code B must be retained" - ); + assert!(account_code_exists(&db, code_commitment_a), "baseline code A must be retained"); + assert!(account_code_exists(&db, code_commitment_b), "current code B must be retained"); // Second prune: the code-B row is now at or below the cutoff and supersedes the block-0 row, so // code A is no longer reachable from any in-window read. let (_, _, codes_deleted) = - prune_history(&mut conn, block_second_prune).expect("prune_history failed"); + prune_history(&db, block_second_prune).expect("prune_history failed"); assert_eq!(codes_deleted, 1, "exactly one code (A) must be pruned"); - assert!(!account_code_exists(&mut conn, code_commitment_a), "old code A must be pruned"); - assert!( - account_code_exists(&mut conn, code_commitment_b), - "current code B must be retained" - ); + assert!(!account_code_exists(&db, code_commitment_a), "old code A must be pruned"); + assert!(account_code_exists(&db, code_commitment_b), "current code B must be retained"); } /// Returns the cutoff recorded in `prune_progress`, if any. -fn codes_prune_cutoff(conn: &mut SqliteConnection) -> Option { - SelectDsl::select(schema::prune_progress::table, schema::prune_progress::codes_cutoff) - .first(conn) - .optional() - .expect("Failed to query prune_progress") +fn codes_prune_cutoff(db: &TestDb) -> Option { + const SQL: &str = "SELECT codes_cutoff FROM prune_progress"; + + db.read::<_, DatabaseError, _>(|tx| { + Ok(tx.query(SQL, &[], |row| row.get::(0))?.into_iter().next()) + }) + .expect("Failed to query prune_progress") } /// Prune test 5: the incremental (windowed) codes prune must not delete a code whose expiring @@ -1581,7 +1594,7 @@ fn codes_prune_cutoff(conn: &mut SqliteConnection) -> Option { /// it once the last reference expires in a later window. #[test] fn prune_account_code_incremental_cross_account_reference() { - let mut conn = setup_test_db(); + let db = TestDb::new(); // The "switcher" account changes code first; the "holdout" account keeps code A pinned. // Both accounts are created with code A at block 0. @@ -1600,7 +1613,7 @@ fn prune_account_code_incremental_cross_account_reference() { let block_third_prune = BlockNumber::from(4 * HISTORICAL_BLOCK_RETENTION + 3); for block in [block_0, block_switcher_to_b, block_holdout_to_b] { - insert_block_header(&mut conn, block); + insert_block_header(&db, block); } let switcher_on_a = build_account_with_code(1); @@ -1619,7 +1632,7 @@ fn prune_account_code_incremental_cross_account_reference() { for account in [&switcher_on_a, &holdout_on_a] { upsert_accounts( - &mut conn, + &db, &[make_full_state_update(account)], block_0, &precomputed_states_from_account(account), @@ -1628,11 +1641,11 @@ fn prune_account_code_incremental_cross_account_reference() { } let (_, _, codes_deleted) = - prune_history(&mut conn, block_first_prune).expect("prune_history failed"); + prune_history(&db, block_first_prune).expect("prune_history failed"); assert_eq!(codes_deleted, 0, "no code is collectable while both accounts run code A"); upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&switcher_on_b)], block_switcher_to_b, &precomputed_states_from_account(&switcher_on_b), @@ -1640,15 +1653,15 @@ fn prune_account_code_incremental_cross_account_reference() { .expect("switcher code-change upsert failed"); let (_, _, codes_deleted) = - prune_history(&mut conn, block_second_prune).expect("prune_history failed"); + prune_history(&db, block_second_prune).expect("prune_history failed"); assert_eq!(codes_deleted, 0, "code A must survive while the holdout still references it"); assert!( - account_code_exists(&mut conn, code_commitment_a), + account_code_exists(&db, code_commitment_a), "code A must be retained while the holdout references it" ); upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&holdout_on_b)], block_holdout_to_b, &precomputed_states_from_account(&holdout_on_b), @@ -1656,20 +1669,17 @@ fn prune_account_code_incremental_cross_account_reference() { .expect("holdout code-change upsert failed"); let (_, _, codes_deleted) = - prune_history(&mut conn, block_third_prune).expect("prune_history failed"); + prune_history(&db, block_third_prune).expect("prune_history failed"); assert_eq!(codes_deleted, 1, "exactly one code (A) must be pruned"); - assert!(!account_code_exists(&mut conn, code_commitment_a), "code A must be pruned"); - assert!( - account_code_exists(&mut conn, code_commitment_b), - "current code B must be retained" - ); + assert!(!account_code_exists(&db, code_commitment_a), "code A must be pruned"); + assert!(account_code_exists(&db, code_commitment_b), "current code B must be retained"); } /// Prune test 6: `prune_progress` records the cutoff of the last codes prune; re-pruning at the /// same or a lower cutoff deletes nothing and never moves the marker backwards. #[test] fn prune_account_codes_marker_never_regresses() { - let mut conn = setup_test_db(); + let db = TestDb::new(); // Same shape as prune test 2: code A at block 0 is superseded by code B at block R+1, so a // prune at tip 2R+1 (cutoff R+1) collects code A. @@ -1677,59 +1687,54 @@ fn prune_account_codes_marker_never_regresses() { let block_code_b = BlockNumber::from(HISTORICAL_BLOCK_RETENTION + 1); let block_prune = BlockNumber::from(2 * HISTORICAL_BLOCK_RETENTION + 1); - insert_block_header(&mut conn, block_0); - insert_block_header(&mut conn, block_code_b); + insert_block_header(&db, block_0); + insert_block_header(&db, block_code_b); let account_a = build_account_with_code(1); let account_b = build_account_with_code(2); upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_a)], block_0, &precomputed_states_from_account(&account_a), ) .expect("block 0 upsert failed"); upsert_accounts( - &mut conn, + &db, &[make_full_state_update(&account_b)], block_code_b, &precomputed_states_from_account(&account_b), ) .expect("code-change upsert failed"); - assert_eq!(codes_prune_cutoff(&mut conn), None, "no marker before the first prune"); + assert_eq!(codes_prune_cutoff(&db), None, "no marker before the first prune"); let cutoff = i64::from(HISTORICAL_BLOCK_RETENTION + 1); - let (_, _, codes_deleted) = prune_history(&mut conn, block_prune).expect("first prune failed"); + let (_, _, codes_deleted) = prune_history(&db, block_prune).expect("first prune failed"); assert_eq!(codes_deleted, 1, "exactly one code (A) must be pruned"); - assert_eq!(codes_prune_cutoff(&mut conn), Some(cutoff), "marker must record the cutoff"); + assert_eq!(codes_prune_cutoff(&db), Some(cutoff), "marker must record the cutoff"); // Re-pruning at the same tip is a no-op. - let (_, _, codes_deleted) = prune_history(&mut conn, block_prune).expect("second prune failed"); + let (_, _, codes_deleted) = prune_history(&db, block_prune).expect("second prune failed"); assert_eq!(codes_deleted, 0, "re-pruning at the same cutoff must delete nothing"); - assert_eq!(codes_prune_cutoff(&mut conn), Some(cutoff), "marker must be unchanged"); + assert_eq!(codes_prune_cutoff(&db), Some(cutoff), "marker must be unchanged"); // Pruning at a lower tip (cutoff 0) must not move the marker backwards. - let (_, _, codes_deleted) = - prune_history(&mut conn, BlockNumber::from(HISTORICAL_BLOCK_RETENTION)) - .expect("stale prune failed"); + let (_, _, codes_deleted) = prune_history(&db, BlockNumber::from(HISTORICAL_BLOCK_RETENTION)) + .expect("stale prune failed"); assert_eq!(codes_deleted, 0, "pruning below the marker must delete nothing"); - assert_eq!(codes_prune_cutoff(&mut conn), Some(cutoff), "marker must never regress"); + assert_eq!(codes_prune_cutoff(&db), Some(cutoff), "marker must never regress"); } #[test] #[miden_node_test_macro::enable_logging] fn network_accounts_subset_classifies_correctly() { - use crate::db::models::queries::accounts::{ - AccountRowInsert, - NetworkAccountType, - select_network_accounts_subset, - }; + use crate::db::queries::{AccountRow, NetworkAccountType}; - let mut conn = setup_test_db(); + let db = TestDb::new(); let block_num = BlockNumber::from(1); - insert_block_header(&mut conn, block_num); + insert_block_header(&db, block_num); // Three accounts with distinct classifications. AccountIds are dummies — the queries only care // about the (account_id, network_account_type, valid_until) tuple, not protocol-level validity. @@ -1763,22 +1768,21 @@ fn network_accounts_subset_classifies_correctly() { (public_id, NetworkAccountType::None), (private_id, NetworkAccountType::None), ] { - let row = AccountRowInsert::new_private(id, ty, Word::default(), block_num, block_num); - diesel::insert_into(crate::db::schema::accounts::table) - .values(&row) - .execute(&mut conn) - .unwrap(); + db.write::<_, DatabaseError, _>(move |tx| { + AccountRow::new_private(id, ty, Word::default(), block_num, block_num).upsert(tx) + }) + .unwrap(); } // Batched lookup returns only the network-classified id; public, private, and unknown ids are // all omitted. let subset = - select_network_accounts_subset(&mut conn, &[network_id, public_id, private_id, unknown_id]) + select_network_accounts_subset(&db, &[network_id, public_id, private_id, unknown_id]) .unwrap(); assert_eq!(subset.len(), 1); assert!(subset.contains(&network_id)); // Empty input slice short-circuits to an empty result. - let empty = select_network_accounts_subset(&mut conn, &[]).unwrap(); + let empty = select_network_accounts_subset(&db, &[]).unwrap(); assert!(empty.is_empty()); } diff --git a/crates/store/src/db/queries/upsert_accounts/upsert_account.sql b/crates/store/src/db/queries/upsert_accounts/upsert_account.sql new file mode 100644 index 0000000000..8eb19adf22 --- /dev/null +++ b/crates/store/src/db/queries/upsert_accounts/upsert_account.sql @@ -0,0 +1,27 @@ +-- Writes an account's state at `block_num` as its current, open-ended version. +-- +-- Re-applying the same block overwrites that block's row rather than failing, so an interrupted +-- block application can be replayed. The key columns are excluded from the update: they are what +-- the conflict matched on. +INSERT INTO accounts ( + account_id, + network_account_type, + block_num, + account_commitment, + code_commitment, + nonce, + storage_header, + vault_root, + created_at_block, + valid_until +) +VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10) +ON CONFLICT(account_id, block_num) DO UPDATE SET + network_account_type = excluded.network_account_type, + account_commitment = excluded.account_commitment, + code_commitment = excluded.code_commitment, + nonce = excluded.nonce, + storage_header = excluded.storage_header, + vault_root = excluded.vault_root, + created_at_block = excluded.created_at_block, + valid_until = excluded.valid_until diff --git a/crates/store/src/db/test_db.rs b/crates/store/src/db/test_db.rs new file mode 100644 index 0000000000..590238c4de --- /dev/null +++ b/crates/store/src/db/test_db.rs @@ -0,0 +1,86 @@ +//! A blocking handle over the framework's connection pools, for testing query functions. +//! +//! Query functions take a [`ReadTx`]/[`WriteTx`], which the pools only ever hand out inside a +//! `read`/`write` closure on an async call. That is the right shape for production code, but it +//! would force every query-level test to be `async`. [`TestDb`] owns a current-thread runtime and +//! blocks on those calls, so the tests stay plain `#[test]` functions while still exercising the +//! same pools, PRAGMAs, and transaction behaviour the node uses. + +use std::path::Path; + +use miden_node_db::sqlite::{DbReader, DbWriter, ReadTx, WriteTx}; +use miden_node_db::{DatabaseError, default_connection_pool_size}; +use tokio::runtime::Runtime; + +use crate::db::migrations::bootstrap_database; + +/// A database with framework handles over it, driven synchronously. +pub(crate) struct TestDb { + // Held as `Option` so [`Drop`] can drop the pools inside the runtime's context: their pooled + // connections are closed on a blocking task, which panics without a runtime to spawn it on. + writer: Option, + reader: Option, + runtime: Runtime, +} + +impl TestDb { + /// Bootstraps a throwaway database in the OS temp directory and opens handles over it. + /// + /// The temporary directory is intentionally leaked so the file outlives the handle; these are + /// test databases in the OS temp directory. + pub(crate) fn new() -> Self { + let temp_dir = tempfile::tempdir().expect("failed to create temp directory"); + let path = temp_dir.path().join("test.sqlite3"); + bootstrap_database(&path).expect("database should bootstrap"); + let _kept_dir = temp_dir.keep(); + + Self::open(&path) + } + + /// Opens handles over an existing, already migrated database file. + pub(crate) fn open(path: &Path) -> Self { + let (writer, reader) = + miden_node_db::sqlite::open_with_pool_size(path, default_connection_pool_size()) + .expect("temp file sqlite should always work"); + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("test runtime should build"); + + Self { + writer: Some(writer), + reader: Some(reader), + runtime, + } + } + + /// Runs `query` inside a read-only transaction. + pub(crate) fn read(&self, query: F) -> Result + where + F: FnOnce(&ReadTx<'_>) -> Result + Send + 'static, + R: Send + 'static, + E: From + Send + 'static, + { + let reader = self.reader.as_ref().expect("handles live until drop"); + self.runtime.block_on(reader.read("test read", query)) + } + + /// Runs `query` inside a read-write transaction, committing it if `query` returns `Ok`. + pub(crate) fn write(&self, query: F) -> Result + where + F: FnOnce(&WriteTx<'_>) -> Result + Send + 'static, + R: Send + 'static, + E: From + Send + 'static, + { + let writer = self.writer.as_ref().expect("handles live until drop"); + self.runtime.block_on(writer.write("test write", query)) + } +} + +impl Drop for TestDb { + fn drop(&mut self) { + let _guard = self.runtime.enter(); + self.writer.take(); + self.reader.take(); + } +} diff --git a/crates/store/src/db/tests.rs b/crates/store/src/db/tests.rs index 9dad5cc135..bda3b6c603 100644 --- a/crates/store/src/db/tests.rs +++ b/crates/store/src/db/tests.rs @@ -1,7 +1,8 @@ +use std::ops::RangeInclusive; use std::sync::{Arc, Mutex}; use assert_matches::assert_matches; -use diesel::{Connection, SqliteConnection}; +use miden_node_db::sqlite::WriteTx; use miden_node_proto::domain::account::{AccountSummary, StorageMapEntries}; use miden_node_utils::fee::test_fee_params; use miden_protocol::account::auth::{AuthScheme, PublicKeyCommitment}; @@ -14,6 +15,7 @@ use miden_protocol::account::{ AccountId, AccountIdVersion, AccountPatch, + AccountStorage, AccountStoragePatch, AccountType, AccountUpdateDetails, @@ -26,7 +28,7 @@ use miden_protocol::account::{ StorageSlotName, StorageSlotPatch, }; -use miden_protocol::asset::{Asset, FungibleAsset}; +use miden_protocol::asset::{Asset, AssetId, FungibleAsset}; use miden_protocol::block::{ BlockAccountUpdate, BlockHeader, @@ -50,6 +52,7 @@ use miden_protocol::note::{ NoteHeader, NoteId, NoteMetadata, + NoteScript, NoteTag, NoteType, Nullifier, @@ -88,20 +91,243 @@ use crate::account_state_forest::{ HISTORICAL_BLOCK_RETENTION, TestAccountStateForestExt, }; -use crate::db::models::queries::{ +use crate::db::queries::{ + self, + NOTE_SYNC_BLOCK_OVERHEAD_BYTES, + NOTE_SYNC_RECORD_BYTES, PrecomputedPublicAccountState, PrecomputedPublicAccountStates, StorageMapValue, - insert_account_storage_map_value, + StorageMapValuesPage, }; -use crate::db::models::{queries, utils}; -use crate::errors::DatabaseError; +use crate::db::{AccountVaultValue, BlockHeaderCommitment, NoteSyncUpdate, Result, TestDb, utils}; +use crate::errors::{DatabaseError, NoteSyncError}; -fn create_db() -> SqliteConnection { - crate::db::migrations::test_connection() +// QUERY DRIVERS +// ================================================================================================ +// +// Each driver runs one query function on the test database, so a test body reads the same as the +// production call site with the transaction handle replaced by the test handle. + +fn insert_block_header( + db: &TestDb, + header: &BlockHeader, + signatures: &BlockSignatures, +) -> Result { + let header = header.clone(); + let signatures = signatures.clone(); + db.write(move |tx| queries::insert_block_header(tx, &header, &signatures)) +} + +fn insert_notes(db: &TestDb, notes: &[(NoteRecord, Option)]) -> Result { + let notes = notes.to_vec(); + db.write(move |tx| queries::insert_notes(tx, ¬es)) +} + +fn insert_note_scripts(db: &TestDb, notes: &[NoteRecord]) -> Result { + let notes = notes.to_vec(); + db.write(move |tx| queries::insert_note_scripts(tx, notes.iter())) +} + +fn insert_nullifiers_for_block( + db: &TestDb, + nullifiers: &[Nullifier], + block_num: BlockNumber, +) -> Result { + let nullifiers = nullifiers.to_vec(); + db.write(move |tx| queries::insert_nullifiers_for_block(tx, &nullifiers, block_num)) +} + +fn insert_transactions( + db: &TestDb, + block_num: BlockNumber, + transactions: &OrderedTransactionHeaders, +) -> Result { + let transactions = transactions.clone(); + db.write(move |tx| queries::insert_transactions(tx, block_num, &transactions)) +} + +fn upsert_accounts( + db: &TestDb, + accounts: &[BlockAccountUpdate], + block_num: BlockNumber, + precomputed_public_states: &PrecomputedPublicAccountStates, +) -> Result { + let accounts = accounts.to_vec(); + let precomputed_public_states = precomputed_public_states.clone(); + db.write(move |tx| { + queries::upsert_accounts(tx, &accounts, block_num, &precomputed_public_states) + }) +} + +fn insert_account_storage_map_value( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, + slot_name: StorageSlotName, + key: StorageMapKey, + value: Word, +) -> Result { + db.write(move |tx| { + queries::insert_account_storage_map_value(tx, account_id, block_num, &slot_name, key, value) + }) +} + +fn insert_account_vault_asset( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, + vault_key: AssetId, + asset: Option, +) -> Result { + db.write(move |tx| { + queries::insert_account_vault_asset(tx, account_id, block_num, vault_key, asset) + }) +} + +fn prune_history(db: &TestDb, chain_tip: BlockNumber) -> Result<(usize, usize, usize)> { + db.write(move |tx| queries::prune_history(tx, chain_tip)) +} + +fn select_all_nullifiers(db: &TestDb) -> Result> { + db.read(queries::select_all_nullifiers) +} + +fn select_nullifiers_by_prefix( + db: &TestDb, + prefix_len: u8, + nullifier_prefixes: &[u16], + block_range: RangeInclusive, +) -> Result<(Vec, BlockNumber)> { + let nullifier_prefixes = nullifier_prefixes.to_vec(); + db.read(move |tx| { + queries::select_nullifiers_by_prefix(tx, prefix_len, &nullifier_prefixes, block_range) + }) +} + +fn select_notes_since_block_by_tag( + db: &TestDb, + note_tags: &[u32], + block_range: RangeInclusive, +) -> Result> { + let note_tags = note_tags.to_vec(); + db.read(move |tx| queries::select_notes_since_block_by_tag(tx, ¬e_tags, block_range)) +} + +fn select_notes_by_id(db: &TestDb, note_ids: &[NoteId]) -> Result> { + let note_ids = note_ids.to_vec(); + db.read(move |tx| queries::select_notes_by_id(tx, ¬e_ids)) +} + +fn select_note_script_by_root(db: &TestDb, root: Word) -> Result> { + db.read(move |tx| queries::select_note_script_by_root(tx, root)) +} + +fn get_note_sync_multi( + db: &TestDb, + note_tags: &[u32], + block_range: RangeInclusive, + max_response_payload_bytes: usize, +) -> std::result::Result, NoteSyncError> { + let note_tags = note_tags.to_vec(); + db.read(move |tx| { + queries::get_note_sync_multi(tx, ¬e_tags, block_range, max_response_payload_bytes) + }) +} + +fn select_block_header_by_block_num( + db: &TestDb, + maybe_block_num: Option, +) -> Result> { + db.read(move |tx| queries::select_block_header_by_block_num(tx, maybe_block_num)) +} + +fn select_block_header_and_signatures_by_block_num( + db: &TestDb, + block_num: BlockNumber, +) -> Result> { + db.read(move |tx| queries::select_block_header_and_signatures_by_block_num(tx, block_num)) +} + +fn select_block_headers(db: &TestDb, blocks: Vec) -> Result> { + db.read(move |tx| queries::select_block_headers(tx, blocks.into_iter())) +} + +fn select_all_block_header_commitments(db: &TestDb) -> Result> { + db.read(queries::select_all_block_header_commitments) +} + +fn select_account(db: &TestDb, account_id: AccountId) -> Result { + db.read(move |tx| queries::select_account(tx, account_id)) +} + +fn select_all_accounts(db: &TestDb) -> Result> { + db.read(queries::select_all_accounts) +} + +fn select_account_code_by_commitment( + db: &TestDb, + code_commitment: Word, +) -> Result>> { + db.read(move |tx| queries::select_account_code_by_commitment(tx, code_commitment)) +} + +fn select_latest_account_storage(db: &TestDb, account_id: AccountId) -> Result { + db.read(move |tx| queries::select_latest_account_storage(tx, account_id)) +} + +fn select_account_storage_map_values_paged( + db: &TestDb, + account_id: AccountId, + block_range: RangeInclusive, + limit: usize, +) -> Result { + db.read(move |tx| { + queries::select_account_storage_map_values_paged(tx, account_id, block_range, limit) + }) +} + +fn select_account_vault_assets( + db: &TestDb, + account_id: AccountId, + block_range: RangeInclusive, +) -> Result<(BlockNumber, Vec)> { + db.read(move |tx| queries::select_account_vault_assets(tx, account_id, block_range)) +} + +fn select_account_vault_at_block( + db: &TestDb, + account_id: AccountId, + block_num: BlockNumber, +) -> Result> { + db.read(move |tx| queries::select_account_vault_at_block(tx, account_id, block_num)) } -fn create_block(conn: &mut SqliteConnection, block_num: BlockNumber) { +fn select_transactions_records( + db: &TestDb, + account_ids: &[AccountId], + block_range: RangeInclusive, +) -> Result<(BlockNumber, Vec)> { + let account_ids = account_ids.to_vec(); + db.read(move |tx| queries::select_transactions_records(tx, &account_ids, block_range)) +} + +// TEST HELPERS +// ================================================================================================ + +fn create_block(db: &TestDb, block_num: BlockNumber) { + let (block_header, signatures) = mock_block(block_num); + insert_block_header(db, &block_header, &signatures).unwrap(); +} + +/// [`create_block`] for tests that already hold a write transaction. +fn create_block_in(tx: &WriteTx<'_>, block_num: BlockNumber) -> Result<()> { + let (block_header, signatures) = mock_block(block_num); + queries::insert_block_header(tx, &block_header, &signatures)?; + Ok(()) +} + +fn mock_block(block_num: BlockNumber) -> (BlockHeader, BlockSignatures) { let block_header = BlockHeader::new( 1_u8.into(), num_to_word(2), @@ -120,11 +346,7 @@ fn create_block(conn: &mut SqliteConnection, block_num: BlockNumber) { let dummy_signature = BlockSignatures::new(vec![SigningKey::new().sign(block_header.commitment())]).unwrap(); - conn.transaction(|conn| { - queries::insert_block_header(conn, &block_header, &dummy_signature)?; - Ok::<_, DatabaseError>(()) - }) - .unwrap(); + (block_header, dummy_signature) } fn precomputed_states_from_account(account: &Account) -> PrecomputedPublicAccountStates { @@ -147,32 +369,27 @@ fn precomputed_states_from_account(account: &Account) -> PrecomputedPublicAccoun #[test] #[miden_node_test_macro::enable_logging] fn sql_insert_nullifiers_for_block() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let nullifiers = [num_to_nullifier(1 << 48)]; let block_num = 1.into(); - create_block(conn, block_num); + create_block(db, block_num); // Insert a new nullifier succeeds { - conn.transaction(|conn| { - let res = queries::insert_nullifiers_for_block(conn, &nullifiers, block_num); - assert_eq!(res.unwrap(), nullifiers.len(), "There should be one entry"); - Ok::<_, DatabaseError>(()) - }) - .unwrap(); + let res = insert_nullifiers_for_block(db, &nullifiers, block_num); + assert_eq!(res.unwrap(), nullifiers.len(), "There should be one entry"); } // Inserting the nullifier twice is an error { - let res = queries::insert_nullifiers_for_block(conn, &nullifiers, block_num); + let res = insert_nullifiers_for_block(db, &nullifiers, block_num); assert!(res.is_err(), "Inserting the same nullifier twice is an error"); } // even if the block number is different { - let res = queries::insert_nullifiers_for_block(conn, &nullifiers, block_num + 1); + let res = insert_nullifiers_for_block(db, &nullifiers, block_num + 1); assert!( res.is_err(), @@ -185,7 +402,7 @@ fn sql_insert_nullifiers_for_block() { let nullifiers: Vec<_> = (0..10).map(num_to_nullifier).collect(); let block_num = 1.into(); - let res = queries::insert_nullifiers_for_block(conn, &nullifiers, block_num); + let res = insert_nullifiers_for_block(db, &nullifiers, block_num); assert_eq!(res.unwrap(), nullifiers.len(), "There should be 10 entries"); } @@ -194,9 +411,8 @@ fn sql_insert_nullifiers_for_block() { #[test] #[miden_node_test_macro::enable_logging] fn sql_insert_transactions() { - let mut conn = create_db(); - let conn = &mut conn; - let count = insert_transactions(conn); + let db = &TestDb::new(); + let count = insert_mock_transactions(db); assert_eq!(count, 2, "Two elements must have been inserted"); } @@ -204,13 +420,12 @@ fn sql_insert_transactions() { #[test] #[miden_node_test_macro::enable_logging] fn sql_select_nullifiers() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let block_num = 1.into(); - create_block(conn, block_num); + create_block(db, block_num); // test querying empty table - let nullifiers = queries::select_all_nullifiers(conn).unwrap(); + let nullifiers = select_all_nullifiers(db).unwrap(); assert!(nullifiers.is_empty()); // test multiple entries @@ -219,10 +434,10 @@ fn sql_select_nullifiers() { let nullifier = num_to_nullifier(i); state.push(NullifierInfo { nullifier, block_num }); - let res = queries::insert_nullifiers_for_block(conn, &[nullifier], block_num); + let res = insert_nullifiers_for_block(db, &[nullifier], block_num); assert_eq!(res.unwrap(), 1, "One element must have been inserted"); - let nullifiers = queries::select_all_nullifiers(conn).unwrap(); + let nullifiers = select_all_nullifiers(db).unwrap(); assert_eq!(nullifiers, state); } } @@ -248,18 +463,17 @@ pub fn create_note(account_id: AccountId) -> Note { #[test] #[miden_node_test_macro::enable_logging] fn sql_select_note_script_by_root() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(conn, block_num); + create_block(db, block_num); let account_id = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); - queries::upsert_accounts( - conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -278,57 +492,51 @@ fn sql_select_note_script_by_root() { }; state.push(note.clone()); - let res = queries::insert_scripts(conn, [¬e]); + let res = insert_note_scripts(db, std::slice::from_ref(¬e)); assert_eq!(res.unwrap(), 1, "One element must have been inserted"); // test querying the script by the root - let note_script = - queries::select_note_script_by_root(conn, Word::from(new_note.script().root())).unwrap(); + let note_script = select_note_script_by_root(db, Word::from(new_note.script().root())).unwrap(); assert_eq!(note_script, Some(new_note.script().clone())); // test querying the script by the root that is not in the database - let note_script = queries::select_note_script_by_root(conn, [0_u16; 4].into()).unwrap(); + let note_script = select_note_script_by_root(db, [0_u16; 4].into()).unwrap(); assert_eq!(note_script, None); } // Generates an account, inserts into the database, and creates a note for it. fn make_account_and_note( - conn: &mut SqliteConnection, + db: &TestDb, block_num: BlockNumber, init_seed: [u8; 32], account_type: AccountType, ) -> (AccountId, Note) { - conn.transaction(|conn| { - let account = mock_account_code_and_storage(account_type, [], Some(init_seed)); - let account_id = account.id(); - queries::upsert_accounts( - conn, - &[BlockAccountUpdate::new( - account_id, - account.to_commitment(), - AccountUpdateDetails::Public(AccountPatch::try_from(account.clone()).unwrap()), - )], - block_num, - &precomputed_states_from_account(&account), - ) - .unwrap(); + let account = mock_account_code_and_storage(account_type, [], Some(init_seed)); + let account_id = account.id(); + upsert_accounts( + db, + &[BlockAccountUpdate::new( + account_id, + account.to_commitment(), + AccountUpdateDetails::Public(AccountPatch::try_from(account.clone()).unwrap()), + )], + block_num, + &precomputed_states_from_account(&account), + ) + .unwrap(); - let new_note = create_note(account_id); - Ok::<_, DatabaseError>((account_id, new_note)) - }) - .unwrap() + (account_id, create_note(account_id)) } #[test] #[miden_node_test_macro::enable_logging] fn sql_select_accounts() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let block_num = 1.into(); - create_block(conn, block_num); + create_block(db, block_num); // test querying empty table - let accounts = queries::select_all_accounts(conn).unwrap(); + let accounts = select_all_accounts(db).unwrap(); assert!(accounts.is_empty()); // test multiple entries let mut state = vec![]; @@ -349,19 +557,19 @@ fn sql_select_accounts() { details: None, }); - let res = queries::upsert_accounts( - conn, + let res = upsert_accounts( + db, &[BlockAccountUpdate::new( account_id, account_commitment, AccountUpdateDetails::Private, )], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ); assert_eq!(res.unwrap(), 1, "One element must have been inserted"); - let accounts = queries::select_all_accounts(conn).unwrap(); + let accounts = select_all_accounts(db).unwrap(); assert_eq!(accounts, state); } } @@ -369,8 +577,7 @@ fn sql_select_accounts() { #[test] #[miden_node_test_macro::enable_logging] fn sync_account_vault_basic_validation() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); // Create a public account for vault testing let public_account_id = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap(); @@ -380,16 +587,16 @@ fn sync_account_vault_basic_validation() { let invalid_block_from: BlockNumber = 10.into(); // Create blocks - create_block(conn, block_from); - create_block(conn, block_mid); - create_block(conn, block_to); + create_block(db, block_from); + create_block(db, block_mid); + create_block(db, block_to); for block in [block_from, block_mid, block_to] { - queries::upsert_accounts( - conn, + upsert_accounts( + db, &[mock_block_account_update(public_account_id, 0)], block, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); } @@ -402,16 +609,16 @@ fn sync_account_vault_basic_validation() { let vault_key_2 = fungible_asset_2.id(); // Insert vault assets for the public account at different blocks - queries::insert_account_vault_asset( - conn, + insert_account_vault_asset( + db, public_account_id, block_from, vault_key_1, Some(fungible_asset_1), ) .unwrap(); - queries::insert_account_vault_asset( - conn, + insert_account_vault_asset( + db, public_account_id, block_mid, vault_key_2, @@ -422,8 +629,8 @@ fn sync_account_vault_basic_validation() { // Update an existing vault asset (sets previous as not latest) let updated_fungible_asset_1 = Asset::Fungible(FungibleAsset::new(public_account_id, 1500).unwrap()); - queries::insert_account_vault_asset( - conn, + insert_account_vault_asset( + db, public_account_id, block_to, vault_key_1, @@ -432,11 +639,7 @@ fn sync_account_vault_basic_validation() { .unwrap(); // Test invalid block range - should return error - let result = queries::select_account_vault_assets( - conn, - public_account_id, - invalid_block_from..=block_to, - ); + let result = select_account_vault_assets(db, public_account_id, invalid_block_from..=block_to); assert!(result.is_err(), "expected error for invalid block range"); let Err(crate::errors::DatabaseError::InvalidBlockRange { .. }) = result else { @@ -445,8 +648,7 @@ fn sync_account_vault_basic_validation() { // Test with valid block range - should return vault assets let (last_block, values) = - queries::select_account_vault_assets(conn, public_account_id, block_from..=block_to) - .unwrap(); + select_account_vault_assets(db, public_account_id, block_from..=block_to).unwrap(); // Should return assets we inserted assert!(!values.is_empty(), "vault assets should have data"); @@ -463,25 +665,24 @@ fn sync_account_vault_basic_validation() { #[miden_node_test_macro::enable_logging] fn select_nullifiers_by_prefix_works() { const PREFIX_LEN: u8 = 16; - let mut conn = create_db(); - let conn = &mut conn; // test empty table + let db = &TestDb::new(); + // test empty table let block_number0 = 0.into(); let block_number10 = 10.into(); let (nullifiers, block_number_reached) = - queries::select_nullifiers_by_prefix(conn, PREFIX_LEN, &[], block_number0..=block_number10) - .unwrap(); + select_nullifiers_by_prefix(db, PREFIX_LEN, &[], block_number0..=block_number10).unwrap(); assert!(nullifiers.is_empty()); assert_eq!(block_number_reached, block_number10); // test single item let nullifier1 = num_to_nullifier(1 << 48); let block_number1 = 1.into(); - create_block(conn, block_number1); + create_block(db, block_number1); - queries::insert_nullifiers_for_block(conn, &[nullifier1], block_number1).unwrap(); + insert_nullifiers_for_block(db, &[nullifier1], block_number1).unwrap(); - let (nullifiers, block_number_reached) = queries::select_nullifiers_by_prefix( - conn, + let (nullifiers, block_number_reached) = select_nullifiers_by_prefix( + db, PREFIX_LEN, &[utils::get_nullifier_prefix(&nullifier1)], block_number0..=block_number10, @@ -500,16 +701,16 @@ fn select_nullifiers_by_prefix_works() { // test two elements let nullifier2 = num_to_nullifier(2 << 48); let block_number2 = 2.into(); - create_block(conn, block_number2); + create_block(db, block_number2); - queries::insert_nullifiers_for_block(conn, &[nullifier2], block_number2).unwrap(); + insert_nullifiers_for_block(db, &[nullifier2], block_number2).unwrap(); - let nullifiers = queries::select_all_nullifiers(conn).unwrap(); + let nullifiers = select_all_nullifiers(db).unwrap(); assert_eq!(nullifiers, vec![(nullifier1, block_number1), (nullifier2, block_number2)]); // only the nullifiers matching the prefix are included - let (nullifiers, _) = queries::select_nullifiers_by_prefix( - conn, + let (nullifiers, _) = select_nullifiers_by_prefix( + db, PREFIX_LEN, &[utils::get_nullifier_prefix(&nullifier1)], block_number0..=block_number10, @@ -522,8 +723,8 @@ fn select_nullifiers_by_prefix_works() { block_num: block_number1 }] ); - let (nullifiers, _) = queries::select_nullifiers_by_prefix( - conn, + let (nullifiers, _) = select_nullifiers_by_prefix( + db, PREFIX_LEN, &[utils::get_nullifier_prefix(&nullifier2)], block_number0..=block_number10, @@ -538,8 +739,8 @@ fn select_nullifiers_by_prefix_works() { ); // All matching nullifiers are included - let (nullifiers, _) = queries::select_nullifiers_by_prefix( - conn, + let (nullifiers, _) = select_nullifiers_by_prefix( + db, PREFIX_LEN, &[ utils::get_nullifier_prefix(&nullifier1), @@ -563,8 +764,8 @@ fn select_nullifiers_by_prefix_works() { ); // If a non-matching prefix is provided, no nullifiers are returned - let (nullifiers, _) = queries::select_nullifiers_by_prefix( - conn, + let (nullifiers, _) = select_nullifiers_by_prefix( + db, PREFIX_LEN, &[utils::get_nullifier_prefix(&num_to_nullifier(3 << 48))], block_number0..=block_number10, @@ -574,8 +775,8 @@ fn select_nullifiers_by_prefix_works() { // If a block number is provided, only matching nullifiers created at or after that block are // returned - let (nullifiers, _) = queries::select_nullifiers_by_prefix( - conn, + let (nullifiers, _) = select_nullifiers_by_prefix( + db, PREFIX_LEN, &[ utils::get_nullifier_prefix(&nullifier1), @@ -595,12 +796,12 @@ fn select_nullifiers_by_prefix_works() { // Nullifiers are not returned if the block number is after the last nullifier let nullifier3 = num_to_nullifier(3 << 48); let block_number3 = 3.into(); - create_block(conn, block_number3); + create_block(db, block_number3); - queries::insert_nullifiers_for_block(conn, &[nullifier3], block_number3).unwrap(); + insert_nullifiers_for_block(db, &[nullifier3], block_number3).unwrap(); - let (nullifiers, block_number_reached) = queries::select_nullifiers_by_prefix( - conn, + let (nullifiers, block_number_reached) = select_nullifiers_by_prefix( + db, PREFIX_LEN, &[ utils::get_nullifier_prefix(&nullifier1), @@ -629,13 +830,13 @@ fn select_nullifiers_by_prefix_works() { #[test] #[miden_node_test_macro::enable_logging] fn db_block_header() { - let mut conn = create_db(); - let conn = &mut conn; // test querying empty table + let db = &TestDb::new(); + // test querying empty table let block_number = 1; - let res = queries::select_block_header_by_block_num(conn, Some(block_number.into())).unwrap(); + let res = select_block_header_by_block_num(db, Some(block_number.into())).unwrap(); assert!(res.is_none()); - let res = queries::select_block_header_by_block_num(conn, None).unwrap(); + let res = select_block_header_by_block_num(db, None).unwrap(); assert!(res.is_none()); let block_header = BlockHeader::new( @@ -656,20 +857,20 @@ fn db_block_header() { let dummy_signature = BlockSignatures::new(vec![SigningKey::new().sign(block_header.commitment())]).unwrap(); - queries::insert_block_header(conn, &block_header, &dummy_signature).unwrap(); + insert_block_header(db, &block_header, &dummy_signature).unwrap(); + let first_signature = dummy_signature; // test fetch unknown block header let block_number = 1; - let res = queries::select_block_header_by_block_num(conn, Some(block_number.into())).unwrap(); + let res = select_block_header_by_block_num(db, Some(block_number.into())).unwrap(); assert!(res.is_none()); // test fetch block header by block number - let res = - queries::select_block_header_by_block_num(conn, Some(block_header.block_num())).unwrap(); + let res = select_block_header_by_block_num(db, Some(block_header.block_num())).unwrap(); assert_eq!(res.unwrap(), block_header); // test fetch latest block header - let res = queries::select_block_header_by_block_num(conn, None).unwrap(); + let res = select_block_header_by_block_num(db, None).unwrap(); assert_eq!(res.unwrap(), block_header); let block_header2 = BlockHeader::new( @@ -689,46 +890,59 @@ fn db_block_header() { let dummy_signature = BlockSignatures::new(vec![SigningKey::new().sign(block_header2.commitment())]).unwrap(); - queries::insert_block_header(conn, &block_header2, &dummy_signature).unwrap(); + insert_block_header(db, &block_header2, &dummy_signature).unwrap(); - let res = queries::select_block_header_by_block_num(conn, None).unwrap(); + let res = select_block_header_by_block_num(db, None).unwrap(); assert_eq!(res.unwrap(), block_header2); - let res = queries::select_block_headers( - conn, - [block_header.block_num(), block_header2.block_num()].into_iter(), - ) - .unwrap(); - assert_eq!(res, [block_header, block_header2]); + let res = select_block_headers(db, vec![block_header.block_num(), block_header2.block_num()]) + .unwrap(); + assert_eq!(res, [block_header.clone(), block_header2.clone()]); + + // commitments come back in block number order + let commitments = select_all_block_header_commitments(db).unwrap(); + assert_eq!( + commitments, + [ + BlockHeaderCommitment::new(&block_header), + BlockHeaderCommitment::new(&block_header2), + ] + ); + + // test fetch block header with its signatures + let stored = + select_block_header_and_signatures_by_block_num(db, block_header.block_num()).unwrap(); + assert_eq!(stored, Some((block_header, first_signature))); + + let missing = select_block_header_and_signatures_by_block_num(db, 1.into()).unwrap(); + assert!(missing.is_none()); } #[test] #[miden_node_test_macro::enable_logging] fn notes() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let block_num_1 = 1.into(); - create_block(conn, block_num_1); + create_block(db, block_num_1); let block_range = BlockNumber::GENESIS..=BlockNumber::from(1); // test empty table - let res = queries::select_notes_since_block_by_tag(conn, &[], block_range.clone()).unwrap(); + let res = select_notes_since_block_by_tag(db, &[], block_range.clone()).unwrap(); assert!(res.is_empty()); - let res = - queries::select_notes_since_block_by_tag(conn, &[1, 2, 3], block_range.clone()).unwrap(); + let res = select_notes_since_block_by_tag(db, &[1, 2, 3], block_range.clone()).unwrap(); assert!(res.is_empty()); let sender = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); // test insertion - queries::upsert_accounts( - conn, + upsert_accounts( + db, &[mock_block_account_update(sender, 0)], block_num_1, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -755,24 +969,24 @@ fn notes() { inclusion_path: inclusion_path.clone(), }; - queries::insert_scripts(conn, [¬e]).unwrap(); - queries::insert_notes(conn, &[(note.clone(), None)]).unwrap(); + insert_note_scripts(db, std::slice::from_ref(¬e)).unwrap(); + insert_notes(db, &[(note.clone(), None)]).unwrap(); // test empty tags - let res = queries::select_notes_since_block_by_tag(conn, &[], block_range.clone()).unwrap(); + let res = select_notes_since_block_by_tag(db, &[], block_range.clone()).unwrap(); assert!(res.is_empty()); let block_range_1 = 2.into()..=2.into(); // test no updates - let res = queries::select_notes_since_block_by_tag(conn, &[tag], block_range_1).unwrap(); + let res = select_notes_since_block_by_tag(db, &[tag], block_range_1).unwrap(); assert!(res.is_empty()); // test match - let res = queries::select_notes_since_block_by_tag(conn, &[tag], block_range.clone()).unwrap(); + let res = select_notes_since_block_by_tag(db, &[tag], block_range.clone()).unwrap(); assert_eq!(res, vec![note.clone().into()]); let block_num_2 = note.block_num + 1; - create_block(conn, block_num_2); + create_block(db, block_num_2); // insertion second note with same tag, but on higher block let note2 = NoteRecord { @@ -785,19 +999,19 @@ fn notes() { inclusion_path: inclusion_path.clone(), }; - queries::insert_notes(conn, &[(note2.clone(), None)]).unwrap(); + insert_notes(db, &[(note2.clone(), None)]).unwrap(); let block_range = 0.into()..=2.into(); // only the first matching block is returned; `get_note_sync_multi` loops this inside a single // database transaction when multiple blocks are requested. - let res = queries::select_notes_since_block_by_tag(conn, &[tag], block_range).unwrap(); + let res = select_notes_since_block_by_tag(db, &[tag], block_range).unwrap(); assert_eq!(res, vec![note.clone().into()]); let block_range = 2.into()..=2.into(); // only the second note is returned when range is restricted to block 2 - let res = queries::select_notes_since_block_by_tag(conn, &[tag], block_range).unwrap(); + let res = select_notes_since_block_by_tag(db, &[tag], block_range).unwrap(); assert_eq!(res, vec![note2.clone().into()]); // test query notes by id @@ -805,7 +1019,7 @@ fn notes() { let note_ids = Vec::from_iter(notes.iter().map(|note| NoteId::from_raw(note.note_id))); - let res = queries::select_notes_by_id(conn, ¬e_ids).unwrap(); + let res = select_notes_by_id(db, ¬e_ids).unwrap(); assert_eq!(res, notes); // test notes have correct details @@ -820,8 +1034,7 @@ fn notes() { #[test] #[miden_node_test_macro::enable_logging] fn note_sync_across_multiple_blocks() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let sender = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); @@ -831,12 +1044,12 @@ fn note_sync_across_multiple_blocks() { for block_num_raw in 1..=3u32 { let block_num = BlockNumber::from(block_num_raw); - create_block(conn, block_num); - queries::upsert_accounts( - conn, + create_block(db, block_num); + upsert_accounts( + db, &[mock_block_account_update(sender, block_num_raw.into())], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -870,8 +1083,8 @@ fn note_sync_across_multiple_blocks() { attachments, inclusion_path, }; - queries::insert_scripts(conn, [¬e]).unwrap(); - queries::insert_notes(conn, &[(note, None)]).unwrap(); + insert_note_scripts(db, std::slice::from_ref(¬e)).unwrap(); + insert_notes(db, &[(note, None)]).unwrap(); } // Build an MMR with enough leaves to cover all blocks (0..=3). @@ -884,8 +1097,8 @@ fn note_sync_across_multiple_blocks() { // A single call to get_note_sync_multi should return all 3 blocks. let block_range = BlockNumber::GENESIS..=BlockNumber::from(3); - let updates = queries::get_note_sync_multi( - conn, + let updates = get_note_sync_multi( + db, &[tag], block_range, miden_node_utils::limiter::MAX_RESPONSE_PAYLOAD_BYTES, @@ -915,8 +1128,7 @@ fn note_sync_across_multiple_blocks() { #[test] #[miden_node_test_macro::enable_logging] fn note_sync_multi_respects_payload_limit() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let sender = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); let tag = 43u32; @@ -924,12 +1136,12 @@ fn note_sync_multi_respects_payload_limit() { for block_num_raw in 1..=3u32 { let block_num = BlockNumber::from(block_num_raw); - create_block(conn, block_num); - queries::upsert_accounts( - conn, + create_block(db, block_num); + upsert_accounts( + db, &[mock_block_account_update(sender, block_num_raw.into())], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -963,14 +1175,13 @@ fn note_sync_multi_respects_payload_limit() { attachments, inclusion_path, }; - queries::insert_scripts(conn, [¬e]).unwrap(); - queries::insert_notes(conn, &[(note, None)]).unwrap(); + insert_note_scripts(db, std::slice::from_ref(¬e)).unwrap(); + insert_notes(db, &[(note, None)]).unwrap(); } - let one_block_budget = - queries::NOTE_SYNC_BLOCK_OVERHEAD_BYTES + queries::NOTE_SYNC_RECORD_BYTES; - let updates = queries::get_note_sync_multi( - conn, + let one_block_budget = NOTE_SYNC_BLOCK_OVERHEAD_BYTES + NOTE_SYNC_RECORD_BYTES; + let updates = get_note_sync_multi( + db, &[tag], BlockNumber::GENESIS..=BlockNumber::from(3), one_block_budget, @@ -991,17 +1202,16 @@ fn note_sync_multi_respects_payload_limit() { #[test] #[miden_node_test_macro::enable_logging] fn note_sync_no_matching_tags() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let sender = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); let block_num = BlockNumber::from(1); - create_block(conn, block_num); - queries::upsert_accounts( - conn, + create_block(db, block_num); + upsert_accounts( + db, &[mock_block_account_update(sender, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -1026,13 +1236,13 @@ fn note_sync_no_matching_tags() { attachments: NoteAttachments::default(), inclusion_path, }; - queries::insert_scripts(conn, [¬e]).unwrap(); - queries::insert_notes(conn, &[(note, None)]).unwrap(); + insert_note_scripts(db, std::slice::from_ref(¬e)).unwrap(); + insert_notes(db, &[(note, None)]).unwrap(); // Query with a different tag should return empty vec. let range = BlockNumber::GENESIS..=BlockNumber::from(1); - let result = queries::get_note_sync_multi( - conn, + let result = get_note_sync_multi( + db, &[999], range, miden_node_utils::limiter::MAX_RESPONSE_PAYLOAD_BYTES, @@ -1042,7 +1252,7 @@ fn note_sync_no_matching_tags() { } fn insert_account_patch( - conn: &mut SqliteConnection, + db: &TestDb, account_id: AccountId, block_number: BlockNumber, patch: &AccountPatch, @@ -1050,7 +1260,7 @@ fn insert_account_patch( for (slot_name, slot_patch) in patch.storage().maps() { for (k, v) in slot_patch.entries().into_iter().flat_map(StorageMapPatchEntries::as_map) { insert_account_storage_map_value( - conn, + db, account_id, block_number, slot_name.clone(), @@ -1069,29 +1279,28 @@ fn sql_account_storage_map_values_insertion() { use miden_protocol::account::StorageMapPatch; - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let block1: BlockNumber = 1.into(); let block2: BlockNumber = 2.into(); - create_block(conn, block1); - create_block(conn, block2); + create_block(db, block1); + create_block(db, block2); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE_2).unwrap(); - queries::upsert_accounts( - conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block1, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block2, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -1114,10 +1323,10 @@ fn sql_account_storage_map_values_insertion() { Some(Felt::new_unchecked(2)), ) .unwrap(); - insert_account_patch(conn, account_id, block1, &patch1); + insert_account_patch(db, account_id, block1, &patch1); - let storage_map_page = queries::select_account_storage_map_values_paged( - conn, + let storage_map_page = select_account_storage_map_values_paged( + db, account_id, BlockNumber::GENESIS..=block1, 1024, @@ -1137,10 +1346,10 @@ fn sql_account_storage_map_values_insertion() { Some(Felt::new_unchecked(3)), ) .unwrap(); - insert_account_patch(conn, account_id, block2, &patch2); + insert_account_patch(db, account_id, block2, &patch2); - let storage_map_values = queries::select_account_storage_map_values_paged( - conn, + let storage_map_values = select_account_storage_map_values_paged( + db, account_id, BlockNumber::GENESIS..=block2, 1024, @@ -1167,7 +1376,7 @@ fn sql_account_storage_map_values_insertion() { #[test] fn select_storage_map_sync_values() { - let mut conn = create_db(); + let db = &TestDb::new(); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); let slot_name = StorageSlotName::mock(5); @@ -1183,69 +1392,34 @@ fn select_storage_map_sync_values() { let block3 = BlockNumber::from(3); for block in [block1, block2, block3] { - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); } // Insert data across multiple blocks using individual inserts Block 1: key1 -> value1, key2 -> // value2 - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block1, - slot_name.clone(), - key1, - value1, - ) - .unwrap(); - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block1, - slot_name.clone(), - key2, - value2, - ) - .unwrap(); + insert_account_storage_map_value(db, account_id, block1, slot_name.clone(), key1, value1) + .unwrap(); + insert_account_storage_map_value(db, account_id, block1, slot_name.clone(), key2, value2) + .unwrap(); // Block 2: key2 -> value3 (update), key3 -> value3 (new) - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block2, - slot_name.clone(), - key2, - value3, - ) - .unwrap(); - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block2, - slot_name.clone(), - key3, - value3, - ) - .unwrap(); + insert_account_storage_map_value(db, account_id, block2, slot_name.clone(), key2, value3) + .unwrap(); + insert_account_storage_map_value(db, account_id, block2, slot_name.clone(), key3, value3) + .unwrap(); // Block 3: key1 -> value2 (update) - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block3, - slot_name.clone(), - key1, - value2, - ) - .unwrap(); + insert_account_storage_map_value(db, account_id, block3, slot_name.clone(), key1, value2) + .unwrap(); - let page = queries::select_account_storage_map_values_paged( - &mut conn, + let page = select_account_storage_map_values_paged( + db, account_id, BlockNumber::from(2)..=BlockNumber::from(3), 1024, @@ -1281,28 +1455,20 @@ fn select_storage_map_sync_values() { #[test] fn select_storage_map_sync_values_for_network_account() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); - let (account_id, _) = - make_account_and_note(&mut conn, block_num, [42u8; 32], AccountType::Public); + let (account_id, _) = make_account_and_note(db, block_num, [42u8; 32], AccountType::Public); let slot_name = StorageSlotName::mock(7); let key = StorageMapKey::from_index(1); let value = num_to_word(10); - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block_num, - slot_name.clone(), - key, - value, - ) - .unwrap(); + insert_account_storage_map_value(db, account_id, block_num, slot_name.clone(), key, value) + .unwrap(); - let page = queries::select_account_storage_map_values_paged( - &mut conn, + let page = select_account_storage_map_values_paged( + db, account_id, BlockNumber::GENESIS..=block_num, 1024, @@ -1318,7 +1484,7 @@ fn select_storage_map_sync_values_for_network_account() { #[test] fn select_storage_map_sync_values_paginates_until_last_block() { - let mut conn = create_db(); + let db = &TestDb::new(); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); let slot_name = StorageSlotName::mock(7); @@ -1326,34 +1492,34 @@ fn select_storage_map_sync_values_paginates_until_last_block() { let block2 = BlockNumber::from(2); let block3 = BlockNumber::from(3); - create_block(&mut conn, block1); - create_block(&mut conn, block2); - create_block(&mut conn, block3); + create_block(db, block1); + create_block(db, block2); + create_block(db, block3); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block1, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 1)], block2, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 2)], block3, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::insert_account_storage_map_value( - &mut conn, + insert_account_storage_map_value( + db, account_id, block1, slot_name.clone(), @@ -1361,8 +1527,8 @@ fn select_storage_map_sync_values_paginates_until_last_block() { num_to_word(11), ) .unwrap(); - queries::insert_account_storage_map_value( - &mut conn, + insert_account_storage_map_value( + db, account_id, block2, slot_name.clone(), @@ -1370,8 +1536,8 @@ fn select_storage_map_sync_values_paginates_until_last_block() { num_to_word(22), ) .unwrap(); - queries::insert_account_storage_map_value( - &mut conn, + insert_account_storage_map_value( + db, account_id, block3, slot_name.clone(), @@ -1380,13 +1546,9 @@ fn select_storage_map_sync_values_paginates_until_last_block() { ) .unwrap(); - let page = queries::select_account_storage_map_values_paged( - &mut conn, - account_id, - BlockNumber::GENESIS..=block3, - 1, - ) - .unwrap(); + let page = + select_account_storage_map_values_paged(db, account_id, BlockNumber::GENESIS..=block3, 1) + .unwrap(); assert_eq!(page.last_block_included, block1, "should truncate at block 1"); assert_eq!(page.values.len(), 1, "should include block 1 only"); @@ -1397,25 +1559,25 @@ fn select_storage_map_sync_values_paginates_until_last_block() { /// `last_block_num.saturating_sub(1) = -1` which failed `BlockNumber::from_raw_sql`. #[test] fn select_storage_map_sync_values_all_entries_in_genesis_block() { - let mut conn = create_db(); + let db = &TestDb::new(); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); let slot_name = StorageSlotName::mock(8); let genesis = BlockNumber::GENESIS; - create_block(&mut conn, genesis); + create_block(db, genesis); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], genesis, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); // Insert 3 entries, all in genesis block for i in 0..3 { - queries::insert_account_storage_map_value( - &mut conn, + insert_account_storage_map_value( + db, account_id, genesis, slot_name.clone(), @@ -1428,12 +1590,7 @@ fn select_storage_map_sync_values_all_entries_in_genesis_block() { // Query with limit=1 so that raw.len() (3) > limit (1), triggering the pagination branch. All // entries are in block 0, so take_while produces nothing and last_block_num.saturating_sub(1) = // -1. - let result = queries::select_account_storage_map_values_paged( - &mut conn, - account_id, - genesis..=genesis, - 1, - ); + let result = select_account_storage_map_values_paged(db, account_id, genesis..=genesis, 1); // Should not error - should return a valid page (possibly with empty values indicating no // progress, which the caller interprets as limit_exceeded) @@ -1450,24 +1607,24 @@ fn select_storage_map_sync_values_all_entries_in_genesis_block() { /// data. #[test] fn select_storage_map_sync_values_all_entries_in_single_non_genesis_block() { - let mut conn = create_db(); + let db = &TestDb::new(); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); let slot_name = StorageSlotName::mock(10); let block5 = BlockNumber::from(5); - create_block(&mut conn, block5); + create_block(db, block5); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block5, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); for i in 0..3 { - queries::insert_account_storage_map_value( - &mut conn, + insert_account_storage_map_value( + db, account_id, block5, slot_name.clone(), @@ -1478,9 +1635,7 @@ fn select_storage_map_sync_values_all_entries_in_single_non_genesis_block() { } // limit=1, so 3 rows > 1 triggers pagination. All in block 5. - let page = - queries::select_account_storage_map_values_paged(&mut conn, account_id, block5..=block5, 1) - .unwrap(); + let page = select_account_storage_map_values_paged(db, account_id, block5..=block5, 1).unwrap(); assert!(page.values.is_empty(), "should have no values when single block exceeds limit"); assert_eq!(page.last_block_included, block5, "should signal no progress at block 5"); @@ -1490,7 +1645,7 @@ fn select_storage_map_sync_values_all_entries_in_single_non_genesis_block() { /// limit causing block 3 to be dropped. #[test] fn select_storage_map_sync_values_multi_block_pagination() { - let mut conn = create_db(); + let db = &TestDb::new(); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); let slot_name = StorageSlotName::mock(11); @@ -1498,35 +1653,35 @@ fn select_storage_map_sync_values_multi_block_pagination() { let block2 = BlockNumber::from(2); let block3 = BlockNumber::from(3); - create_block(&mut conn, block1); - create_block(&mut conn, block2); - create_block(&mut conn, block3); + create_block(db, block1); + create_block(db, block2); + create_block(db, block3); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block1, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 1)], block2, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 2)], block3, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); // 1 entry in block 1, 1 in block 2, 1 in block 3 - queries::insert_account_storage_map_value( - &mut conn, + insert_account_storage_map_value( + db, account_id, block1, slot_name.clone(), @@ -1534,8 +1689,8 @@ fn select_storage_map_sync_values_multi_block_pagination() { num_to_word(11), ) .unwrap(); - queries::insert_account_storage_map_value( - &mut conn, + insert_account_storage_map_value( + db, account_id, block2, slot_name.clone(), @@ -1543,8 +1698,8 @@ fn select_storage_map_sync_values_multi_block_pagination() { num_to_word(22), ) .unwrap(); - queries::insert_account_storage_map_value( - &mut conn, + insert_account_storage_map_value( + db, account_id, block3, slot_name.clone(), @@ -1554,13 +1709,9 @@ fn select_storage_map_sync_values_multi_block_pagination() { .unwrap(); // limit=2: query fetches 3 rows (limit+1), drops block 3, keeps blocks 1-2 - let page = queries::select_account_storage_map_values_paged( - &mut conn, - account_id, - BlockNumber::GENESIS..=block3, - 2, - ) - .unwrap(); + let page = + select_account_storage_map_values_paged(db, account_id, BlockNumber::GENESIS..=block3, 2) + .unwrap(); assert_eq!(page.values.len(), 2, "should include entries from blocks 1 and 2"); assert_eq!(page.last_block_included, block2, "last included block should be 2"); @@ -1582,60 +1733,33 @@ async fn reconstruct_storage_map_from_db_pages_until_latest() { crate::db::migrations::bootstrap_database(&db_path).unwrap(); let db = crate::db::Db::load(db_path).await.unwrap(); let slot_name_for_db = slot_name.clone(); - db.query("insert paged values", move |db_conn| { - db_conn.transaction(|db_conn| { - create_block(db_conn, block1); - create_block(db_conn, block2); - create_block(db_conn, block3); - - queries::upsert_accounts( - db_conn, - &[mock_block_account_update(account_id, 0)], - block1, - &queries::PrecomputedPublicAccountStates::new(), - )?; - queries::upsert_accounts( - db_conn, - &[mock_block_account_update(account_id, 1)], - block2, - &queries::PrecomputedPublicAccountStates::new(), - )?; - queries::upsert_accounts( - db_conn, - &[mock_block_account_update(account_id, 2)], - block3, - &queries::PrecomputedPublicAccountStates::new(), - )?; + db.writer() + .write::<_, DatabaseError, _>("insert paged values", move |tx| { + for block in [block1, block2, block3] { + create_block_in(tx, block)?; + } - queries::insert_account_storage_map_value( - db_conn, - account_id, - block1, - slot_name_for_db.clone(), - num_to_storage_map_key(1), - num_to_word(10), - )?; - queries::insert_account_storage_map_value( - db_conn, - account_id, - block2, - slot_name_for_db.clone(), - num_to_storage_map_key(2), - num_to_word(20), - )?; - queries::insert_account_storage_map_value( - db_conn, - account_id, - block3, - slot_name_for_db.clone(), - num_to_storage_map_key(3), - num_to_word(30), - )?; - Ok::<_, DatabaseError>(()) + for (index, block) in [block1, block2, block3].into_iter().enumerate() { + queries::upsert_accounts( + tx, + &[mock_block_account_update(account_id, index as u64)], + block, + &PrecomputedPublicAccountStates::new(), + )?; + let entry = (index + 1) as u64; + queries::insert_account_storage_map_value( + tx, + account_id, + block, + &slot_name_for_db, + num_to_storage_map_key(entry), + num_to_word(entry * 10), + )?; + } + Ok(()) }) - }) - .await - .unwrap(); + .await + .unwrap(); let details = db .reconstruct_storage_map_from_db( @@ -1670,33 +1794,32 @@ async fn reconstruct_storage_map_from_db_returns_limit_exceeded_for_single_block crate::db::migrations::bootstrap_database(&db_path).unwrap(); let db = crate::db::Db::load(db_path).await.unwrap(); let slot_name_for_db = slot_name.clone(); - db.query("insert entries in single block", move |db_conn| { - db_conn.transaction(|db_conn| { - create_block(db_conn, block5); + db.writer() + .write::<_, DatabaseError, _>("insert entries in single block", move |tx| { + create_block_in(tx, block5)?; queries::upsert_accounts( - db_conn, + tx, &[mock_block_account_update(account_id, 0)], block5, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), )?; // Insert 3 entries, all in the same block for i in 1..=3 { queries::insert_account_storage_map_value( - db_conn, + tx, account_id, block5, - slot_name_for_db.clone(), + &slot_name_for_db, num_to_storage_map_key(i), num_to_word(i * 10), )?; } - Ok::<_, DatabaseError>(()) + Ok(()) }) - }) - .await - .unwrap(); + .await + .unwrap(); // Use limit=1 so that 3 entries in a single block exceed the limit. block_range_start is block5 // (the first block with data), and the target is also block5. @@ -1836,33 +1959,25 @@ fn mock_block_transaction_with_output_notes( ) } -fn insert_transactions(conn: &mut SqliteConnection) -> usize { +/// Inserts an account and two transactions against it at block 1, returning the rows written. +fn insert_mock_transactions(db: &TestDb) -> usize { let block_num = 1.into(); - create_block(conn, block_num); + create_block(db, block_num); - conn.transaction(|conn| { - let account_id = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); + let account_id = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); - let account_updates = vec![mock_block_account_update(account_id, 1)]; + let account_updates = vec![mock_block_account_update(account_id, 1)]; - let mock_tx1 = - mock_block_transaction(AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(), 1); - let mock_tx2 = - mock_block_transaction(AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(), 2); - let ordered_tx_headers = OrderedTransactionHeaders::new_unchecked(vec![mock_tx1, mock_tx2]); + let mock_tx1 = + mock_block_transaction(AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(), 1); + let mock_tx2 = + mock_block_transaction(AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(), 2); + let ordered_tx_headers = OrderedTransactionHeaders::new_unchecked(vec![mock_tx1, mock_tx2]); - queries::upsert_accounts( - conn, - &account_updates, - block_num, - &queries::PrecomputedPublicAccountStates::new(), - ) + upsert_accounts(db, &account_updates, block_num, &PrecomputedPublicAccountStates::new()) .unwrap(); - let count = queries::insert_transactions(conn, block_num, &ordered_tx_headers).unwrap(); - Ok::<_, DatabaseError>(count) - }) - .unwrap() + insert_transactions(db, block_num, &ordered_tx_headers).unwrap() } fn mock_account_code_and_storage( @@ -1914,12 +2029,12 @@ fn mock_account_code_and_storage( #[test] fn test_select_account_code_by_commitment() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num_1 = BlockNumber::from(1); // Create block 1 - create_block(&mut conn, block_num_1); + create_block(db, block_num_1); // Create an account with code at block 1 using the existing mock function let account = mock_account_code_and_storage(AccountType::Public, [], None); @@ -1929,8 +2044,8 @@ fn test_select_account_code_by_commitment() { let expected_code = account.code().to_bytes(); // Insert the account at block 1 - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[BlockAccountUpdate::new( account.id(), account.to_commitment(), @@ -1942,7 +2057,7 @@ fn test_select_account_code_by_commitment() { .unwrap(); // Query code by commitment - should return the code - let code = queries::select_account_code_by_commitment(&mut conn, code_commitment) + let code = select_account_code_by_commitment(db, code_commitment) .unwrap() .expect("Code should exist"); assert_eq!(code, expected_code); @@ -1950,21 +2065,20 @@ fn test_select_account_code_by_commitment() { // Query code for non-existent commitment - should return None let non_existent_commitment = [0u8; 32]; let non_existent_commitment = Word::read_from_bytes(&non_existent_commitment).unwrap(); - let code_other = - queries::select_account_code_by_commitment(&mut conn, non_existent_commitment).unwrap(); + let code_other = select_account_code_by_commitment(db, non_existent_commitment).unwrap(); assert!(code_other.is_none(), "Code should not exist for non-existent commitment"); } #[test] fn test_select_account_code_by_commitment_multiple_codes() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num_1 = BlockNumber::from(1); let block_num_2 = BlockNumber::from(2); // Create blocks - create_block(&mut conn, block_num_1); - create_block(&mut conn, block_num_2); + create_block(db, block_num_1); + create_block(db, block_num_2); // Create account with code v1 at block 1 let code_v1_str = "\ @@ -1979,8 +2093,8 @@ fn test_select_account_code_by_commitment_multiple_codes() { let code_v1 = account_v1.code().to_bytes(); // Insert the account at block 1 - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[BlockAccountUpdate::new( account_v1.id(), account_v1.to_commitment(), @@ -2014,8 +2128,8 @@ fn test_select_account_code_by_commitment_multiple_codes() { ); // Insert the updated account at block 2 - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[BlockAccountUpdate::new( account_v2.id(), account_v2.to_commitment(), @@ -2027,16 +2141,14 @@ fn test_select_account_code_by_commitment_multiple_codes() { .unwrap(); // Both codes should be retrievable by their respective commitments - let code_from_v1_commitment = - queries::select_account_code_by_commitment(&mut conn, code_v1_commitment) - .unwrap() - .expect("v1 code should exist"); + let code_from_v1_commitment = select_account_code_by_commitment(db, code_v1_commitment) + .unwrap() + .expect("v1 code should exist"); assert_eq!(code_from_v1_commitment, code_v1, "v1 commitment should return v1 code"); - let code_from_v2_commitment = - queries::select_account_code_by_commitment(&mut conn, code_v2_commitment) - .unwrap() - .expect("v2 code should exist"); + let code_from_v2_commitment = select_account_code_by_commitment(db, code_v2_commitment) + .unwrap() + .expect("v2 code should exist"); assert_eq!(code_from_v2_commitment, code_v2, "v2 commitment should return v2 code"); } @@ -2086,7 +2198,7 @@ async fn genesis_with_account_assets() { let temp_dir = tempdir().unwrap(); let db_path = temp_dir.path().join("store.sqlite"); - crate::db::Db::bootstrap(db_path, genesis_block).unwrap(); + crate::db::Db::bootstrap(db_path, genesis_block).await.unwrap(); } /// Verifies genesis block with account containing storage maps can be inserted. @@ -2158,7 +2270,7 @@ async fn genesis_with_account_storage_map() { let temp_dir = tempdir().unwrap(); let db_path = temp_dir.path().join("store.sqlite"); - crate::db::Db::bootstrap(db_path, genesis_block).unwrap(); + crate::db::Db::bootstrap(db_path, genesis_block).await.unwrap(); } /// Verifies genesis block with account containing both vault assets and storage maps. @@ -2223,7 +2335,7 @@ async fn genesis_with_account_assets_and_storage() { let temp_dir = tempdir().unwrap(); let db_path = temp_dir.path().join("store.sqlite"); - crate::db::Db::bootstrap(db_path, genesis_block).unwrap(); + crate::db::Db::bootstrap(db_path, genesis_block).await.unwrap(); } /// Verifies genesis block with multiple accounts of different types. Tests realistic genesis @@ -2324,15 +2436,15 @@ async fn genesis_with_multiple_accounts() { let temp_dir = tempdir().unwrap(); let db_path = temp_dir.path().join("store.sqlite"); - crate::db::Db::bootstrap(db_path, genesis_block).unwrap(); + crate::db::Db::bootstrap(db_path, genesis_block).await.unwrap(); } #[test] #[miden_node_test_macro::enable_logging] fn regression_1461_full_state_delta_inserts_vault_assets() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num: BlockNumber = 1.into(); - create_block(&mut conn, block_num); + create_block(db, block_num); let faucet_id = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap(); let fungible_asset = FungibleAsset::new(faucet_id, 5000).unwrap(); @@ -2354,20 +2466,11 @@ fn regression_1461_full_state_delta_inserts_vault_assets() { AccountUpdateDetails::Public(account_patch), ); - queries::upsert_accounts( - &mut conn, - &[block_update], - block_num, - &precomputed_states_from_account(&account), - ) - .unwrap(); + upsert_accounts(db, &[block_update], block_num, &precomputed_states_from_account(&account)) + .unwrap(); - let (_, vault_assets) = queries::select_account_vault_assets( - &mut conn, - account_id, - BlockNumber::GENESIS..=block_num, - ) - .unwrap(); + let (_, vault_assets) = + select_account_vault_assets(db, account_id, BlockNumber::GENESIS..=block_num).unwrap(); // Before the fix, vault_assets was empty let vault_asset = vault_assets.first().unwrap(); @@ -2504,7 +2607,7 @@ fn serialization_symmetry_note_id_vec() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_block_header() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_header = BlockHeader::new( 1_u8.into(), @@ -2524,13 +2627,12 @@ fn db_roundtrip_block_header() { // Insert let dummy_signature = BlockSignatures::new(vec![SigningKey::new().sign(block_header.commitment())]).unwrap(); - queries::insert_block_header(&mut conn, &block_header, &dummy_signature).unwrap(); + insert_block_header(db, &block_header, &dummy_signature).unwrap(); // Retrieve - let retrieved = - queries::select_block_header_by_block_num(&mut conn, Some(block_header.block_num())) - .unwrap() - .expect("Block header should exist"); + let retrieved = select_block_header_by_block_num(db, Some(block_header.block_num())) + .unwrap() + .expect("Block header should exist"); assert_eq!(block_header, retrieved, "BlockHeader DB roundtrip must be symmetric"); } @@ -2538,17 +2640,17 @@ fn db_roundtrip_block_header() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_nullifiers() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let nullifiers: Vec = (0..5).map(|i| num_to_nullifier(i << 48)).collect(); // Insert - queries::insert_nullifiers_for_block(&mut conn, &nullifiers, block_num).unwrap(); + insert_nullifiers_for_block(db, &nullifiers, block_num).unwrap(); // Retrieve - let retrieved = queries::select_all_nullifiers(&mut conn).unwrap(); + let retrieved = select_all_nullifiers(db).unwrap(); assert_eq!(nullifiers.len(), retrieved.len(), "Should retrieve same number of nullifiers"); for (orig, info) in nullifiers.iter().zip(retrieved.iter()) { @@ -2560,9 +2662,9 @@ fn db_roundtrip_nullifiers() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_account() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let account = mock_account_code_and_storage(AccountType::Public, [], Some([99u8; 32])); let account_id = account.id(); @@ -2575,16 +2677,11 @@ fn db_roundtrip_account() { account_commitment, AccountUpdateDetails::Public(account_patch), ); - queries::upsert_accounts( - &mut conn, - &[block_update], - block_num, - &precomputed_states_from_account(&account), - ) - .unwrap(); + upsert_accounts(db, &[block_update], block_num, &precomputed_states_from_account(&account)) + .unwrap(); // Retrieve - let retrieved = queries::select_all_accounts(&mut conn).unwrap(); + let retrieved = select_all_accounts(db).unwrap(); assert_eq!(retrieved.len(), 1, "Should have one account"); let retrieved_info = &retrieved[0]; @@ -2602,16 +2699,16 @@ fn db_roundtrip_account() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_notes() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let sender = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(sender, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -2629,12 +2726,12 @@ fn db_roundtrip_notes() { }; // Insert - queries::insert_scripts(&mut conn, [¬e]).unwrap(); - queries::insert_notes(&mut conn, &[(note.clone(), None)]).unwrap(); + insert_note_scripts(db, std::slice::from_ref(¬e)).unwrap(); + insert_notes(db, &[(note.clone(), None)]).unwrap(); // Retrieve let note_ids = vec![NoteId::from_raw(note.note_id)]; - let retrieved = queries::select_notes_by_id(&mut conn, ¬e_ids).unwrap(); + let retrieved = select_notes_by_id(db, ¬e_ids).unwrap(); assert_eq!(retrieved.len(), 1, "Should have one note"); let retrieved_note = &retrieved[0]; @@ -2657,19 +2754,19 @@ fn db_roundtrip_notes() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_vault_assets() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let faucet_id = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap(); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); // Create account first - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -2678,16 +2775,11 @@ fn db_roundtrip_vault_assets() { let vault_key = asset.id(); // Insert vault asset - queries::insert_account_vault_asset(&mut conn, account_id, block_num, vault_key, Some(asset)) - .unwrap(); + insert_account_vault_asset(db, account_id, block_num, vault_key, Some(asset)).unwrap(); // Retrieve - let (_, vault_assets) = queries::select_account_vault_assets( - &mut conn, - account_id, - BlockNumber::GENESIS..=block_num, - ) - .unwrap(); + let (_, vault_assets) = + select_account_vault_assets(db, account_id, BlockNumber::GENESIS..=block_num).unwrap(); assert_eq!(vault_assets.len(), 1, "Should have one vault asset"); let retrieved = &vault_assets[0]; @@ -2700,44 +2792,37 @@ fn db_roundtrip_vault_assets() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_storage_map_values() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); let slot_name = StorageSlotName::mock(5); let key = StorageMapKey::from_index(12345u32); let value = num_to_word(67890); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 1)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); // Insert - queries::insert_account_storage_map_value( - &mut conn, - account_id, - block_num, - slot_name.clone(), - key, - value, - ) - .unwrap(); + insert_account_storage_map_value(db, account_id, block_num, slot_name.clone(), key, value) + .unwrap(); // Retrieve - let page = queries::select_account_storage_map_values_paged( - &mut conn, + let page = select_account_storage_map_values_paged( + db, account_id, BlockNumber::GENESIS..=block_num, 1024, @@ -2758,9 +2843,9 @@ fn db_roundtrip_storage_map_values() { fn db_roundtrip_account_storage_with_maps() { use miden_protocol::account::StorageMap; - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); // Create storage with both value slots and map slots let storage_map = StorageMap::with_entries(vec![ @@ -2823,17 +2908,12 @@ fn db_roundtrip_account_storage_with_maps() { account.to_commitment(), AccountUpdateDetails::Public(account_patch), ); - queries::upsert_accounts( - &mut conn, - &[block_update], - block_num, - &precomputed_states_from_account(&account), - ) - .unwrap(); + upsert_accounts(db, &[block_update], block_num, &precomputed_states_from_account(&account)) + .unwrap(); // Retrieve the storage using select_latest_account_storage (reconstructs from header + map // values) - let retrieved_storage = queries::select_latest_account_storage(&mut conn, account_id).unwrap(); + let retrieved_storage = select_latest_account_storage(db, account_id).unwrap(); let retrieved_commitment = retrieved_storage.to_commitment(); // Verify the commitment matches (this proves the reconstruction is correct) @@ -2873,7 +2953,7 @@ fn db_roundtrip_account_storage_with_maps() { } // Also verify full account reconstruction via select_account (which calls select_full_account) - let account_info = queries::select_account(&mut conn, account_id).unwrap(); + let account_info = select_account(db, account_id).unwrap(); assert!(account_info.details.is_some(), "Public account should have details"); let retrieved_account = account_info.details.unwrap(); assert_eq!( @@ -2886,12 +2966,11 @@ fn db_roundtrip_account_storage_with_maps() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_note_metadata_attachment() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); - let (account_id, _) = - make_account_and_note(&mut conn, block_num, [1u8; 32], AccountType::Public); + let (account_id, _) = make_account_and_note(db, block_num, [1u8; 32], AccountType::Public); let target = NetworkAccountTarget::new(account_id, NoteExecutionHint::Always) .expect("NetworkAccountTarget creation should succeed for network account"); @@ -2912,11 +2991,11 @@ fn db_roundtrip_note_metadata_attachment() { inclusion_path: SparseMerklePath::default(), }; - queries::insert_scripts(&mut conn, [¬e]).unwrap(); - queries::insert_notes(&mut conn, &[(note.clone(), None)]).unwrap(); + insert_note_scripts(db, std::slice::from_ref(¬e)).unwrap(); + insert_notes(db, &[(note.clone(), None)]).unwrap(); // Fetch the note back and verify the attachment is preserved - let retrieved = queries::select_notes_by_id(&mut conn, &[NoteId::from_raw(note.note_id)]) + let retrieved = select_notes_by_id(db, &[NoteId::from_raw(note.note_id)]) .expect("select_notes_by_id should succeed"); assert_eq!(retrieved.len(), 1, "Should retrieve exactly one note"); @@ -2937,8 +3016,8 @@ fn db_roundtrip_note_metadata_attachment() { // Note sync uses a narrower record than `select_notes_by_id`, but it must retain attachments so // the RPC layer can expose single-word values. - let synced = queries::select_notes_since_block_by_tag( - &mut conn, + let synced = select_notes_since_block_by_tag( + db, &[metadata.tag().as_u32()], BlockNumber::GENESIS..=block_num, ) @@ -2950,8 +3029,7 @@ fn db_roundtrip_note_metadata_attachment() { #[test] #[miden_node_test_macro::enable_logging] fn test_prune_history() { - let mut conn = create_db(); - let conn = &mut conn; + let db = &TestDb::new(); let public_account_id = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET).unwrap(); @@ -2968,16 +3046,16 @@ fn test_prune_history() { let block_tip: BlockNumber = (HISTORICAL_BLOCK_RETENTION + CUTOFF_BLOCK_OFFSET).into(); for block in [block_0, block_old, block_cutoff, block_update, block_tip] { - create_block(conn, block); + create_block(db, block); } // Create account for block in [block_0, block_old, block_cutoff, block_update, block_tip] { - queries::upsert_accounts( - conn, + upsert_accounts( + db, &[mock_block_account_update(public_account_id, 0)], block, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); } @@ -2995,29 +3073,17 @@ fn test_prune_history() { // Stale entry at block_0, superseded at block_old which is also below the cutoff — should be // deleted. let stale_asset = Asset::Fungible(FungibleAsset::new(public_account_id, 500).unwrap()); - queries::insert_account_vault_asset( - conn, - public_account_id, - block_0, - vault_key_old, - Some(stale_asset), - ) - .unwrap(); + insert_account_vault_asset(db, public_account_id, block_0, vault_key_old, Some(stale_asset)) + .unwrap(); // Entry at block_old, superseded only at block_update which is above the cutoff — must be // retained as the key's baseline for reads at block_cutoff. - queries::insert_account_vault_asset( - conn, - public_account_id, - block_old, - vault_key_old, - Some(asset_1), - ) - .unwrap(); + insert_account_vault_asset(db, public_account_id, block_old, vault_key_old, Some(asset_1)) + .unwrap(); // Entry exactly at cutoff (block_cutoff, should be retained) - queries::insert_account_vault_asset( - conn, + insert_account_vault_asset( + db, public_account_id, block_cutoff, vault_key_cutoff, @@ -3026,19 +3092,13 @@ fn test_prune_history() { .unwrap(); // Recent entry (should always be retained) - queries::insert_account_vault_asset( - conn, - public_account_id, - block_tip, - vault_key_recent, - Some(asset_3), - ) - .unwrap(); + insert_account_vault_asset(db, public_account_id, block_tip, vault_key_recent, Some(asset_3)) + .unwrap(); // Update an entry to create a non-latest version let updated_asset = Asset::Fungible(FungibleAsset::new(public_account_id, 1500).unwrap()); - queries::insert_account_vault_asset( - conn, + insert_account_vault_asset( + db, public_account_id, block_update, vault_key_old, @@ -3060,7 +3120,7 @@ fn test_prune_history() { // Stale entry at block_0, superseded at block_old which is also below the cutoff — should be // deleted. insert_account_storage_map_value( - conn, + db, public_account_id, block_0, slot_name.clone(), @@ -3072,7 +3132,7 @@ fn test_prune_history() { // Entry at block_old, superseded only at block_update which is above the cutoff — must be // retained as the key's baseline for reads at block_cutoff. insert_account_storage_map_value( - conn, + db, public_account_id, block_old, slot_name.clone(), @@ -3083,7 +3143,7 @@ fn test_prune_history() { // Storage map entry at cutoff boundary (block_cutoff) insert_account_storage_map_value( - conn, + db, public_account_id, block_cutoff, slot_name.clone(), @@ -3094,7 +3154,7 @@ fn test_prune_history() { // Recent storage map entry insert_account_storage_map_value( - conn, + db, public_account_id, block_tip, slot_name.clone(), @@ -3105,7 +3165,7 @@ fn test_prune_history() { // Update map_key_old to create a non-latest entry at block_update insert_account_storage_map_value( - conn, + db, public_account_id, block_update, slot_name.clone(), @@ -3116,16 +3176,12 @@ fn test_prune_history() { // Verify initial state - should have 5 vault assets and 5 storage map values let (_, initial_vault_assets) = - queries::select_account_vault_assets(conn, public_account_id, block_0..=block_tip).unwrap(); + select_account_vault_assets(db, public_account_id, block_0..=block_tip).unwrap(); assert_eq!(initial_vault_assets.len(), 5, "should have 5 vault assets before cleanup"); - let initial_storage_values = queries::select_account_storage_map_values_paged( - conn, - public_account_id, - block_0..=block_tip, - 1024, - ) - .unwrap(); + let initial_storage_values = + select_account_storage_map_values_paged(db, public_account_id, block_0..=block_tip, 1024) + .unwrap(); assert_eq!( initial_storage_values.values.len(), 5, @@ -3134,8 +3190,7 @@ fn test_prune_history() { // Run cleanup with chain_tip = block_tip, cutoff will be block_tip - HISTORICAL_BLOCK_RETENTION // = block_cutoff - let (vault_deleted, storage_deleted, _codes_deleted) = - queries::prune_history(conn, block_tip).unwrap(); + let (vault_deleted, storage_deleted, _codes_deleted) = prune_history(db, block_tip).unwrap(); // Only the block_0 rows are deletable: they are superseded at block_old, which is also below // the cutoff. The block_old rows are superseded only above the cutoff, so they remain the @@ -3145,7 +3200,7 @@ fn test_prune_history() { // Verify remaining vault assets - should have 4 (baseline at block_old, cutoff, update, tip) let (_, remaining_vault_assets) = - queries::select_account_vault_assets(conn, public_account_id, block_0..=block_tip).unwrap(); + select_account_vault_assets(db, public_account_id, block_0..=block_tip).unwrap(); assert_eq!(remaining_vault_assets.len(), 4, "should have 4 vault assets after cleanup"); // Verify no vault asset at block_0 remains @@ -3174,13 +3229,9 @@ fn test_prune_history() { // Verify remaining storage map values - should have 4 (baseline at block_old, cutoff, update, // tip) - let remaining_storage_values = queries::select_account_storage_map_values_paged( - conn, - public_account_id, - block_0..=block_tip, - 1024, - ) - .unwrap(); + let remaining_storage_values = + select_account_storage_map_values_paged(db, public_account_id, block_0..=block_tip, 1024) + .unwrap(); assert_eq!( remaining_storage_values.values.len(), 4, @@ -3214,7 +3265,7 @@ fn test_prune_history() { // Regression check for baseline loss: reconstructing the vault at the cutoff block must still // see block_old's value, even though that row is older than the cutoff. let assets_at_cutoff = - queries::select_account_vault_at_block(conn, public_account_id, block_cutoff).unwrap(); + select_account_vault_at_block(db, public_account_id, block_cutoff).unwrap(); assert!( assets_at_cutoff.contains(&asset_1), "vault reconstruction at the cutoff must include the baseline written at block_old" @@ -3225,8 +3276,8 @@ fn test_prune_history() { let faucet_4 = AccountId::try_from(ACCOUNT_ID_PUBLIC_FUNGIBLE_FAUCET_3).unwrap(); let asset_old = Asset::Fungible(FungibleAsset::new(faucet_4, 9999).unwrap()); let vault_key_old_latest = asset_old.id(); - queries::insert_account_vault_asset( - conn, + insert_account_vault_asset( + db, public_account_id, block_0, vault_key_old_latest, @@ -3235,14 +3286,14 @@ fn test_prune_history() { .unwrap(); // This entry at block 0 keeps an open validity interval. Run cleanup again - let (vault_deleted_2, ..) = queries::prune_history(conn, block_tip).unwrap(); + let (vault_deleted_2, ..) = prune_history(db, block_tip).unwrap(); // The old open-ended entry should not be deleted (vault_deleted_2 should be 0) assert_eq!(vault_deleted_2, 0, "should not delete any open-ended entries"); // Verify the old open-ended entry still exists let (_, vault_assets_with_latest) = - queries::select_account_vault_assets(conn, public_account_id, block_0..=block_tip).unwrap(); + select_account_vault_assets(db, public_account_id, block_0..=block_tip).unwrap(); assert!( vault_assets_with_latest .iter() @@ -3263,13 +3314,13 @@ fn account_state_forest_matches_db_storage_map_roots_across_updates() { /// Reconstructs storage map root from DB entries at a specific block. fn reconstruct_storage_map_root_from_db( - conn: &mut SqliteConnection, + db: &TestDb, account_id: AccountId, slot_name: &StorageSlotName, block_num: BlockNumber, ) -> Option { - let storage_values = queries::select_account_storage_map_values_paged( - conn, + let storage_values = select_account_storage_map_values_paged( + db, account_id, BlockNumber::GENESIS..=block_num, 1024, @@ -3315,7 +3366,7 @@ fn account_state_forest_matches_db_storage_map_roots_across_updates() { Some(smt.root()) } - let mut conn = create_db(); + let db = &TestDb::new(); let mut forest = AccountStateForest::new(); let account_id = AccountId::try_from(ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE).unwrap(); @@ -3323,29 +3374,29 @@ fn account_state_forest_matches_db_storage_map_roots_across_updates() { let block2 = BlockNumber::from(2); let block3 = BlockNumber::from(3); - create_block(&mut conn, block1); - create_block(&mut conn, block2); - create_block(&mut conn, block3); + create_block(db, block1); + create_block(db, block2); + create_block(db, block3); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 0)], block1, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 1)], block2, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(account_id, 2)], block3, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -3378,12 +3429,12 @@ fn account_state_forest_matches_db_storage_map_roots_across_updates() { ) .unwrap(); - insert_account_patch(&mut conn, account_id, block1, &patch_1); + insert_account_patch(db, account_id, block1, &patch_1); forest.update_account(block1, &patch_1); // Verify forest matches DB for block 1 let forest_root_1 = forest.get_storage_map_root(account_id, &slot_map, block1).unwrap(); - let db_root_1 = reconstruct_storage_map_root_from_db(&mut conn, account_id, &slot_map, block1) + let db_root_1 = reconstruct_storage_map_root_from_db(db, account_id, &slot_map, block1) .expect("DB should have storage map root"); assert_eq!( @@ -3411,12 +3462,12 @@ fn account_state_forest_matches_db_storage_map_roots_across_updates() { ) .unwrap(); - insert_account_patch(&mut conn, account_id, block2, &patch_2); + insert_account_patch(db, account_id, block2, &patch_2); forest.update_account(block2, &patch_2); // Verify forest matches DB for block 2 let forest_root_2 = forest.get_storage_map_root(account_id, &slot_map, block2).unwrap(); - let db_root_2 = reconstruct_storage_map_root_from_db(&mut conn, account_id, &slot_map, block2) + let db_root_2 = reconstruct_storage_map_root_from_db(db, account_id, &slot_map, block2) .expect("DB should have storage map root"); assert_eq!( @@ -3444,12 +3495,12 @@ fn account_state_forest_matches_db_storage_map_roots_across_updates() { ) .unwrap(); - insert_account_patch(&mut conn, account_id, block3, &patch_3); + insert_account_patch(db, account_id, block3, &patch_3); forest.update_account(block3, &patch_3); // Verify forest matches DB for block 3 let forest_root_3 = forest.get_storage_map_root(account_id, &slot_map, block3).unwrap(); - let db_root_3 = reconstruct_storage_map_root_from_db(&mut conn, account_id, &slot_map, block3) + let db_root_3 = reconstruct_storage_map_root_from_db(db, account_id, &slot_map, block3) .expect("DB should have storage map root"); assert_eq!( @@ -3459,9 +3510,8 @@ fn account_state_forest_matches_db_storage_map_roots_across_updates() { // Verify we can query historical roots let forest_root_1_check = forest.get_storage_map_root(account_id, &slot_map, block1).unwrap(); - let db_root_1_check = - reconstruct_storage_map_root_from_db(&mut conn, account_id, &slot_map, block1) - .expect("DB should have storage map root"); + let db_root_1_check = reconstruct_storage_map_root_from_db(db, account_id, &slot_map, block1) + .expect("DB should have storage map root"); assert_eq!( forest_root_1_check, db_root_1_check, "Historical query for block 1 should match" @@ -3772,16 +3822,16 @@ fn account_state_forest_preserves_most_recent_vault_only() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_transactions() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let bob = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(bob, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -3806,12 +3856,11 @@ fn db_roundtrip_transactions() { ) }) .collect(); - queries::insert_notes(&mut conn, &output_notes).unwrap(); - queries::insert_transactions(&mut conn, block_num, &ordered).unwrap(); + insert_notes(db, &output_notes).unwrap(); + insert_transactions(db, block_num, &ordered).unwrap(); let retrieved = - queries::select_transactions_records(&mut conn, &[bob], BlockNumber::GENESIS..=block_num) - .unwrap(); + select_transactions_records(db, &[bob], BlockNumber::GENESIS..=block_num).unwrap(); let record = retrieved.1.first().expect("entry should exist"); let expected_sync_records: Vec<_> = tx @@ -3843,16 +3892,16 @@ fn db_roundtrip_transactions() { #[test] #[miden_node_test_macro::enable_logging] fn db_roundtrip_transactions_filters_missing_output_note_sync_records() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let bob = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(bob, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -3861,11 +3910,10 @@ fn db_roundtrip_transactions_filters_missing_output_note_sync_records() { // Notes erased within the same block are not inserted into the `notes` table, so transaction // sync should classify them as erased instead of failing the whole request. - queries::insert_transactions(&mut conn, block_num, &ordered).unwrap(); + insert_transactions(db, block_num, &ordered).unwrap(); let retrieved = - queries::select_transactions_records(&mut conn, &[bob], BlockNumber::GENESIS..=block_num) - .unwrap(); + select_transactions_records(db, &[bob], BlockNumber::GENESIS..=block_num).unwrap(); let record = retrieved.1.first().expect("entry should exist"); let expected = TransactionRecord { @@ -3885,16 +3933,16 @@ fn db_roundtrip_transactions_filters_missing_output_note_sync_records() { #[test] #[miden_node_test_macro::enable_logging] fn select_transactions_records_resolves_consumed_public_note_refs() { - let mut conn = create_db(); + let db = &TestDb::new(); let block_num = BlockNumber::from(1); - create_block(&mut conn, block_num); + create_block(db, block_num); let bob = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(bob, 0)], block_num, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -3917,12 +3965,11 @@ fn select_transactions_records_resolves_consumed_public_note_refs() { attachments: NoteAttachments::default(), inclusion_path: SparseMerklePath::default(), }; - queries::insert_notes(&mut conn, &[(note_record, Some(nullifier))]).unwrap(); - queries::insert_transactions(&mut conn, block_num, &ordered).unwrap(); + insert_notes(db, &[(note_record, Some(nullifier))]).unwrap(); + insert_transactions(db, block_num, &ordered).unwrap(); let retrieved = - queries::select_transactions_records(&mut conn, &[bob], BlockNumber::GENESIS..=block_num) - .unwrap(); + select_transactions_records(db, &[bob], BlockNumber::GENESIS..=block_num).unwrap(); let record = retrieved.1.first().expect("entry should exist"); assert_eq!(record.consumed_note_refs, vec![(nullifier, note_id)]); @@ -3938,25 +3985,25 @@ const OUTPUT_NOTE_SIZE_BYTES: usize = 700; /// every transaction after the one that did not fit. #[test] fn select_transactions_records_reports_truncation_below_payload_cap() { - let mut conn = create_db(); + let db = &TestDb::new(); let bob = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); let block1 = BlockNumber::from(1); let block2 = BlockNumber::from(2); - create_block(&mut conn, block1); - create_block(&mut conn, block2); - queries::upsert_accounts( - &mut conn, + create_block(db, block1); + create_block(db, block2); + upsert_accounts( + db, &[mock_block_account_update(bob, 0)], block1, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); - queries::upsert_accounts( - &mut conn, + upsert_accounts( + db, &[mock_block_account_update(bob, 1)], block2, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); @@ -3969,22 +4016,12 @@ fn select_transactions_records_reports_truncation_below_payload_cap() { let tx1 = mock_block_transaction_with_output_notes(bob, 1, block1_notes); let tx2 = mock_block_transaction_with_output_notes(bob, 2, block2_notes); - queries::insert_transactions( - &mut conn, - block1, - &OrderedTransactionHeaders::new_unchecked(vec![tx1.clone()]), - ) - .unwrap(); - queries::insert_transactions( - &mut conn, - block2, - &OrderedTransactionHeaders::new_unchecked(vec![tx2]), - ) - .unwrap(); + insert_transactions(db, block1, &OrderedTransactionHeaders::new_unchecked(vec![tx1.clone()])) + .unwrap(); + insert_transactions(db, block2, &OrderedTransactionHeaders::new_unchecked(vec![tx2])).unwrap(); let (last_block_included, records) = - queries::select_transactions_records(&mut conn, &[bob], BlockNumber::GENESIS..=block2) - .unwrap(); + select_transactions_records(db, &[bob], BlockNumber::GENESIS..=block2).unwrap(); assert_eq!(last_block_included, block1, "cursor must point at the last complete block"); assert_eq!(records.len(), 1, "only the complete block's transaction should be returned"); @@ -3996,31 +4033,25 @@ fn select_transactions_records_reports_truncation_below_payload_cap() { /// the query must surface an explicit error instead. #[test] fn select_transactions_records_errors_when_single_block_exceeds_payload_cap() { - let mut conn = create_db(); + let db = &TestDb::new(); let bob = AccountId::try_from(ACCOUNT_ID_PRIVATE_SENDER).unwrap(); let block1 = BlockNumber::from(1); - create_block(&mut conn, block1); - queries::upsert_accounts( - &mut conn, + create_block(db, block1); + upsert_accounts( + db, &[mock_block_account_update(bob, 0)], block1, - &queries::PrecomputedPublicAccountStates::new(), + &PrecomputedPublicAccountStates::new(), ) .unwrap(); let cap = miden_node_utils::limiter::MAX_RESPONSE_PAYLOAD_BYTES; let oversized_notes = cap / OUTPUT_NOTE_SIZE_BYTES + 100; let tx = mock_block_transaction_with_output_notes(bob, 1, oversized_notes); - queries::insert_transactions( - &mut conn, - block1, - &OrderedTransactionHeaders::new_unchecked(vec![tx]), - ) - .unwrap(); + insert_transactions(db, block1, &OrderedTransactionHeaders::new_unchecked(vec![tx])).unwrap(); - let result = - queries::select_transactions_records(&mut conn, &[bob], BlockNumber::GENESIS..=block1); + let result = select_transactions_records(db, &[bob], BlockNumber::GENESIS..=block1); assert_matches!( result, diff --git a/crates/store/src/db/utils.rs b/crates/store/src/db/utils.rs new file mode 100644 index 0000000000..0bb7d89887 --- /dev/null +++ b/crates/store/src/db/utils.rs @@ -0,0 +1,9 @@ +//! Small conversion helpers shared by the store's queries. + +use miden_protocol::note::Nullifier; + +/// Returns the high 16 bits of the provided nullifier. +pub fn get_nullifier_prefix(nullifier: &Nullifier) -> u16 { + // The shift leaves exactly the 16 bits the prefix is defined as. + (nullifier.most_significant_felt().as_canonical_u64() >> 48) as u16 +} diff --git a/crates/store/src/errors.rs b/crates/store/src/errors.rs index fb1f41843b..6d5c7bdf3b 100644 --- a/crates/store/src/errors.rs +++ b/crates/store/src/errors.rs @@ -1,5 +1,6 @@ use std::io; +use miden_node_db::DatabaseTypeConversionError; use miden_node_proto::domain::block::InvalidBlockRange; use miden_node_proto::errors::ConversionError; use miden_node_utils::limiter::QueryLimitError; @@ -25,8 +26,6 @@ use miden_protocol::transaction::OutputNote; use thiserror::Error; use tokio::sync::oneshot::error::RecvError; -use crate::db::models::conv::DatabaseTypeConversionError; - /// Errors produced while preparing or rebuilding account-state forest updates. /// /// The underlying [`LargeSmtForestError`] is preserved so callers can distinguish fatal backend diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index ec40d3fe3b..4c49d026b6 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -13,8 +13,6 @@ pub use accounts::PersistentAccountTree; pub use accounts::{AccountTreeWithHistory, HistoricalError, InMemoryAccountTree}; pub use blocks::BlockStore; pub use data_directory::DataDirectory; -pub use db::models::conv::SqlTypeConvert; -pub use db::models::queries::StorageMapValuesPage; pub use db::{ AccountVaultValue, DatabaseOptions, @@ -23,6 +21,7 @@ pub use db::{ NoteSyncRecord, NoteSyncUpdate, NullifierInfo, + StorageMapValuesPage, TransactionRecord, }; pub use errors::{ @@ -39,6 +38,7 @@ pub use errors::{ StateSyncError, }; pub use genesis::GenesisState; +pub use miden_node_db::SqlTypeConvert; pub use state::{ BlockWriter, LoadedState, @@ -65,18 +65,17 @@ pub fn default_sqlite_connection_pool_size() -> std::num::NonZeroUsize { /// This module is hidden from public docs and not part of the stable API. It exists so /// integration tests in sibling crates (e.g. `miden-node-rpc`) can seed network-account /// rows directly into the store's SQLite database without us widening the visibility of -/// internal diesel types. +/// the internal query layer. #[doc(hidden)] pub mod test_support { use std::path::Path; - use diesel::prelude::*; use miden_protocol::Word; use miden_protocol::account::AccountId; use miden_protocol::block::BlockNumber; - use crate::db::models::queries::{AccountRowInsert, NetworkAccountType}; - use crate::db::schema; + use crate::db::queries::{AccountRow, NetworkAccountType}; + use crate::errors::DatabaseError; /// Opens a fresh connection to the store's SQLite database and inserts a private /// network-account row for `account_id`, marking it as a network account in the @@ -85,20 +84,22 @@ pub mod test_support { /// Intended for integration tests that need to exercise the network-account gate /// without running a transaction through the block producer. The store's WAL mode /// makes a secondary connection safe. - pub fn seed_network_account(db_path: &Path, account_id: AccountId) { - let mut conn = SqliteConnection::establish(db_path.to_str().expect("db path is utf-8")) - .expect("connect to store sqlite"); + pub async fn seed_network_account(db_path: &Path, account_id: AccountId) { + let (writer, _reader) = + miden_node_db::sqlite::open(db_path).expect("connect to store sqlite"); - let row = AccountRowInsert::new_private( - account_id, - NetworkAccountType::Network, - Word::default(), - BlockNumber::from(0), - BlockNumber::from(0), - ); - diesel::insert_into(schema::accounts::table) - .values(&row) - .execute(&mut conn) + writer + .write::<_, DatabaseError, _>("seed network account", move |tx| { + AccountRow::new_private( + account_id, + NetworkAccountType::Network, + Word::default(), + BlockNumber::from(0), + BlockNumber::from(0), + ) + .upsert(tx) + }) + .await .expect("insert network account row"); } } diff --git a/crates/store/src/state/bootstrap.rs b/crates/store/src/state/bootstrap.rs index ae6cc65e30..26fa47e4c0 100644 --- a/crates/store/src/state/bootstrap.rs +++ b/crates/store/src/state/bootstrap.rs @@ -17,7 +17,7 @@ impl State { name = "store.bootstrap", err, )] - pub fn bootstrap(genesis: GenesisBlock, data_directory: &Path) -> anyhow::Result<()> { + pub async fn bootstrap(genesis: GenesisBlock, data_directory: &Path) -> anyhow::Result<()> { let data_directory = DataDirectory::load(data_directory.to_path_buf()).with_context(|| { format!("failed to load data directory at {}", data_directory.display()) @@ -32,7 +32,7 @@ impl State { tracing::debug!(target: LOG_TARGET, path=%block_store.display(), "Block store created"); let database_filepath = data_directory.database_path(); - Db::bootstrap(database_filepath.clone(), genesis).with_context(|| { + Db::bootstrap(database_filepath.clone(), genesis).await.with_context(|| { format!("failed to bootstrap database at {}", database_filepath.display()) })?; tracing::debug!(target: LOG_TARGET, path=%database_filepath.display(), "Database created"); diff --git a/crates/store/src/state/loader.rs b/crates/store/src/state/loader.rs index 7f5cb49c63..88a0ab2597 100644 --- a/crates/store/src/state/loader.rs +++ b/crates/store/src/state/loader.rs @@ -39,8 +39,7 @@ use crate::COMPONENT; #[cfg(feature = "rocksdb")] use crate::LOG_TARGET; use crate::account_state_forest::AccountStateForest; -use crate::db::Db; -use crate::db::models::queries::BlockHeaderCommitment; +use crate::db::{BlockHeaderCommitment, Db}; use crate::errors::{DatabaseError, StateInitializationError}; // CONSTANTS @@ -731,7 +730,6 @@ fn verify_account_state_forest_record( #[cfg(test)] mod tests { - use diesel::{ExpressionMethods, RunQueryDsl}; use miden_protocol::account::{ AccountId, AccountStorageHeader, @@ -743,7 +741,6 @@ mod tests { use miden_protocol::crypto::dsa::ecdsa_k256_keccak::SigningKey; use miden_protocol::crypto::merkle::mmr::Mmr; use miden_protocol::testing::account_id::ACCOUNT_ID_REGULAR_PUBLIC_ACCOUNT_IMMUTABLE_CODE; - use miden_protocol::utils::serde::Serializable; use super::*; @@ -808,27 +805,27 @@ mod tests { let signing_key = SigningKey::new(); let mut db = crate::db::Db::load(db_path).await.expect("test database should load"); - db.query("insert corrupted block headers", move |conn| { - for header in &headers { - let signatures = miden_protocol::block::BlockSignatures::new(vec![ - signing_key.sign(header.commitment()), - ]) - .expect("one signature is within bounds"); - crate::db::models::queries::insert_block_header(conn, header, &signatures)?; - } - - diesel::update(crate::db::schema::block_headers::table) - .filter(crate::db::schema::block_headers::block_num.eq(2_i64)) - .set( - crate::db::schema::block_headers::commitment - .eq(Word::from([42, 0, 0, 0u32]).to_bytes()), - ) - .execute(conn)?; - - Ok::<_, DatabaseError>(()) - }) - .await - .expect("test block headers should be inserted"); + db.writer() + .write::<_, DatabaseError, _>("insert corrupted block headers", move |tx| { + for header in &headers { + let signatures = miden_protocol::block::BlockSignatures::new(vec![ + signing_key.sign(header.commitment()), + ]) + .expect("one signature is within bounds"); + crate::db::queries::insert_block_header(tx, header, &signatures)?; + } + + // Corrupt the stored commitment of one header so it disagrees with the header it + // was stored alongside. + tx.execute( + "UPDATE block_headers SET commitment = ?1 WHERE block_num = ?2", + &[&Word::from([42, 0, 0, 0u32]), &BlockNumber::from(2)], + )?; + + Ok(()) + }) + .await + .expect("test block headers should be inserted"); let error = load_mmr(&mut db) .await diff --git a/crates/store/src/state/view/sync.rs b/crates/store/src/state/view/sync.rs index 603c078beb..2c3d1c700b 100644 --- a/crates/store/src/state/view/sync.rs +++ b/crates/store/src/state/view/sync.rs @@ -7,8 +7,7 @@ use miden_protocol::crypto::merkle::mmr::{Forest, MmrDelta, MmrProof}; use super::StateView; use crate::COMPONENT; -use crate::db::models::queries::StorageMapValuesPage; -use crate::db::{AccountVaultValue, NoteSyncUpdate, NullifierInfo}; +use crate::db::{AccountVaultValue, NoteSyncUpdate, NullifierInfo, StorageMapValuesPage}; use crate::errors::{DatabaseError, NoteSyncError, StateSyncError}; // STATE SYNCHRONIZATION ENDPOINTS