Delete local storage in a background task

dev-tools/omdb/src/bin/omdb/db.rs

@@ -2649,11 +2649,11 @@ async fn cmd_db_disk_info(
     datastore: &DataStore,
     args: &DiskInfoArgs,
 ) -> Result<(), anyhow::Error> {
+    let conn = datastore.pool_connection_for_tests().await?;
+
     let disk = {
         use nexus_db_schema::schema::disk::dsl;
 
-        let conn = datastore.pool_connection_for_tests().await?;
-
         dsl::disk
             .filter(dsl::id.eq(args.uuid))
             .select(nexus_db_model::Disk::as_select())
@@ -2662,7 +2662,7 @@ async fn cmd_db_disk_info(
             .context("failed to find disk")?
     };
 
-    match datastore.disk_get_with_model(opctx, disk).await? {
+    match datastore.disk_get_with_model_on_connection(&conn, disk).await? {
         Disk::Crucible(disk) => {
             crucible_disk_info(opctx, datastore, disk).await
         }

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -71,6 +71,7 @@ use nexus_types::internal_api::background::IncompleteBootstoreConfigReport;
 use nexus_types::internal_api::background::InstanceReincarnationStatus;
 use nexus_types::internal_api::background::InstanceUpdaterStatus;
 use nexus_types::internal_api::background::InventoryLoadStatus;
+use nexus_types::internal_api::background::LocalStorageDeleteStatus;
 use nexus_types::internal_api::background::LookupRegionPortStatus;
 use nexus_types::internal_api::background::PhysicalDiskAdoptionStatus;
 use nexus_types::internal_api::background::ProbeDistributorStatus;
@@ -1433,6 +1434,9 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) {
         "switch_port_config_manager" => {
             print_task_switch_port_settings_manager(details);
         }
+        "local_storage_delete" => {
+            print_task_local_storage_delete(details);
+        }
         _ => {
             println!(
                 "warning: unknown background task: {:?} \
@@ -4386,6 +4390,50 @@ fn print_task_physical_disk_adoption(details: &serde_json::Value) {
     }
 }
 
+fn print_task_local_storage_delete(details: &serde_json::Value) {
+    match serde_json::from_value::<LocalStorageDeleteStatus>(details.clone()) {
+        Err(error) => eprintln!(
+            "warning: failed to interpret task details: {:?}: {:?}",
+            error, details
+        ),
+
+        Ok(status) => {
+            let LocalStorageDeleteStatus {
+                total_allocations_to_delete,
+                page_size,
+                delete_results,
+                deallocate_results,
+                errors,
+            } = &status;
+
+            println!(
+                "    total allocations left to delete: \
+                {total_allocations_to_delete}"
+            );
+
+            println!(
+                "    number of allocations deleted per invoked task: \
+                {page_size}"
+            );
+
+            println!("    results of deleting local storage:");
+            for result in delete_results {
+                println!("    > {result}");
+            }
+
+            println!("    results of deallocating local storage:");
+            for result in deallocate_results {
+                println!("    > {result}");
+            }
+
+            println!("    errors: {}", errors.len());
+            for error in errors {
+                println!("    > {error}");
+            }
+        }
+    }
+}
+
 const ERRICON: &str = "/!\\";
 
 fn warn_if_nonzero(n: usize) -> &'static str {

dev-tools/omdb/tests/env.out

@@ -158,6 +158,10 @@ task: "inventory_loader"
     loads the latest inventory collection from the DB
 
 
+task: "local_storage_delete"
+    delete resources for disks backed by local storage
+
+
 task: "lookup_region_port"
     fill in missing ports for region records
 
@@ -430,6 +434,10 @@ task: "inventory_loader"
     loads the latest inventory collection from the DB
 
 
+task: "local_storage_delete"
+    delete resources for disks backed by local storage
+
+
 task: "lookup_region_port"
     fill in missing ports for region records
 
@@ -689,6 +697,10 @@ task: "inventory_loader"
     loads the latest inventory collection from the DB
 
 
+task: "local_storage_delete"
+    delete resources for disks backed by local storage
+
+
 task: "lookup_region_port"
     fill in missing ports for region records
 

dev-tools/omdb/tests/successes.out

@@ -402,6 +402,10 @@ task: "inventory_loader"
     loads the latest inventory collection from the DB
 
 
+task: "local_storage_delete"
+    delete resources for disks backed by local storage
+
+
 task: "lookup_region_port"
     fill in missing ports for region records
 
@@ -883,6 +887,16 @@ task: "inventory_loader"
     loaded latest inventory collection as of <REDACTED_TIMESTAMP>:
         collection ..........<REDACTED_UUID>..........., taken at <REDACTED_TIMESTAMP>
 
+task: "local_storage_delete"
+  configured period: every <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
+  last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    total allocations left to delete: 0
+    number of allocations deleted per invoked task: 128
+    results of deleting local storage:
+    results of deallocating local storage:
+    errors: 0
+
 task: "lookup_region_port"
   configured period: every <REDACTED_DURATION>m
   last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
@@ -1611,6 +1625,16 @@ task: "inventory_loader"
     loaded latest inventory collection as of <REDACTED_TIMESTAMP>:
         collection ..........<REDACTED_UUID>..........., taken at <REDACTED_TIMESTAMP>
 
+task: "local_storage_delete"
+  configured period: every <REDACTED_DURATION>h <REDACTED_DURATION>m <REDACTED_DURATION>s
+  last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>
+    started at <REDACTED_TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+    total allocations left to delete: 0
+    number of allocations deleted per invoked task: 128
+    results of deleting local storage:
+    results of deallocating local storage:
+    errors: 0
+
 task: "lookup_region_port"
   configured period: every <REDACTED_DURATION>m
   last completed activation: <REDACTED ITERATIONS>, triggered by <TRIGGERED_BY_REDACTED>

nexus-config/src/nexus_config.rs

@@ -486,6 +486,8 @@ pub struct BackgroundTaskConfig {
     pub audit_log_cleanup: AuditLogCleanupConfig,
     /// configuration for populate switch ports task
     pub populate_switch_ports: PopulateSwitchPortsConfig,
+    /// configuration for local storage delete task
+    pub local_storage_delete: LocalStorageDeleteConfig,
 }
 
 #[serde_as]
@@ -1100,6 +1102,14 @@ pub struct TrustQuorumConfig {
     pub period_secs: Duration,
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct LocalStorageDeleteConfig {
+    /// period (in seconds) for periodic activations of this background task
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs: Duration,
+}
+
 /// Configuration for a nexus server
 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
 pub struct PackageConfig {
@@ -1393,6 +1403,7 @@ mod test {
             audit_log_cleanup.retention_days = 90
             audit_log_cleanup.max_deleted_per_activation = 10000
             populate_switch_ports.period_secs = 31
+            local_storage_delete.period_secs = 30
             [default_region_allocation_strategy]
             type = "random"
             seed = 0
@@ -1679,6 +1690,10 @@ mod test {
                         populate_switch_ports: PopulateSwitchPortsConfig {
                             period_secs: Duration::from_secs(31),
                         },
+                        local_storage_delete:
+                            LocalStorageDeleteConfig {
+                                period_secs: Duration::from_secs(30),
+                            },
                     },
                     multicast: MulticastConfig { enabled: false },
                     default_region_allocation_strategy:
@@ -1796,6 +1811,7 @@ mod test {
             audit_log_cleanup.retention_days = 90
             audit_log_cleanup.max_deleted_per_activation = 10000
             populate_switch_ports.period_secs = 31
+            local_storage_delete.period_secs = 30
 
             [default_region_allocation_strategy]
             type = "random"

nexus/background-task-interface/src/init.rs

@@ -64,6 +64,7 @@ pub struct BackgroundTasks {
     pub task_attached_subnet_manager: Activator,
     pub task_session_cleanup: Activator,
     pub task_populate_switch_ports: Activator,
+    pub task_local_storage_delete: Activator,
 
     // Handles to activate background tasks that do not get used by Nexus
     // at-large.  These background tasks are implementation details as far as

nexus/db-queries/src/db/datastore/disk.rs

@@ -34,6 +34,8 @@ use crate::db::model::Volume;
 use crate::db::model::to_db_typed_uuid;
 use crate::db::pagination::paginated;
 use crate::db::queries::disk::DiskSetClauseForAttach;
+use crate::db::queries::virtual_provisioning_collection_update;
+use crate::db::queries::virtual_provisioning_collection_update::*;
 use crate::db::update_and_check::UpdateAndCheck;
 use crate::db::update_and_check::UpdateStatus;
 use async_bb8_diesel::AsyncRunQueryDsl;
@@ -52,6 +54,7 @@ use nexus_types::identity::Asset;
 use omicron_common::api;
 use omicron_common::api::external;
 use omicron_common::api::external::CreateResult;
+use omicron_common::api::external::DeleteResult;
 use omicron_common::api::external::Error;
 use omicron_common::api::external::ListResultVec;
 use omicron_common::api::external::LookupResult;
@@ -510,31 +513,33 @@ impl DataStore {
         let (.., disk) =
             LookupPath::new(opctx, self).disk_id(disk_id).fetch().await?;
 
-        self.disk_get_with_model(opctx, disk).await
+        let conn = self.pool_connection_authorized(opctx).await?;
+
+        self.disk_get_with_model_on_connection(&conn, disk).await
     }
 
     /// Return a `datastore::Disk` given a `model::Disk`
     ///
     /// Note: basically all of Nexus should _not_ be using this, and should be
     /// using `disk_get` instead: this version of the function bypasses the
     /// LookupPath induced permissions check and should only called from omdb.
-    pub async fn disk_get_with_model(
+    /// Code that is looking up deleted disks should also use this method, as
+    /// `LookupPath` will not return deleted resources.
+    pub async fn disk_get_with_model_on_connection(
         &self,
-        opctx: &OpContext,
+        conn: &async_bb8_diesel::Connection<DbConnection>,
         disk: model::Disk,
     ) -> LookupResult<Disk> {
         let disk_id = disk.id();
 
-        let conn = self.pool_connection_authorized(opctx).await?;
-
         let disk = match disk.disk_type {
             db::model::DiskType::Crucible => {
                 use nexus_db_schema::schema::disk_type_crucible::dsl;
 
                 let disk_type_crucible = dsl::disk_type_crucible
                     .filter(dsl::disk_id.eq(disk_id))
                     .select(DiskTypeCrucible::as_select())
-                    .first_async(&*conn)
+                    .first_async(conn)
                     .await
                     .map_err(|e| {
                         public_error_from_diesel(e, ErrorHandler::Server)
@@ -552,7 +557,7 @@ impl DataStore {
                 let disk_type_local_storage = dsl::disk_type_local_storage
                     .filter(dsl::disk_id.eq(disk_id))
                     .select(DiskTypeLocalStorage::as_select())
-                    .first_async(&*conn)
+                    .first_async(conn)
                     .await
                     .map_err(|e| {
                         public_error_from_diesel(e, ErrorHandler::Server)
@@ -1549,8 +1554,22 @@ impl DataStore {
         disk_id: &Uuid,
         ok_to_delete_states: &[api::external::DiskState],
     ) -> Result<model::Disk, Error> {
-        use nexus_db_schema::schema::disk::dsl;
         let conn = self.pool_connection_unauthorized().await?;
+
+        Self::project_delete_disk_no_auth_on_connection(
+            &conn,
+            disk_id,
+            ok_to_delete_states,
+        )
+        .await
+    }
+
+    async fn project_delete_disk_no_auth_on_connection(
+        conn: &async_bb8_diesel::Connection<DbConnection>,
+        disk_id: &Uuid,
+        ok_to_delete_states: &[api::external::DiskState],
+    ) -> Result<model::Disk, Error> {
+        use nexus_db_schema::schema::disk::dsl;
         let now = Utc::now();
 
         let ok_to_delete_state_labels: Vec<_> =
@@ -2083,6 +2102,63 @@ impl DataStore {
 
         Ok(disk)
     }
+
+    /// In a single transaction, set time_deleted for a disk and delete the
+    /// storage from the appropriate virtual provisioning collection.
+    pub async fn delete_disk_and_update_provisioning_collection(
+        &self,
+        opctx: &OpContext,
+        project: &authz::Project,
+        disk: &Disk,
+        ok_to_delete_states: &[api::external::DiskState],
+    ) -> DeleteResult {
+        let err = OptionalError::new();
+        let conn = self.pool_connection_authorized(opctx).await?;
+
+        let provisions = self
+            .transaction_retry_wrapper(
+                "delete_disk_and_update_provisioning_collection",
+            )
+            .transaction(&conn, |conn| {
+                let err = err.clone();
+                async move {
+                    Self::project_delete_disk_no_auth_on_connection(
+                        &conn,
+                        &disk.id(),
+                        ok_to_delete_states,
+                    )
+                    .await
+                    .map_err(|e| err.bail(e))?;
+
+                    let provisions =
+                        VirtualProvisioningCollectionUpdate::new_delete_storage(
+                            disk.id(),
+                            disk.size(),
+                            project.id(),
+                        )
+                        .get_results_async(&conn)
+                        .await
+                        .map_err(|e| err.bail(
+                            virtual_provisioning_collection_update::from_diesel(e)
+                        ))?;
+
+                    Ok(provisions)
+                }
+            })
+            .await
+            .map_err(|e| {
+                if let Some(err) = err.take() {
+                    err
+                } else {
+                    public_error_from_diesel(e, ErrorHandler::Server)
+                }
+            })?;
+
+        self.virtual_provisioning_collection_producer
+            .append_disk_metrics(&provisions)?;
+
+        Ok(())
+    }
 }
 
 #[cfg(test)]