Distinguish grpc user errors from system errors

bin/ntx-builder/Cargo.toml

@@ -36,7 +36,6 @@ tokio                  = { features = ["macros", "net", "rt-multi-thread"], work
 tokio-stream           = { features = ["net"], workspace = true }
 tonic                  = { workspace = true }
 tonic-reflection       = { workspace = true }
-tower-http             = { workspace = true }
 tracing                = { workspace = true }
 url                    = { workspace = true }
 

bin/ntx-builder/src/server.rs

@@ -3,11 +3,10 @@ use miden_node_proto::server::ntx_builder_api;
 use miden_node_proto_build::ntx_builder_api_descriptor;
 use miden_node_utils::panic::{CatchPanicLayer, catch_panic_layer_fn};
 use miden_node_utils::shutdown::CancellationToken;
-use miden_node_utils::tracing::grpc::grpc_trace_fn;
+use miden_node_utils::tracing::grpc::grpc_trace_layer;
 use tokio::net::TcpListener;
 use tokio_stream::wrappers::TcpListenerStream;
 use tonic_reflection::server;
-use tower_http::trace::TraceLayer;
 
 use crate::LOG_TARGET;
 use crate::db::NtxDbReader;
@@ -57,7 +56,7 @@ impl NtxBuilderRpcServer {
 
         tonic::transport::Server::builder()
             .layer(CatchPanicLayer::custom(catch_panic_layer_fn))
-            .layer(TraceLayer::new_for_grpc().make_span_with(grpc_trace_fn))
+            .layer(grpc_trace_layer())
             .add_service(api_service)
             .add_service(reflection_service)
             .serve_with_incoming_shutdown(

bin/ntx-builder/src/server/get_network_note_status.rs

@@ -26,7 +26,7 @@ impl grpc::server::ntx_builder_api::GetNetworkNoteStatus for NtxBuilderRpcServer
         fields (
             note.id = %note_id,
         ),
-        err,
+        err(fault_only),
     )]
     async fn handle(
         &self,

bin/remote-prover/Cargo.toml

@@ -31,7 +31,7 @@ tonic                  = { default-features = false, features = ["codegen", "rou
 tonic-health           = { workspace = true }
 tonic-reflection       = { workspace = true }
 tonic-web              = { workspace = true }
-tower-http             = { features = ["trace"], workspace = true }
+tower-http             = { features = ["catch-panic"], workspace = true }
 tracing                = { workspace = true }
 
 [dev-dependencies]

bin/remote-prover/src/server/mod.rs

@@ -6,14 +6,13 @@ use miden_node_utils::cors::cors_for_grpc_web_layer;
 use miden_node_utils::logging::OpenTelemetry;
 use miden_node_utils::panic::catch_panic_layer_fn;
 use miden_node_utils::shutdown::CancellationToken;
-use miden_node_utils::tracing::grpc::grpc_trace_fn;
+use miden_node_utils::tracing::grpc::grpc_trace_layer;
 use proof_kind::ProofKind;
 use tokio::net::TcpListener;
 use tokio::task::JoinHandle;
 use tokio_stream::wrappers::TcpListenerStream;
 use tonic_web::GrpcWebLayer;
 use tower_http::catch_panic::CatchPanicLayer;
-use tower_http::trace::TraceLayer;
 
 use crate::LOG_TARGET;
 use crate::server::service::ProverService;
@@ -109,7 +108,7 @@ impl Server {
             .accept_http1(true)
             .timeout(self.timeout)
             .layer(CatchPanicLayer::custom(catch_panic_layer_fn))
-            .layer(TraceLayer::new_for_grpc().make_span_with(grpc_trace_fn))
+            .layer(grpc_trace_layer())
             .layer(cors_for_grpc_web_layer())
             .layer(GrpcWebLayer::new())
             .add_service(prover_service)

bin/remote-prover/src/server/prove.rs

@@ -15,7 +15,7 @@ impl grpc::server::remote_prover_api::Prove for ProverService {
     #[miden_instrument(
         target = COMPONENT,
         name = "remote_prover.prove",
-        err,
+        err(fault_only),
     )]
     async fn handle(
         &self,

bin/remote-prover/src/server/prover.rs

@@ -71,7 +71,7 @@ trait ProveRequest: Send + Sync {
     #[miden_instrument(
         target=COMPONENT,
         name="prove",
-        err,
+        err(fault_only),
     )]
     fn prove_request(&self, request: proto::ProofRequest) -> Result<proto::Proof, tonic::Status> {
         let input = Self::decode_request(request)?;
@@ -80,7 +80,7 @@ trait ProveRequest: Send + Sync {
 
     #[miden_instrument(
         target=COMPONENT,
-        err,
+        err(fault_only),
     )]
     fn decode_request(request: proto::ProofRequest) -> Result<Self::Input, tonic::Status> {
         use miden_protocol::utils::serde::Deserializable;

bin/remote-prover/src/server/service.rs

@@ -27,7 +27,7 @@ impl ProverService {
 
     #[miden_instrument(
         target=COMPONENT,
-        err,
+        err(fault_only),
     )]
     pub(super) fn acquire_permit(&self) -> Result<OwnedSemaphorePermit, tonic::Status> {
         Arc::clone(&self.permits)

bin/validator/Cargo.toml

@@ -51,7 +51,7 @@ tokio-stream           = { features = ["net"], workspace = true }
 toml                   = { workspace = true }
 tonic                  = { default-features = true, features = ["transport"], workspace = true }
 tonic-reflection       = { workspace = true }
-tower-http             = { features = ["util"], workspace = true }
+tower-http             = { features = ["catch-panic"], workspace = true }
 tracing                = { workspace = true }
 zeroize                = { workspace = true }
 

bin/validator/src/server/mod.rs

@@ -7,11 +7,10 @@ use miden_node_store::BlockStore;
 use miden_node_utils::clap::GrpcOptions;
 use miden_node_utils::panic::catch_panic_layer_fn;
 use miden_node_utils::shutdown::CancellationToken;
-use miden_node_utils::tracing::grpc::grpc_trace_fn;
+use miden_node_utils::tracing::grpc::grpc_trace_layer;
 use tokio::net::TcpListener;
 use tokio_stream::wrappers::TcpListenerStream;
 use tower_http::catch_panic::CatchPanicLayer;
-use tower_http::trace::TraceLayer;
 
 use crate::db::{ValidatorDbReader, ValidatorDbWriter};
 use crate::{
@@ -153,7 +152,7 @@ impl ValidatorServer {
         // Build the gRPC server with the API service and trace layer.
         tonic::transport::Server::builder()
             .layer(CatchPanicLayer::custom(catch_panic_layer_fn))
-            .layer(TraceLayer::new_for_grpc().make_span_with(grpc_trace_fn))
+            .layer(grpc_trace_layer())
             .timeout(self.grpc_options.request_timeout)
             .add_service(validator_api::service(service))
             .add_service(reflection_service)

bin/validator/src/server/validator_service/block_subscription.rs

@@ -48,7 +48,7 @@ impl grpc::server::validator_api::BlockSubscription for ValidatorService {
     #[miden_instrument(
         target = COMPONENT,
         name = "validator.block_subscription",
-        err,
+        err(fault_only),
     )]
     async fn handle(
         &self,

bin/validator/src/server/validator_service/get_transaction_encryption_key.rs

@@ -21,7 +21,7 @@ impl grpc::server::validator_api::GetTransactionEncryptionKey for ValidatorServi
     #[miden_instrument(
         target = COMPONENT,
         name = "get_transaction_encryption_key",
-        err,
+        err(fault_only),
     )]
     async fn handle(
         &self,

bin/validator/src/server/validator_service/submit_proven_transaction.rs

@@ -23,7 +23,7 @@ impl grpc::server::validator_api::SubmitProvenTransaction for ValidatorService {
     #[miden_instrument(
         target = COMPONENT,
         name = "submit_proven_transaction",
-        err,
+        err(fault_only),
     )]
     async fn handle(
         &self,

crates/block-producer/src/errors.rs

@@ -62,15 +62,18 @@ pub enum MempoolSubmissionError {
     #[error(
         "transaction expired at block height {expired_at} but the block height limit was {limit}"
     )]
+    #[grpc(failed_precondition)]
     Expired {
         expired_at: BlockNumber,
         limit: BlockNumber,
     },
 
     #[error("transaction conflicts with current mempool state")]
+    #[grpc(failed_precondition)]
     StateConflict(#[source] StateConflict),
 
     #[error("the mempool is at capacity")]
+    #[grpc(resource_exhausted)]
     CapacityExceeded,
 
     #[error("mempool lock is poisoned")]

crates/block-producer/src/server/mod.rs

@@ -311,7 +311,7 @@ impl BlockProducerApi {
     #[miden_instrument(
         target = COMPONENT,
         name = "block_producer.api.submit_proven_tx",
-        err,
+        err(fault_only),
     )]
     pub async fn submit_proven_tx(
         &self,
@@ -344,7 +344,7 @@ impl BlockProducerApi {
     #[miden_instrument(
         target = COMPONENT,
         name = "block_producer.api.submit_authenticated_tx",
-        err,
+        err(fault_only),
     )]
     #[expect(clippy::let_and_return, reason = "required to lengthen arc lifetime")]
     pub async fn submit_authenticated_tx(
@@ -363,7 +363,7 @@ impl BlockProducerApi {
     #[miden_instrument(
         target = COMPONENT,
         name = "block_producer.api.submit_proven_tx_batch",
-        err,
+        err(fault_only),
     )]
     pub async fn submit_proven_tx_batch(
         &self,
@@ -395,7 +395,7 @@ impl BlockProducerApi {
     #[miden_instrument(
         target = COMPONENT,
         name = "block_producer.api.submit_authenticated_tx_batch",
-        err,
+        err(fault_only),
     )]
     #[expect(clippy::let_and_return)]
     pub async fn submit_authenticated_tx_batch(

crates/grpc-error-macro/Cargo.toml

@@ -20,5 +20,6 @@ proc-macro = true
 test       = false
 
 [dependencies]
-quote = { workspace = true }
-syn   = { features = ["full"], workspace = true }
+proc-macro2 = { workspace = true }
+quote       = { workspace = true }
+syn         = { features = ["full"], workspace = true }