diff --git a/.env.example b/.env.example index 6deabd08a..5d8c96948 100644 --- a/.env.example +++ b/.env.example @@ -7,6 +7,11 @@ MCP_REGISTRY_VERSION=dev # Database configuration MCP_REGISTRY_DATABASE_URL=postgres://username:password@localhost:5432/mcp-registry +# Per-app limits. Budget max_connections across all replicas, rollout surge, +# admin clients and background work before increasing replica count. +MCP_REGISTRY_DATABASE_MAX_CONNECTIONS=60 +MCP_REGISTRY_DATABASE_MIN_CONNECTIONS=10 + # Path or URL to import seed data (supports local files and HTTP URLs) # For offline development, use: data/seed.json MCP_REGISTRY_SEED_FROM=https://registry.modelcontextprotocol.io/v0/servers diff --git a/cmd/registry/main.go b/cmd/registry/main.go index bb91bb58f..4101375e1 100644 --- a/cmd/registry/main.go +++ b/cmd/registry/main.go @@ -56,6 +56,17 @@ func main() { // Initialize configuration cfg := config.NewConfig() + poolLimits := database.PoolLimits{MaxConns: cfg.DatabaseMaxConnections, MinConns: cfg.DatabaseMinConnections} + if err := poolLimits.Validate(); err != nil { + log.Fatalf("Invalid database pool configuration: %v", err) + } + + stopProfiling, err := telemetry.StartProfiling(os.Getenv("MCP_REGISTRY_ENABLE_PPROF") == "true") + if err != nil { + log.Printf("Failed to start profiling: %v", err) + return + } + defer stopProfiling() // Initialize telemetry before opening the instrumented database pool. shutdownTelemetry, metrics, err := telemetry.InitMetrics(cfg.Version) @@ -83,7 +94,7 @@ func main() { backoff := initialBackoff for attempt := 1; ; attempt++ { attemptCtx, attemptCancel := context.WithTimeout(context.Background(), perAttemptTimeout) - db, err = database.NewPostgreSQL(attemptCtx, cfg.DatabaseURL) + db, err = database.NewPostgreSQLWithPoolLimits(attemptCtx, cfg.DatabaseURL, poolLimits) attemptCancel() if err == nil { break diff --git a/deploy/Pulumi.gcpStaging.yaml b/deploy/Pulumi.gcpStaging.yaml index 5975ebe39..49fcdfcaf 100644 --- a/deploy/Pulumi.gcpStaging.yaml +++ b/deploy/Pulumi.gcpStaging.yaml @@ -14,4 +14,6 @@ config: secure: v1:0NSeI0qWrdcHeVfh:P56IEP/700/e839TSbF7Ns2j2orZnD5cRNXohjxCPOKyIDYn+0bmiTuk8pyFnFk8WibS/w7M2FFgv3/BL0Djo0XMVe9tq+7HjLN4tPEJnCU= mcp-registry:googleOauthClientSecret: secure: v1:MfA48Ri4VssQwUi7:TUF9sBGIgTnBUhePfEF7L+h1VVn5e6j2uGLbOwbGbjyBO9Z72SKC+z/21xHmJtqUSr2t + mcp-registry:burstAutoscaling: "true" + mcp-registry:postgresCPURequest: "2" encryptionsalt: v1:EKBwmTmss1c=:v1:JLd4a7cM0X8Jroh+:z9/q6RSCEFTDzMV6X6h5Tpbw0tnpkA== diff --git a/deploy/README.md b/deploy/README.md index 098a6a45b..f59c5de39 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -632,3 +632,102 @@ kubectl logs -l app=postgres kubectl describe schedule.k8up.io kubectl get backup ``` + +### Burst capacity (staging validation) + +The additional `coordinatedCapacity` opt-in is staged separately and currently +disabled. See the [candidate and test gates](../tools/ingressbench/COORDINATED_CAPACITY.md) +for its 2–4 app HPA, 30/5 pools and required CPU settings. It is rejected outside +staging until the full validation passes. The configuration below describes the +earlier ingress-only profile. + +This is an experimental configuration, not an approved production fix. In the +2026-10-27 staging comparison, CPU reservation protected database readiness, but +ingress/node scaling alone left long application pool queues and publish +timeouts. Adding a 60-second client connection lifetime did not resolve those +failures. The ingress HPA could remain below its average CPU target while the +database node was saturated and the app on that node queued requests. Do not +enable this in production without a successful end-to-end burst validation. +See `tools/ingressbench/VALIDATION.md` for the measured comparison and limitations. +The follow-up [ready-capacity comparison](../tools/ingressbench/WARM_CAPACITY_VALIDATION.md) +also failed with four nodes and six ingress replicas ready before the burst. +Keep the rollout gate on complete offered load and recovery, not only successful +HTTP attempts or quiet current metrics. + +`mcp-registry:registryCPURequest` can override the application CPU reservation +for controlled staging comparisons. The default remains 250m per hosted app +replica (100m locally), with no CPU limit. Raising the reservation changes both +CPU sharing under contention and pod placement; it can prevent consolidation +onto two nodes even when the autoscaler's configured minimum remains two. +Measure node count and placement through recovery, not just request latency. +The replica count and each pool's 60 maximum / 10 minimum remain unchanged. + +`mcp-registry:burstAutoscaling` opts a hosted stack into ingress and GKE node +scaling. It is enabled in staging for validation; production keeps its existing +configuration until a separate promotion. Application replicas and their +connection pools stay fixed at two replicas, each with a maximum of 60 connections. + +| Setting | Fixed baseline | With burst capacity | +| --- | --- | --- | +| GKE nodes | 2 | 2–4, same machine type | +| Ingress controllers | 2 | 2–6 | +| Ingress CPU request | 300m | 800m, no CPU limit | +| Ingress memory request | 90 MiB | 512 MiB | +| Registry DB CPU request | 200m | Configurable separately with `postgresCPURequest` | +| Registry DB CPU limit | 3 | Unchanged | + +The HPA targets 88% of the 800m ingress request (704m), including work serving +cache hits. The request leaves room for the 2-CPU database and existing system +workloads on a baseline node; a 1-CPU ingress request did not fit that placement. +A rolling-update surge can still require a temporary extra node. Scale-up allows +two additional controllers per minute. Scale-down waits +for a ten-minute stabilization window, then removes at most one controller every +two minutes. These are bounds, not a guarantee of immediate capacity: metric +collection, pending-pod detection, node provisioning, readiness and load-balancer +health checks all add delay. The cluster autoscaler adds nodes for unschedulable +pods; high node CPU alone does not trigger it. It can remove spare nodes later, +subject to pod placement, disruption budgets and its own delays. + +Both singleton PostgreSQL clusters inherit `safe-to-evict=false` on their pods +and enable CloudNativePG disruption budgets. This protects them from voluntary +autoscaler eviction, not node failure, operator restarts or manual deletion. +It can also block node maintenance; plan explicit database interruption when +upgrading nodes. Two protected database placements can constrain the minimum +node count. Verify actual scale-down rather than assuming the cluster reaches two. + +`postgresCPURequest` reserves scheduling capacity and increases relative CPU +shares under contention. It is not a dedicated-core guarantee. Raising the +request can restart the singleton database, and the request must fit beneath +its CPU limit and alongside other workloads. Validate a request of 2 CPUs in +staging before promoting it. The initial staging rollout interrupted application +readiness while the singleton database restarted and moved to a new node. Plan +that interruption explicitly; ensure spare schedulable capacity is ready before +changing the database request. Memory, buffers, connection limits and readiness +thresholds are unchanged. + +Each ingress has an independent cache. New replicas start cold and can increase +origin traffic before warming. Existing client connections may stay on older +replicas. With burst capacity enabled, client connections have a 60-second +active lifetime (`keepalive_time`); NGINX finishes the current request before +closing so clients can reconnect to newly added capacity. This differs from the +idle keep-alive timeout and does not abort a long-running request at 60 seconds. +It adds connection/TLS setup work, so validate both HTTP/1.1 and HTTP/2. +Rate limits are also per controller, so the aggregate allowance for a +client can rise as replicas are added; this does not introduce a global limiter. +Ingress metrics are scraped per pod every 30 seconds when burst capacity is on, +so cache-serving demand is visible separately from application arrivals. Cache +outcomes still require response headers or the existing ingress logs. + +Before an update, refresh Pulumi state and inspect the full preview. The node +pool must update in place. Pulumi ignores `nodeCount` and ingress `spec.replicas` +while autoscaling is enabled, so subsequent deploys do not reset controller-owned +counts. Verify this again with a preview while scaled out. A configuration-map +replacement for the metrics scrape config is expected. + +Validate the same sustained workload before and after: per-replica demand, +cache misses, listing and publish latency, errors, pool queues, PostgreSQL CPU +pressure/throttling, readiness, pending pods and node allocation. Require both +scale-up under load and recovery/scale-down after it stops. Do not promote based +on a quiet run or a synthetic workload that fails to reproduce the failure. +To disable autoscaling, remove the opt-in and preview the transition back to +fixed counts; scale-in and a database request rollback may interrupt service. diff --git a/deploy/main.go b/deploy/main.go index 0c778458c..c208d858b 100644 --- a/deploy/main.go +++ b/deploy/main.go @@ -36,6 +36,11 @@ func main() { conf := config.New(ctx, "mcp-registry") environment := conf.Require("environment") + // Reject incomplete experimental capacity profiles before any resources. + if err := k8s.ValidateCapacityConfig(ctx); err != nil { + return err + } + // Create provider provider, err := createProvider(ctx) if err != nil { diff --git a/deploy/pkg/k8s/autoscaling.go b/deploy/pkg/k8s/autoscaling.go new file mode 100644 index 000000000..d836ebc18 --- /dev/null +++ b/deploy/pkg/k8s/autoscaling.go @@ -0,0 +1,85 @@ +package k8s + +import ( + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi/config" +) + +// ingressAutoscalingValues is consumed by ingress-nginx chart 4.13.0. With +// requests.cpu=800m, 88% means 704m average CPU per controller, including cache hits. +// +//nolint:goconst // Keep Helm schema keys literal alongside their values. +func ingressAutoscalingValues() pulumi.Map { + return pulumi.Map{ + "enabled": pulumi.Bool(true), + "minReplicas": pulumi.Int(2), + "maxReplicas": pulumi.Int(6), + "targetCPUUtilizationPercentage": pulumi.Int(88), + // The chart defaults memory to 50%; cache memory persists after a burst + // and would hold replicas up even when CPU demand has subsided. + "targetMemoryUtilizationPercentage": pulumi.Int(0), + "behavior": pulumi.Map{ + "scaleUp": pulumi.Map{ + "stabilizationWindowSeconds": pulumi.Int(0), + "policies": pulumi.Array{pulumi.Map{"type": pulumi.String("Pods"), "value": pulumi.Int(2), "periodSeconds": pulumi.Int(60)}}, + }, + "scaleDown": pulumi.Map{ + "stabilizationWindowSeconds": pulumi.Int(600), + "policies": pulumi.Array{pulumi.Map{"type": pulumi.String("Pods"), "value": pulumi.Int(1), "periodSeconds": pulumi.Int(120)}}, + }, + }, + } +} + +// protectDatabase covers both singleton databases. A node containing either +// database must not be evicted to save capacity. This is not HA and does not +// prevent node failure or operator-managed restarts. CNPG propagates inherited +// metadata to its generated pods; setting this only on the Cluster is ineffective. +func protectDatabase(ctx *pulumi.Context, spec map[string]any) map[string]any { + if config.New(ctx, "mcp-registry").GetBool("burstAutoscaling") { + spec["enablePDB"] = true + spec["inheritedMetadata"] = map[string]any{ + "annotations": map[string]any{"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, + } + } + return spec +} + +// Discover each controller pod, rather than scraping the load-balanced service, +// so arrivals, errors and latency include cache hits on every replica. +// +//nolint:goconst // Keep Prometheus configuration keys literal. +func ingressScrapeConfig() pulumi.Map { + return pulumi.Map{ + "job_name": pulumi.String("ingress-nginx"), + "scrape_interval": pulumi.String("30s"), + "kubernetes_sd_configs": pulumi.Array{pulumi.Map{ + "role": pulumi.String("pod"), + "namespaces": pulumi.Map{"names": pulumi.StringArray{pulumi.String("ingress-nginx")}}, + }}, + "relabel_configs": pulumi.Array{ + pulumi.Map{"source_labels": pulumi.StringArray{pulumi.String("__meta_kubernetes_pod_label_app_kubernetes_io_name")}, "regex": pulumi.String("ingress-nginx"), "action": pulumi.String("keep")}, + pulumi.Map{"source_labels": pulumi.StringArray{pulumi.String("__meta_kubernetes_pod_container_port_name")}, "regex": pulumi.String("metrics"), "action": pulumi.String("keep")}, + pulumi.Map{"source_labels": pulumi.StringArray{pulumi.String("__meta_kubernetes_pod_name")}, "target_label": pulumi.String("pod")}, + }, + } +} + +// These options must reach chart children, not just the Chart component. +func burstResourceTransform(args *pulumi.ResourceTransformationArgs) *pulumi.ResourceTransformationResult { + switch { + case args.Type == "kubernetes:apps/v1:Deployment" && args.Name == "ingress-nginx/ingress-nginx-controller": + return &pulumi.ResourceTransformationResult{Props: args.Props, Opts: append(args.Opts, pulumi.IgnoreChanges([]string{"spec.replicas"}))} + case args.Type == "kubernetes:core/v1:ConfigMap" && (args.Name == "monitoring/victoria-metrics-agent-config" || args.Name == "ingress-nginx/ingress-nginx-controller"): + return &pulumi.ResourceTransformationResult{Props: args.Props, Opts: append(args.Opts, pulumi.DeleteBeforeReplace(true))} + default: + return nil + } +} + +func withIngressScrape(ctx *pulumi.Context, configs pulumi.Array) pulumi.Array { + if config.New(ctx, "mcp-registry").GetBool("burstAutoscaling") { + return append(configs, ingressScrapeConfig()) + } + return configs +} diff --git a/deploy/pkg/k8s/autoscaling_test.go b/deploy/pkg/k8s/autoscaling_test.go new file mode 100644 index 000000000..f5fb13316 --- /dev/null +++ b/deploy/pkg/k8s/autoscaling_test.go @@ -0,0 +1,58 @@ +package k8s //nolint:testpackage // Verify the unexported Pulumi child transformation. + +import ( + "testing" + + "github.com/pulumi/pulumi/sdk/v3/go/common/resource" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" +) + +type ownershipMonitor struct{ t *testing.T } + +func (m ownershipMonitor) Call(a pulumi.MockCallArgs) (resource.PropertyMap, error) { + return a.Args, nil +} +func (m ownershipMonitor) NewResource(a pulumi.MockResourceArgs) (string, resource.PropertyMap, error) { + if a.RegisterRPC != nil { + switch a.TypeToken + "::" + a.Name { + case "kubernetes:apps/v1:Deployment::ingress-nginx/ingress-nginx-controller": + ignored := a.RegisterRPC.GetIgnoreChanges() + if len(ignored) != 1 || ignored[0] != "spec.replicas" { + m.t.Errorf("HPA ownership did not reach chart child: %v", ignored) + } + case "kubernetes:core/v1:ConfigMap::monitoring/victoria-metrics-agent-config", "kubernetes:core/v1:ConfigMap::ingress-nginx/ingress-nginx-controller": + if !a.RegisterRPC.GetDeleteBeforeReplace() { + m.t.Error("fixed-name ConfigMap would be created before deletion") + } + case "kubernetes:apps/v1:Deployment::unrelated-deployment": + if len(a.RegisterRPC.GetIgnoreChanges()) != 0 { + m.t.Error("unrelated replicas must remain managed") + } + } + } + return a.Name + "-id", a.Inputs, nil +} + +func TestChartChildrenKeepControllerOwnership(t *testing.T) { + err := pulumi.RunErr(func(ctx *pulumi.Context) error { + var parent pulumi.ResourceState + if err := ctx.RegisterComponentResource("kubernetes:helm.sh/v3:Chart", "chart", &parent, pulumi.Transformations([]pulumi.ResourceTransformation{burstResourceTransform})); err != nil { + return err + } + for _, r := range []struct{ typ, name string }{ + {"kubernetes:apps/v1:Deployment", "ingress-nginx/ingress-nginx-controller"}, + {"kubernetes:core/v1:ConfigMap", "monitoring/victoria-metrics-agent-config"}, + {"kubernetes:core/v1:ConfigMap", "ingress-nginx/ingress-nginx-controller"}, + {"kubernetes:apps/v1:Deployment", "unrelated-deployment"}, + } { + var child pulumi.CustomResourceState + if err := ctx.RegisterResource(r.typ, r.name, pulumi.Map{}, &child, pulumi.Parent(&parent)); err != nil { + return err + } + } + return nil + }, pulumi.WithMocks("ownership-test", "staging", ownershipMonitor{t})) + if err != nil { + t.Fatal(err) + } +} diff --git a/deploy/pkg/k8s/capacity.go b/deploy/pkg/k8s/capacity.go new file mode 100644 index 000000000..0ec601805 --- /dev/null +++ b/deploy/pkg/k8s/capacity.go @@ -0,0 +1,61 @@ +package k8s + +import ( + "fmt" + + autoscalingv2 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/autoscaling/v2" + metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi/config" +) + +const ( + coordinatedEnvironment = "staging" + coordinatedAppCPU = "800m" + coordinatedDBCPU = "1500m" +) + +// ValidateCapacityConfig runs before creating resources. This experimental +// profile is opt-in and must use the complete node/ingress/DB budget. +func ValidateCapacityConfig(ctx *pulumi.Context) error { + c := config.New(ctx, "mcp-registry") + if !c.GetBool("coordinatedCapacity") { + return nil + } + if !c.GetBool("burstAutoscaling") || c.Get("provider") != "gcp" || c.Get("nodeMachineType") != "e2-standard-4" { + return fmt.Errorf("coordinatedCapacity requires GCP e2-standard-4 nodes and burstAutoscaling") + } + if c.Get("environment") != coordinatedEnvironment { + return fmt.Errorf("coordinatedCapacity is staging-only until burst and recovery validation passes") + } + for key, expected := range map[string]string{"registryCPURequest": coordinatedAppCPU, "postgresCPURequest": coordinatedDBCPU, "postgresCPULimit": "3"} { + if c.Get(key) != expected { + return fmt.Errorf("coordinatedCapacity requires %s=%s", key, expected) + } + } + return nil +} + +func registryAutoscalerSpec() *autoscalingv2.HorizontalPodAutoscalerSpecArgs { + return &autoscalingv2.HorizontalPodAutoscalerSpecArgs{ + MinReplicas: pulumi.Int(2), MaxReplicas: pulumi.Int(4), + ScaleTargetRef: &autoscalingv2.CrossVersionObjectReferenceArgs{ApiVersion: pulumi.String("apps/v1"), Kind: pulumi.String("Deployment"), Name: pulumi.String("mcp-registry")}, + Metrics: autoscalingv2.MetricSpecArray{&autoscalingv2.MetricSpecArgs{ + Type: pulumi.String("Resource"), Resource: &autoscalingv2.ResourceMetricSourceArgs{ + Name: pulumi.String("cpu"), Target: &autoscalingv2.MetricTargetArgs{Type: pulumi.String("Utilization"), AverageUtilization: pulumi.Int(70)}, + }, + }}, + Behavior: &autoscalingv2.HorizontalPodAutoscalerBehaviorArgs{ + ScaleUp: &autoscalingv2.HPAScalingRulesArgs{StabilizationWindowSeconds: pulumi.Int(0), Policies: autoscalingv2.HPAScalingPolicyArray{&autoscalingv2.HPAScalingPolicyArgs{Type: pulumi.String("Pods"), Value: pulumi.Int(2), PeriodSeconds: pulumi.Int(60)}}}, + ScaleDown: &autoscalingv2.HPAScalingRulesArgs{StabilizationWindowSeconds: pulumi.Int(600), Policies: autoscalingv2.HPAScalingPolicyArray{&autoscalingv2.HPAScalingPolicyArgs{Type: pulumi.String("Pods"), Value: pulumi.Int(1), PeriodSeconds: pulumi.Int(120)}}}, + }, + } +} + +func deployRegistryAutoscaler(ctx *pulumi.Context, provider pulumi.ProviderResource) error { + _, err := autoscalingv2.NewHorizontalPodAutoscaler(ctx, "mcp-registry-capacity", &autoscalingv2.HorizontalPodAutoscalerArgs{ + Metadata: &metav1.ObjectMetaArgs{Name: pulumi.String("mcp-registry"), Namespace: pulumi.String("default")}, + Spec: registryAutoscalerSpec(), + }, pulumi.Provider(provider)) + return err +} diff --git a/deploy/pkg/k8s/capacity_test.go b/deploy/pkg/k8s/capacity_test.go new file mode 100644 index 000000000..5caaab2e6 --- /dev/null +++ b/deploy/pkg/k8s/capacity_test.go @@ -0,0 +1,89 @@ +package k8s //nolint:testpackage // Verify guarded experimental profile and rendered HPA. + +import ( + "strings" + "testing" + + "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes" + + "github.com/pulumi/pulumi/sdk/v3/go/common/resource" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" +) + +type capacityMonitor struct { + t *testing.T + hpaSeen bool +} + +func (m *capacityMonitor) Call(a pulumi.MockCallArgs) (resource.PropertyMap, error) { + return a.Args, nil +} +func (m *capacityMonitor) NewResource(a pulumi.MockResourceArgs) (string, resource.PropertyMap, error) { + if a.TypeToken == "kubernetes:autoscaling/v2:HorizontalPodAutoscaler" { + m.hpaSeen = true + spec := a.Inputs["spec"].ObjectValue() + if spec["minReplicas"].NumberValue() != 2 || spec["maxReplicas"].NumberValue()*30 > 120 { + m.t.Error("HPA exceeds connection budget or changes idle count") + } + target := spec["scaleTargetRef"].ObjectValue() + if target["name"].StringValue() != "mcp-registry" || target["kind"].StringValue() != "Deployment" { + m.t.Error("HPA targets the wrong workload") + } + metrics := spec["metrics"].ArrayValue() + if len(metrics) != 1 || metrics[0].ObjectValue()["resource"].ObjectValue()["name"].StringValue() != "cpu" { + m.t.Error("memory/cache must not hold scaling up after recovery") + } + if spec["behavior"].ObjectValue()["scaleDown"].ObjectValue()["stabilizationWindowSeconds"].NumberValue() != 600 { + m.t.Error("missing conservative recovery window") + } + } + return a.Name + "-id", a.Inputs, nil +} + +func TestCapacityRejectsIncompleteOrProductionProfile(t *testing.T) { + for _, tt := range []struct { + name, key, value string + wantError bool + }{ + {"valid", "environment", "staging", false}, + {"production", "environment", "prod", true}, + {"no node scaling", "burstAutoscaling", "false", true}, + {"undersized nodes", "nodeMachineType", "e2-standard-2", true}, + {"unmatched database request", "postgresCPURequest", "2", true}, + {"unmatched app request", "registryCPURequest", "400m", true}, + {"disabled", "coordinatedCapacity", "false", false}, + } { + t.Run(tt.name, func(t *testing.T) { + values := map[string]string{"coordinatedCapacity": "true", "burstAutoscaling": "true", "provider": "gcp", "environment": "staging", "nodeMachineType": "e2-standard-4", "registryCPURequest": "800m", "postgresCPURequest": "1500m", "postgresCPULimit": "3"} + values[tt.key] = tt.value + cfg := map[string]string{} + for k, v := range values { + cfg["mcp-registry:"+k] = v + } + err := pulumi.RunErr(ValidateCapacityConfig, pulumi.WithMocks("capacity-test", "staging", &capacityMonitor{t: t}), func(info *pulumi.RunInfo) { info.Config = cfg }) + if (err != nil) != tt.wantError { + t.Fatalf("unexpected validation result: %v", err) + } + if err != nil && !strings.Contains(err.Error(), "coordinatedCapacity") { + t.Fatalf("unexpected failure: %v", err) + } + }) + } +} + +func TestRegistryAutoscalerBoundsDatabaseConnections(t *testing.T) { + m := &capacityMonitor{t: t} + err := pulumi.RunErr(func(ctx *pulumi.Context) error { + provider, err := kubernetes.NewProvider(ctx, "test-provider", &kubernetes.ProviderArgs{}) + if err != nil { + return err + } + return deployRegistryAutoscaler(ctx, provider) + }, pulumi.WithMocks("capacity-test", "staging", m)) + if err != nil { + t.Fatal(err) + } + if !m.hpaSeen { + t.Fatal("HPA was not registered") + } +} diff --git a/deploy/pkg/k8s/ingress.go b/deploy/pkg/k8s/ingress.go index f9a02441e..f818d88ac 100644 --- a/deploy/pkg/k8s/ingress.go +++ b/deploy/pkg/k8s/ingress.go @@ -1,8 +1,6 @@ package k8s import ( - "strings" - v1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/core/v1" "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/helm/v3" metav1 "github.com/pulumi/pulumi-kubernetes/sdk/v4/go/kubernetes/meta/v1" @@ -18,6 +16,8 @@ import ( const registryIngressLogFormat = `$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] [$proxy_alternative_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status $req_id cache_status="$upstream_cache_status" scheme="$scheme" host="$host" ingress_pod="$hostname" upstream_connect_time="$upstream_connect_time" upstream_header_time="$upstream_header_time" completed_at="$msec"` // SetupIngressController sets up the NGINX Ingress Controller +// +//nolint:goconst // Keep Kubernetes and Helm schema keys literal. func SetupIngressController(ctx *pulumi.Context, cluster *providers.ProviderInfo, environment string) (*helm.Chart, error) { conf := config.New(ctx, "mcp-registry") provider := conf.Get("provider") @@ -74,6 +74,29 @@ func SetupIngressController(ctx *pulumi.Context, cluster *providers.ProviderInfo }} } + // Enable this only after the same stack opts into node autoscaling. Keep + // existing environments unchanged until their explicit promotion. + burstAutoscaling := config.New(ctx, "mcp-registry").GetBool("burstAutoscaling") + ingressMemoryRequest := "90Mi" + clientKeepaliveSnippet := "" + autoscaling := pulumi.Map{"enabled": pulumi.Bool(false)} + chartOptions := []pulumi.ResourceOption{pulumi.Provider(cluster.Provider)} + if burstAutoscaling { + ingressCPURequest = "800m" + ingressMemoryRequest = "512Mi" + // Bound active connection age so added controllers can receive existing + // client traffic. NGINX finishes the current request before closing. + clientKeepaliveSnippet = "keepalive_time 60s;\n" + autoscaling = ingressAutoscalingValues() + if conf.GetBool("coordinatedCapacity") { + // Scale at 560m instead of 704m to add capacity before contention. + autoscaling["targetCPUUtilizationPercentage"] = pulumi.Int(70) + } + // HPA owns replicas. Refresh before updates so the checkpoint follows + // the controller rather than resetting a scaled Deployment to two. + chartOptions = append(chartOptions, pulumi.Transformations([]pulumi.ResourceTransformation{burstResourceTransform})) + } + // Install NGINX Ingress Controller ingressNginx, err := helm.NewChart(ctx, "ingress-nginx", helm.ChartArgs{ Chart: pulumi.String("ingress-nginx"), @@ -85,10 +108,12 @@ func SetupIngressController(ctx *pulumi.Context, cluster *providers.ProviderInfo Values: pulumi.Map{ "controller": pulumi.Map{ "replicaCount": pulumi.Int(replicaCount), + "autoscaling": autoscaling, + "metrics": pulumi.Map{"enabled": pulumi.Bool(burstAutoscaling)}, "resources": pulumi.Map{ "requests": pulumi.StringMap{ "cpu": pulumi.String(ingressCPURequest), - "memory": pulumi.String("90Mi"), + "memory": pulumi.String(ingressMemoryRequest), }, }, "topologySpreadConstraints": spreadConstraints, @@ -125,7 +150,7 @@ map "$request_method:$uri" $registry_read_cache_zone { default off; ~^GET:/v0(?:\.1)?/servers$ registry_read_cache; } -`), +` + clientKeepaliveSnippet), "location-snippet": pulumi.String(` proxy_cache $registry_read_cache_zone; proxy_cache_key "$scheme$request_method$host$request_uri"; @@ -151,7 +176,7 @@ add_header X-Registry-Cache $upstream_cache_status always; }, }, }, - }, pulumi.Provider(cluster.Provider)) + }, chartOptions...) if err != nil { return nil, err } @@ -161,9 +186,7 @@ add_header X-Registry-Cache $upstream_cache_status always; // Look for the ingress-nginx-controller service resourceMap := resources.(map[string]pulumi.Resource) for resourceName, resource := range resourceMap { - if strings.Contains(resourceName, "ingress-nginx-controller") && - !strings.Contains(resourceName, "admission") && - strings.Contains(resourceName, "Service") { + if resourceName == "v1/Service::ingress-nginx/ingress-nginx-controller" { if svc, ok := resource.(*v1.Service); ok { // Return the LoadBalancer ingress IPs return svc.Status.LoadBalancer().Ingress().ApplyT(func(ingresses []v1.LoadBalancerIngress) []string { diff --git a/deploy/pkg/k8s/monitoring.go b/deploy/pkg/k8s/monitoring.go index ed275f0f0..c1bb5e8d0 100644 --- a/deploy/pkg/k8s/monitoring.go +++ b/deploy/pkg/k8s/monitoring.go @@ -51,6 +51,12 @@ func DeployMonitoringStack(ctx *pulumi.Context, cluster *providers.ProviderInfo, return err } + // The chart gives its ConfigMap a fixed name. Pulumi replaces ConfigMaps + // on data changes; delete the old object first to avoid AlreadyExists. + vmOptions := []pulumi.ResourceOption{pulumi.Provider(cluster.Provider)} + if config.New(ctx, "mcp-registry").GetBool("burstAutoscaling") { + vmOptions = append(vmOptions, pulumi.Transformations([]pulumi.ResourceTransformation{burstResourceTransform})) + } // Deploy VMAgent _, err = helm.NewChart(ctx, "victoria-metrics-agent", helm.ChartArgs{ Chart: pulumi.String("victoria-metrics-agent"), @@ -69,7 +75,7 @@ func DeployMonitoringStack(ctx *pulumi.Context, cluster *providers.ProviderInfo, "global": pulumi.Map{ "scrape_interval": pulumi.String("60s"), }, - "scrape_configs": pulumi.Array{ + "scrape_configs": withIngressScrape(ctx, pulumi.Array{ pulumi.Map{ "job_name": pulumi.String("mcp-registry"), "kubernetes_sd_configs": pulumi.Array{ @@ -88,7 +94,7 @@ func DeployMonitoringStack(ctx *pulumi.Context, cluster *providers.ProviderInfo, }, }, }, - }, + }), }, "resources": pulumi.Map{ "requests": pulumi.Map{ @@ -100,7 +106,7 @@ func DeployMonitoringStack(ctx *pulumi.Context, cluster *providers.ProviderInfo, }, }, }, - }, pulumi.Provider(cluster.Provider)) + }, vmOptions...) if err != nil { return err } @@ -452,6 +458,7 @@ func deployOtelCollectorDaemonSet(ctx *pulumi.Context, cluster *providers.Provid return nil } +//nolint:goconst // Keep Kubernetes and Helm schema keys literal. func deployGrafana(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *corev1.Namespace, environment string, ingressNginx *helm.Chart) error { conf := config.New(ctx, "mcp-registry") grafanaSecret, err := corev1.NewSecret(ctx, "grafana-secrets", &corev1.SecretArgs{ @@ -480,13 +487,13 @@ func deployGrafana(ctx *pulumi.Context, cluster *providers.ProviderInfo, ns *cor }, }, OtherFields: map[string]any{ - "spec": map[string]any{ + "spec": protectDatabase(ctx, map[string]any{ "instances": 1, "enablePDB": false, "storage": map[string]any{ "size": "10Gi", }, - }, + }), }, }, pulumi.Provider(cluster.Provider)) if err != nil { diff --git a/deploy/pkg/k8s/postgres.go b/deploy/pkg/k8s/postgres.go index b352fc5d6..0521c0e2c 100644 --- a/deploy/pkg/k8s/postgres.go +++ b/deploy/pkg/k8s/postgres.go @@ -12,6 +12,8 @@ import ( ) // DeployPostgresDatabases deploys the CloudNative PostgreSQL operator and PostgreSQL cluster +// +//nolint:goconst // Keep Kubernetes and Helm schema keys literal. func DeployPostgresDatabases(ctx *pulumi.Context, cluster *providers.ProviderInfo, environment string) (*apiextensions.CustomResource, error) { // Create cnpg-system namespace cnpgNamespace, err := corev1.NewNamespace(ctx, "cnpg-system", &corev1.NamespaceArgs{ @@ -58,6 +60,11 @@ func DeployPostgresDatabases(ctx *pulumi.Context, cluster *providers.ProviderInf cpuLimit = "1500m" } + cpuRequest := config.New(ctx, "mcp-registry").Get("postgresCPURequest") + if cpuRequest == "" { + cpuRequest = "200m" + } + pgCluster, err := apiextensions.NewCustomResource(ctx, "registry-pg", &apiextensions.CustomResourceArgs{ ApiVersion: pulumi.String("postgresql.cnpg.io/v1"), Kind: pulumi.String("Cluster"), @@ -70,7 +77,7 @@ func DeployPostgresDatabases(ctx *pulumi.Context, cluster *providers.ProviderInf }, }, OtherFields: map[string]any{ - "spec": map[string]any{ + "spec": protectDatabase(ctx, map[string]any{ "instances": 1, "enablePDB": false, "storage": map[string]any{ @@ -82,7 +89,7 @@ func DeployPostgresDatabases(ctx *pulumi.Context, cluster *providers.ProviderInf "resources": map[string]any{ "requests": map[string]any{ "memory": "512Mi", - "cpu": "200m", + "cpu": cpuRequest, }, "limits": map[string]any{ "memory": "4Gi", @@ -112,7 +119,7 @@ func DeployPostgresDatabases(ctx *pulumi.Context, cluster *providers.ProviderInf }, }, }, - }, + }), }, }, pulumi.Provider(cluster.Provider), pulumi.DependsOnInputs(cloudNativePG.Ready), pulumi.RetainOnDelete(true)) if err != nil { diff --git a/deploy/pkg/k8s/registry.go b/deploy/pkg/k8s/registry.go index d31a07074..234c7b23d 100644 --- a/deploy/pkg/k8s/registry.go +++ b/deploy/pkg/k8s/registry.go @@ -59,6 +59,9 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env spreadConstraint.MatchLabelKeys = pulumi.StringArray{pulumi.String("pod-template-hash")} spreadConstraint.NodeTaintsPolicy = pulumi.String("Honor") } + if request := conf.Get("registryCPURequest"); request != "" { + registryCPURequest = request + } // Create Secret with sensitive configuration secret, err := corev1.NewSecret(ctx, "mcp-registry-secrets", &corev1.SecretArgs{ @@ -80,6 +83,21 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env return nil, err } + coordinatedCapacity := conf.GetBool("coordinatedCapacity") + deploymentOptions := []pulumi.ResourceOption{pulumi.Provider(cluster.Provider)} + maxSurge, maxUnavailable := 1, 0 + poolEnv := corev1.EnvVarArray{} + if coordinatedCapacity { + // HPA owns the live count; keep the steady-state connection budget at + // 60–120 across 2–4 apps. Terminating pods may overlap during rollouts. + deploymentOptions = append(deploymentOptions, pulumi.IgnoreChanges([]string{"spec.replicas"})) + maxSurge, maxUnavailable = 0, 1 + poolEnv = corev1.EnvVarArray{ + &corev1.EnvVarArgs{Name: pulumi.String("MCP_REGISTRY_DATABASE_MAX_CONNECTIONS"), Value: pulumi.String("30")}, + &corev1.EnvVarArgs{Name: pulumi.String("MCP_REGISTRY_DATABASE_MIN_CONNECTIONS"), Value: pulumi.String("5")}, + } + } + // Create Deployment _, err = v1.NewDeployment(ctx, "mcp-registry", &v1.DeploymentArgs{ Metadata: &metav1.ObjectMetaArgs{ @@ -95,8 +113,8 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env Strategy: &v1.DeploymentStrategyArgs{ Type: pulumi.String("RollingUpdate"), RollingUpdate: &v1.RollingUpdateDeploymentArgs{ - MaxUnavailable: pulumi.IntPtr(0), // Never reduce capacity during updates - MaxSurge: pulumi.IntPtr(1), // Create new pods first, then terminate old + MaxUnavailable: pulumi.IntPtr(maxUnavailable), + MaxSurge: pulumi.IntPtr(maxSurge), }, }, Selector: &metav1.LabelSelectorArgs{ @@ -127,7 +145,7 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env Name: pulumi.String("http"), }, }, - Env: corev1.EnvVarArray{ + Env: append(corev1.EnvVarArray{ &corev1.EnvVarArgs{ Name: pulumi.String("MCP_REGISTRY_DATABASE_URL"), ValueFrom: &corev1.EnvVarSourceArgs{ @@ -198,7 +216,7 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env return "https://" + environment + ".registry.modelcontextprotocol.io" }()), }, - }, + }, poolEnv...), // StartupProbe protects the DB-retry budget in cmd/registry/main.go: // 30 × 5s = 150s allowed before liveness/readiness take over, which // covers the worst-case retry budget (8 attempts × 10s + ~40s sleep). @@ -249,11 +267,17 @@ func DeployMCPRegistry(ctx *pulumi.Context, cluster *providers.ProviderInfo, env }, }, }, - }, pulumi.Provider(cluster.Provider)) + }, deploymentOptions...) if err != nil { return nil, err } + if coordinatedCapacity { + if err := deployRegistryAutoscaler(ctx, cluster.Provider); err != nil { + return nil, err + } + } + // Keep at least one registry pod available during voluntary disruptions // (node drains, evictions, etc.) so a synchronized eviction can't take both // pods down at once. diff --git a/deploy/pkg/providers/gcp/provider.go b/deploy/pkg/providers/gcp/provider.go index 031140062..0ca4891a9 100644 --- a/deploy/pkg/providers/gcp/provider.go +++ b/deploy/pkg/providers/gcp/provider.go @@ -277,6 +277,14 @@ func (p *Provider) CreateCluster(ctx *pulumi.Context, environment string) (*prov // Add provider if we have explicit credentials nodePoolOpts := []pulumi.ResourceOption{} + if config.New(ctx, "mcp-registry").GetBool("burstAutoscaling") { + nodePoolArgs.Autoscaling = &container.NodePoolAutoscalingArgs{ + MinNodeCount: pulumi.IntPtr(2), + MaxNodeCount: pulumi.IntPtr(4), + } + // The autoscaler owns the live count. Always refresh before updating. + nodePoolOpts = append(nodePoolOpts, pulumi.IgnoreChanges([]string{"nodeCount"})) + } if gcpProvider != nil { nodePoolOpts = append(nodePoolOpts, pulumi.Provider(gcpProvider)) } diff --git a/internal/api/handlers/v0/listing_json_test.go b/internal/api/handlers/v0/listing_json_test.go new file mode 100644 index 000000000..1c412fd25 --- /dev/null +++ b/internal/api/handlers/v0/listing_json_test.go @@ -0,0 +1,75 @@ +package v0_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/danielgtaylor/huma/v2" + "github.com/danielgtaylor/huma/v2/adapters/humago" + v0 "github.com/modelcontextprotocol/registry/internal/api/handlers/v0" + "github.com/modelcontextprotocol/registry/internal/config" + "github.com/modelcontextprotocol/registry/internal/database" + "github.com/modelcontextprotocol/registry/internal/service" + apiv0 "github.com/modelcontextprotocol/registry/pkg/api/v0" + "github.com/modelcontextprotocol/registry/pkg/model" + "github.com/stretchr/testify/require" +) + +func TestEncodedListingMatchesTypedAPIThroughUpdates(t *testing.T) { + db := database.NewTestDB(t) + ctx := context.Background() + registry := service.NewRegistryService(db, config.NewConfig()) + stamp := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + server := apiv0.ServerJSON{Schema: model.CurrentSchemaURL, Name: "example/alpha", Version: "1", Description: "original <>&", Meta: &apiv0.ServerMeta{PublisherProvided: map[string]interface{}{"number": 123.5}}} + meta := apiv0.RegistryExtensions{Status: model.StatusActive, PublishedAt: stamp, UpdatedAt: stamp, StatusChangedAt: stamp, IsLatest: true} + _, err := db.CreateServer(ctx, nil, &server, &meta) + require.NoError(t, err) + second := server + second.Name = "example/beta" + _, err = db.CreateServer(ctx, nil, &second, &meta) + require.NoError(t, err) + muxes := []*http.ServeMux{http.NewServeMux(), http.NewServeMux()} + var schemas [][]byte + for i, mux := range muxes { + cfg := huma.DefaultConfig("test", "1") + cfg.CreateHooks = []func(huma.Config) huma.Config{} + api := humago.New(mux, cfg) + v0.RegisterServersEndpoints(api, "/v0", registry, i == 1) + v0.RegisterServersEndpoints(api, "/v0.1", registry, i == 1) + raw, err := json.Marshal(api.OpenAPI()) + require.NoError(t, err) + schemas = append(schemas, raw) + } + require.JSONEq(t, string(schemas[0]), string(schemas[1]), "optimization must not change OpenAPI") + check := func() { + t.Helper() + for _, path := range []string{"/v0/servers", "/v0.1/servers?limit=1", "/v0.1/servers?cursor=example/alpha:1&limit=1", "/v0.1/servers?search=alpha", "/v0.1/servers?search=absent", "/v0.1/servers?version=latest", "/v0.1/servers?include_deleted=true", "/v0.1/servers?updated_since=2025-01-01T00:00:00Z"} { + var bodies []string + for _, mux := range muxes { + w := httptest.NewRecorder() + mux.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + require.Equal(t, http.StatusOK, w.Code, path) + bodies = append(bodies, w.Body.String()) + } + require.JSONEq(t, bodies[0], bodies[1], path) + } + } + check() + check() // Repeated reads must preserve fresh metadata. + server.Description = "updated payload" + _, err = db.UpdateServer(ctx, nil, server.Name, server.Version, &server) + require.NoError(t, err) + check() + _, err = db.SetServerStatus(ctx, nil, server.Name, server.Version, model.StatusDeleted, nil) + require.NoError(t, err) + check() + third := second + third.Name = "example/new" + _, err = db.CreateServer(ctx, nil, &third, &meta) + require.NoError(t, err) + check() +} diff --git a/internal/api/handlers/v0/servers.go b/internal/api/handlers/v0/servers.go index 86ae1afce..1c16a2477 100644 --- a/internal/api/handlers/v0/servers.go +++ b/internal/api/handlers/v0/servers.go @@ -96,7 +96,7 @@ type ServerVersionsInput struct { } // RegisterServersEndpoints registers all server-related endpoints with a custom path prefix -func RegisterServersEndpoints(api huma.API, pathPrefix string, registry service.RegistryService) { +func RegisterServersEndpoints(api huma.API, pathPrefix string, registry service.RegistryService, encodedResponse ...bool) { // List servers endpoint huma.Register(api, huma.Operation{ OperationID: "list-servers" + strings.ReplaceAll(pathPrefix, "/", "-"), @@ -144,27 +144,11 @@ func RegisterServersEndpoints(api huma.API, pathPrefix string, registry service. } filter.IncludeDeleted = &includeDeleted - // Get paginated results with filtering - servers, nextCursor, err := registry.ListServers(ctx, filter, input.Cursor, input.Limit) + body, err := listServersBody(ctx, registry, filter, input, len(encodedResponse) > 0 && encodedResponse[0]) if err != nil { return nil, ListServersError(ctx, err) } - - // Convert []*ServerResponse to []ServerResponse - serverValues := make([]apiv0.ServerResponse, len(servers)) - for i, server := range servers { - serverValues[i] = *server - } - - return &Response[apiv0.ServerListResponse]{ - Body: apiv0.ServerListResponse{ - Servers: serverValues, - Metadata: apiv0.Metadata{ - NextCursor: nextCursor, - Count: len(servers), - }, - }, - }, nil + return &Response[apiv0.ServerListResponse]{Body: body}, nil }) // Get specific server version endpoint (supports "latest" as special version) @@ -250,3 +234,23 @@ func RegisterServersEndpoints(api huma.API, pathPrefix string, registry service. }, nil }) } + +// Encoded responses are enabled only by the production router, which disables +// Huma's schema-link transformer. That transformer reconstructs structs from +// exported fields and cannot preserve a private serialized representation. +func listServersBody(ctx context.Context, registry service.RegistryService, filter *database.ServerFilter, input *ListServersInput, encoded bool) (apiv0.ServerListResponse, error) { + if fast, ok := registry.(interface { + ListServersJSON(context.Context, *database.ServerFilter, string, int) (apiv0.ServerListResponse, error) + }); ok && encoded { + return fast.ListServersJSON(ctx, filter, input.Cursor, input.Limit) + } + servers, next, err := registry.ListServers(ctx, filter, input.Cursor, input.Limit) + if err != nil { + return apiv0.ServerListResponse{}, err + } + values := make([]apiv0.ServerResponse, len(servers)) + for i, server := range servers { + values[i] = *server + } + return apiv0.ServerListResponse{Servers: values, Metadata: apiv0.Metadata{Count: len(values), NextCursor: next}}, nil +} diff --git a/internal/api/router/v0.go b/internal/api/router/v0.go index 3b57c67ea..f9a2d903a 100644 --- a/internal/api/router/v0.go +++ b/internal/api/router/v0.go @@ -17,7 +17,7 @@ func RegisterV0Routes( v0.RegisterHealthEndpoint(api, "/v0", cfg, metrics) v0.RegisterPingEndpoint(api, "/v0") v0.RegisterVersionEndpoint(api, "/v0", versionInfo) - v0.RegisterServersEndpoints(api, "/v0", registry) + v0.RegisterServersEndpoints(api, "/v0", registry, true) v0.RegisterEditEndpoints(api, "/v0", registry, cfg) v0.RegisterStatusEndpoints(api, "/v0", registry, cfg) v0.RegisterAllVersionsStatusEndpoints(api, "/v0", registry, cfg) @@ -32,7 +32,7 @@ func RegisterV0_1Routes( v0.RegisterHealthEndpoint(api, "/v0.1", cfg, metrics) v0.RegisterPingEndpoint(api, "/v0.1") v0.RegisterVersionEndpoint(api, "/v0.1", versionInfo) - v0.RegisterServersEndpoints(api, "/v0.1", registry) + v0.RegisterServersEndpoints(api, "/v0.1", registry, true) v0.RegisterEditEndpoints(api, "/v0.1", registry, cfg) v0.RegisterStatusEndpoints(api, "/v0.1", registry, cfg) v0.RegisterAllVersionsStatusEndpoints(api, "/v0.1", registry, cfg) diff --git a/internal/config/config.go b/internal/config/config.go index a15a67a11..f3fb95610 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,6 +9,8 @@ import ( type Config struct { ServerAddress string `env:"SERVER_ADDRESS" envDefault:":8080"` DatabaseURL string `env:"DATABASE_URL" envDefault:"postgres://localhost:5432/mcp-registry?sslmode=disable"` + DatabaseMaxConnections int32 `env:"DATABASE_MAX_CONNECTIONS" envDefault:"60"` + DatabaseMinConnections int32 `env:"DATABASE_MIN_CONNECTIONS" envDefault:"10"` SeedFrom string `env:"SEED_FROM" envDefault:""` Version string `env:"VERSION" envDefault:"dev"` GithubClientID string `env:"GITHUB_CLIENT_ID" envDefault:""` diff --git a/internal/config/pool_limits_test.go b/internal/config/pool_limits_test.go new file mode 100644 index 000000000..ad9290e16 --- /dev/null +++ b/internal/config/pool_limits_test.go @@ -0,0 +1,17 @@ +package config_test + +import ( + "github.com/modelcontextprotocol/registry/internal/config" + "github.com/stretchr/testify/require" + "testing" +) + +func TestDatabasePoolEnvironment(t *testing.T) { + t.Setenv("MCP_REGISTRY_DATABASE_MAX_CONNECTIONS", "30") + t.Setenv("MCP_REGISTRY_DATABASE_MIN_CONNECTIONS", "5") + cfg := config.NewConfig() + require.EqualValues(t, 30, cfg.DatabaseMaxConnections) + require.EqualValues(t, 5, cfg.DatabaseMinConnections) + t.Setenv("MCP_REGISTRY_DATABASE_MAX_CONNECTIONS", "not-an-integer") + require.Panics(t, func() { config.NewConfig() }) +} diff --git a/internal/database/listing_backfill.go b/internal/database/listing_backfill.go new file mode 100644 index 000000000..53facf479 --- /dev/null +++ b/internal/database/listing_backfill.go @@ -0,0 +1,88 @@ +package database + +import ( + "context" + "fmt" + + "github.com/jackc/pgx/v5" +) + +// ListingBackfillResult describes one bounded, resumable keyset batch. A row +// changed concurrently is left for a later pass; authoritative data is untouched. +type ListingBackfillResult struct { + Scanned int `json:"scanned"` + Prepared int64 `json:"prepared"` + SkippedIdentity int `json:"skipped_identity"` + LastName string `json:"last_name"` + LastVersion string `json:"last_version"` +} + +type listingBackfillRow struct { + name, version string + value []byte +} + +const prepareListingSQL = `UPDATE servers SET listing_value = $1, listing_codec = $2 + WHERE server_name = $3 AND version = $4 AND value = $5::jsonb` + +// BackfillListingJSONBatch requires migration 018. The caller controls the +// connection's statement/lock timeouts and delay between batches. No row lock is +// held while Go normalizes JSON; the conditional update rejects stale snapshots. +func BackfillListingJSONBatch(ctx context.Context, conn *pgx.Conn, afterName, afterVersion string, limit int) (ListingBackfillResult, error) { + result := ListingBackfillResult{LastName: afterName, LastVersion: afterVersion} + if limit < 1 || limit > 1000 { + return result, fmt.Errorf("batch size must be 1..1000") + } + rows, err := conn.Query(ctx, `SELECT server_name, version, value FROM servers + WHERE (server_name, version) > ($1, $2) + AND (listing_codec IS DISTINCT FROM $3 OR listing_value IS NULL) + ORDER BY server_name, version LIMIT $4`, afterName, afterVersion, listingCodecVersion, limit) + if err != nil { + return result, err + } + var page []listingBackfillRow + for rows.Next() { + var row listingBackfillRow + if err := rows.Scan(&row.name, &row.version, &row.value); err != nil { + rows.Close() + return result, err + } + page = append(page, row) + } + rows.Close() + if err := rows.Err(); err != nil { + return result, err + } + batch := &pgx.Batch{} + for _, row := range page { + result.Scanned++ + result.LastName, result.LastVersion = row.name, row.version + server, err := normalizeListingJSON(row.value) + if err != nil { + return result, err + } + if server.name != row.name || server.version != row.version { + result.SkippedIdentity++ + continue + } + batch.Queue(prepareListingSQL, []byte(server.json), listingCodecVersion, row.name, row.version, row.value) + } + if batch.Len() == 0 { + return result, nil + } + pending := conn.SendBatch(ctx, batch) + defer pending.Close() + for range batch.Len() { + tag, err := pending.Exec() + if err != nil { + return result, err + } + result.Prepared += tag.RowsAffected() + } + // SendBatch executes in an implicit transaction. Only claim committed counts + // once Close confirms the entire batch completed successfully. + if err := pending.Close(); err != nil { + return result, err + } + return result, nil +} diff --git a/internal/database/listing_json.go b/internal/database/listing_json.go new file mode 100644 index 000000000..3eb819283 --- /dev/null +++ b/internal/database/listing_json.go @@ -0,0 +1,88 @@ +package database + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/modelcontextprotocol/registry/internal/telemetry" + apiv0 "github.com/modelcontextprotocol/registry/pkg/api/v0" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +// Bump whenever ServerJSON's wire normalization changes. Unknown codecs fall +// back to authoritative JSONB, allowing mixed-version deployments and rollback. +const listingCodecVersion = 1 + +type normalizedServer struct { + json json.RawMessage + name, version string +} + +func normalizeListingJSON(value []byte) (normalizedServer, error) { + var server apiv0.ServerJSON + if err := json.Unmarshal(value, &server); err != nil { + return normalizedServer{}, fmt.Errorf("failed to unmarshal server JSON: %w", err) + } + encoded, err := json.Marshal(server) + if err != nil { + return normalizedServer{}, fmt.Errorf("encode server JSON: %w", err) + } + return normalizedServer{json: encoded, name: server.Name, version: server.Version}, nil +} + +func listingPayload(prefix string) string { + valid := fmt.Sprintf("(%slisting_codec = %d AND %slisting_value IS NOT NULL)", prefix, listingCodecVersion, prefix) + return fmt.Sprintf("CASE WHEN %s THEN %slisting_value ELSE convert_to(%svalue::text, 'UTF8') END, COALESCE(%s, false)", valid, prefix, prefix, valid) +} + +// ListServersJSON is an HTTP serialization path. Typed listings remain in use +// for transactions, publish validation and other callers that inspect records. +func (db *PostgreSQL) ListServersJSON(ctx context.Context, filter *ServerFilter, cursor string, limit int) (_ apiv0.ServerListResponse, resultErr error) { + diagnostic := telemetry.ListingFromContext(ctx) + defer func() { diagnostic.DatabaseDone(resultErr) }() + page, err := db.listServerRows(ctx, nil, filter, cursor, limit, true) + if err != nil { + return apiv0.ServerListResponse{}, err + } + if limit <= 0 { + limit = 10 + } + if db.listingMetrics != nil { + var prepared int64 + for _, row := range page { + if row.prepared { + prepared++ + } + } + db.listingMetrics.Add(ctx, prepared, metric.WithAttributes(attribute.String("source", "prepared"))) + db.listingMetrics.Add(ctx, int64(len(page))-prepared, metric.WithAttributes(attribute.String("source", "fallback"))) + } + return encodeListingPage(ctx, page, limit, diagnostic) +} + +func encodeListingPage(ctx context.Context, page []listingRow, limit int, diagnostic *telemetry.ListingDiagnostic) (apiv0.ServerListResponse, error) { + diagnostic.Phase("row_decode") + records := make([]apiv0.EncodedServerRecord, 0, len(page)) + nextCursor := "" + for i := range page { + if err := ctx.Err(); err != nil { + return apiv0.ServerListResponse{}, err + } + server := normalizedServer{json: page[i].value, name: page[i].name, version: page[i].version} + if !page[i].prepared { + var err error + server, err = normalizeListingJSON(page[i].value) + if err != nil { + return apiv0.ServerListResponse{}, err + } + } + page[i].value = nil + records = append(records, apiv0.EncodedServerRecord{Server: server.json, Meta: apiv0.ResponseMeta{Official: &page[i].metadata}}) + if i == len(page)-1 && len(page) >= limit { + nextCursor = server.name + ":" + server.version + } + } + return apiv0.NewEncodedServerList(records, apiv0.Metadata{Count: len(records), NextCursor: nextCursor}), nil +} diff --git a/internal/database/listing_json_internal_test.go b/internal/database/listing_json_internal_test.go new file mode 100644 index 000000000..240b0e531 --- /dev/null +++ b/internal/database/listing_json_internal_test.go @@ -0,0 +1,172 @@ +package database + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + apiv0 "github.com/modelcontextprotocol/registry/pkg/api/v0" + "github.com/modelcontextprotocol/registry/pkg/model" + "github.com/stretchr/testify/require" +) + +func TestListingJSONPreservesNormalizationAndFreshMetadata(t *testing.T) { + raw := []byte(`{"name":"example/server","version":"1.0","unknown":"discard","description":"\u2028","packages":[{"registryType":"npm","identifier":"x","version":"1","runtimeArguments":[{"type":"positional","value":"","valueHint":""}]}],"_meta":{"io.modelcontextprotocol.registry/publisher-provided":{"n":123.5}}}`) + for _, status := range []model.Status{model.StatusActive, model.StatusDeleted} { + meta := apiv0.RegistryExtensions{Status: status, IsLatest: status == model.StatusActive, UpdatedAt: time.Unix(100, 0).UTC()} + typed, err := decodeListingPage(context.Background(), []listingRow{{value: raw, metadata: meta}}, nil) + require.NoError(t, err) + expected, err := json.Marshal(apiv0.ServerListResponse{Servers: []apiv0.ServerResponse{*typed[0]}, Metadata: apiv0.Metadata{Count: 1, NextCursor: "example/server:1.0"}}) + require.NoError(t, err) + wire, err := encodeListingPage(context.Background(), []listingRow{{value: raw, metadata: meta}}, 1, nil) + require.NoError(t, err) + actual, err := json.Marshal(wire) + require.NoError(t, err) + require.JSONEq(t, string(expected), string(actual)) + require.NotContains(t, string(actual), "discard") + } + changed := []byte(strings.Replace(string(raw), "", "changed", 1)) + got, err := normalizeListingJSON(changed) + require.NoError(t, err) + require.Contains(t, string(got.json), "changed") + _, err = normalizeListingJSON([]byte(`{`)) + require.Error(t, err) +} + +func TestListingJSONEmptyAndCanceled(t *testing.T) { + empty, err := encodeListingPage(context.Background(), nil, 30, nil) + require.NoError(t, err) + encoded, err := json.Marshal(empty) + require.NoError(t, err) + require.JSONEq(t, `{"servers":[],"metadata":{"count":0}}`, string(encoded)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err = encodeListingPage(ctx, []listingRow{{value: []byte(`{}`)}}, 30, nil) + require.ErrorIs(t, err, context.Canceled) +} + +func BenchmarkListingJSON(b *testing.B) { + page := make([]listingRow, 50) + for i := range page { + server := apiv0.ServerJSON{Name: fmt.Sprintf("example/server-%d", i), Version: "1.0", Description: strings.Repeat("description ", 8), Meta: &apiv0.ServerMeta{PublisherProvided: map[string]interface{}{"documentation": strings.Repeat("sample metadata and instructions. ", 150)}}} + value, err := json.Marshal(server) + if err != nil { + b.Fatal(err) + } + page[i] = listingRow{value: value, metadata: apiv0.RegistryExtensions{Status: model.StatusActive}} + } + for _, mode := range []string{"typed_roundtrip", "prepared_json"} { + b.Run(mode, func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + copyPage := append([]listingRow(nil), page...) + for i := range copyPage { + copyPage[i].prepared = mode == "prepared_json" + } + var response apiv0.ServerListResponse + if mode == "typed_roundtrip" { + rows, err := decodeListingPage(context.Background(), copyPage, nil) + if err != nil { + b.Fatal(err) + } + values := make([]apiv0.ServerResponse, len(rows)) + for i, row := range rows { + values[i] = *row + } + response = apiv0.ServerListResponse{Servers: values, Metadata: apiv0.Metadata{Count: 50}} + } else { + var err error + response, err = encodeListingPage(context.Background(), copyPage, 100, nil) + if err != nil { + b.Fatal(err) + } + } + var output any = response + if mode == "typed_roundtrip" { + type originalList apiv0.ServerListResponse + output = originalList(response) + } + if _, err := json.Marshal(output); err != nil { + b.Fatal(err) + } + } + }) + } +} + +func TestPreparedListingWritesFallbackAndBackfill(t *testing.T) { + db := NewTestDB(t).(*PostgreSQL) + ctx := context.Background() + conn, err := db.pool.Acquire(ctx) + require.NoError(t, err) + defer conn.Release() + server := apiv0.ServerJSON{Name: "example/prepared", Version: "1", Description: "original", Meta: &apiv0.ServerMeta{PublisherProvided: map[string]interface{}{"large": int64(9007199254740993)}}} + stamp := time.Now().UTC() + meta := apiv0.RegistryExtensions{Status: model.StatusActive, IsLatest: true, UpdatedAt: stamp, PublishedAt: stamp, StatusChangedAt: stamp} + _, err = db.CreateServer(ctx, nil, &server, &meta) + require.NoError(t, err) + check := func(wantPrepared bool) { + t.Helper() + rows, err := db.listServerRows(ctx, nil, nil, "", 10, true) + require.NoError(t, err) + require.Len(t, rows, 1) + require.Equal(t, wantPrepared, rows[0].prepared) + fast, err := db.ListServersJSON(ctx, nil, "", 1) + require.NoError(t, err) + typed, cursor, err := db.ListServers(ctx, nil, nil, "", 1) + require.NoError(t, err) + expected, err := json.Marshal(apiv0.ServerListResponse{Servers: []apiv0.ServerResponse{*typed[0]}, Metadata: apiv0.Metadata{Count: 1, NextCursor: cursor}}) + require.NoError(t, err) + actual, err := json.Marshal(fast) + require.NoError(t, err) + require.JSONEq(t, string(expected), string(actual)) + } + check(true) + // Metadata-only changes are immediately visible and do not require rewriting payloads. + _, err = db.pool.Exec(ctx, `UPDATE servers SET status_message = 'fresh metadata', updated_at = now()`) + require.NoError(t, err) + check(true) + // Simulate an older binary: JSONB changes without writing prepared fields. + _, err = db.pool.Exec(ctx, `UPDATE servers SET value = jsonb_set(value, '{description}', '"old writer"')`) + require.NoError(t, err) + check(false) + var value []byte + require.NoError(t, conn.QueryRow(ctx, `SELECT value FROM servers`).Scan(&value)) + normalized, err := normalizeListingJSON(value) + require.NoError(t, err) + // A concurrent update after the backfill read must not install stale bytes. + _, err = db.pool.Exec(ctx, `UPDATE servers SET value = jsonb_set(value, '{description}', '"concurrent writer"')`) + require.NoError(t, err) + tag, err := conn.Exec(ctx, prepareListingSQL, []byte(normalized.json), listingCodecVersion, server.Name, server.Version, value) + require.NoError(t, err) + require.Zero(t, tag.RowsAffected()) + check(false) + result, err := BackfillListingJSONBatch(ctx, conn.Conn(), "", "", 1) + require.NoError(t, err) + require.EqualValues(t, 1, result.Prepared) + require.Equal(t, 1, result.Scanned) + check(true) + result, err = BackfillListingJSONBatch(ctx, conn.Conn(), result.LastName, result.LastVersion, 1) + require.NoError(t, err) + require.Zero(t, result.Scanned) + // Unknown future codecs safely fall back during rolling versions/rollback. + _, err = db.pool.Exec(ctx, `UPDATE servers SET listing_codec = 999`) + require.NoError(t, err) + check(false) + server.Description = "new writer" + _, err = db.UpdateServer(ctx, nil, server.Name, server.Version, &server) + require.NoError(t, err) + check(true) + // Identity mismatches in legacy rows retain their old cursor semantics. + _, err = db.pool.Exec(ctx, `UPDATE servers SET value = jsonb_set(value, '{name}', '"example/legacy-name"')`) + require.NoError(t, err) + result, err = BackfillListingJSONBatch(ctx, conn.Conn(), "", "", 10) + require.NoError(t, err) + require.Equal(t, 1, result.SkippedIdentity) + require.Zero(t, result.Prepared) + check(false) +} diff --git a/internal/database/listing_release_test.go b/internal/database/listing_release_test.go new file mode 100644 index 000000000..ae302ae21 --- /dev/null +++ b/internal/database/listing_release_test.go @@ -0,0 +1,97 @@ +package database_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/modelcontextprotocol/registry/internal/database" + "github.com/stretchr/testify/require" +) + +type listingTestTx struct { + pgx.Tx + rows pgx.Rows +} + +func (tx listingTestTx) Query(context.Context, string, ...any) (pgx.Rows, error) { + return tx.rows, nil +} + +type listingTestRows struct { + pgx.Rows + values []string + read int + closed bool + scanErr error + endErr error + onClose func() +} + +func (r *listingTestRows) Next() bool { + if r.read < len(r.values) && !r.closed { + r.read++ + return true + } + r.Close() + return false +} + +func (r *listingTestRows) Scan(dest ...any) error { + if r.scanErr != nil { + return r.scanErr + } + *dest[0].(*string), *dest[1].(*string), *dest[2].(*string) = "example/server", "1.0", "active" + *dest[3].(*time.Time) = time.Unix(100, 0).UTC() + *dest[5].(*time.Time), *dest[6].(*time.Time) = time.Unix(200, 0).UTC(), time.Unix(300, 0).UTC() + *dest[7].(*bool) = true + *dest[8].(*[]byte) = []byte(r.values[r.read-1]) + return nil +} + +func (r *listingTestRows) Close() { + if !r.closed && r.onClose != nil { + r.onClose() + } + r.closed = true +} + +func (r *listingTestRows) Err() error { return r.endErr } + +func TestListingReleasesRowsBeforeJSONDecode(t *testing.T) { + // The first row is malformed. Decoding in the fetch loop would return before + // consuming the second row and leave application work inside the query span. + rows := &listingTestRows{values: []string{"{", `{}`}} + db := &database.PostgreSQL{} + _, _, err := db.ListServers(context.Background(), listingTestTx{rows: rows}, nil, "", 2) + require.ErrorContains(t, err, "failed to unmarshal") + require.Equal(t, 2, rows.read) + require.True(t, rows.closed) +} + +func TestListingReleaseErrorsAndCancellation(t *testing.T) { + sentinel := errors.New("row failure") + for _, test := range []struct { + name string + rows *listingTestRows + }{ + {"scan", &listingTestRows{values: []string{`{}`}, scanErr: sentinel}}, + {"iteration", &listingTestRows{values: []string{`{}`}, endErr: sentinel}}, + } { + t.Run(test.name, func(t *testing.T) { + _, _, err := (&database.PostgreSQL{}).ListServers(context.Background(), listingTestTx{rows: test.rows}, nil, "", 2) + require.ErrorIs(t, err, sentinel) + require.True(t, test.rows.closed) + }) + } + t.Run("canceled before decode", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + rows := &listingTestRows{values: []string{"{"}, onClose: cancel} + _, _, err := (&database.PostgreSQL{}).ListServers(ctx, listingTestTx{rows: rows}, nil, "", 2) + require.ErrorIs(t, err, context.Canceled) + require.True(t, rows.closed) + }) +} diff --git a/internal/database/migrations/018_add_prepared_listing_json.sql b/internal/database/migrations/018_add_prepared_listing_json.sql new file mode 100644 index 000000000..9b32b32ea --- /dev/null +++ b/internal/database/migrations/018_add_prepared_listing_json.sql @@ -0,0 +1,23 @@ +-- Derived, versioned HTTP representation. Nullable columns avoid a table rewrite +-- and let existing rows use the ordinary typed normalization until backfilled. +SET LOCAL lock_timeout = '5s'; +ALTER TABLE servers ADD COLUMN listing_value bytea; +ALTER TABLE servers ADD COLUMN listing_codec integer; + +-- Older binaries update only value. Invalidate their derived bytes immediately +-- so rolling deploys and rollback cannot serve stale payloads. Metadata is read +-- separately on every request and does not require invalidation. +CREATE FUNCTION invalidate_listing_json() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NEW.value IS DISTINCT FROM OLD.value + AND NEW.listing_value IS NOT DISTINCT FROM OLD.listing_value + AND NEW.listing_codec IS NOT DISTINCT FROM OLD.listing_codec THEN + NEW.listing_value := NULL; + NEW.listing_codec := NULL; + END IF; + RETURN NEW; +END +$$; +CREATE TRIGGER invalidate_listing_json_before_update +BEFORE UPDATE OF value ON servers +FOR EACH ROW EXECUTE FUNCTION invalidate_listing_json(); diff --git a/internal/database/pool_limits.go b/internal/database/pool_limits.go new file mode 100644 index 000000000..22cd20bbd --- /dev/null +++ b/internal/database/pool_limits.go @@ -0,0 +1,19 @@ +package database + +import "fmt" + +// PoolLimits applies to one application replica, not the whole deployment. +type PoolLimits struct { + MaxConns int32 + MinConns int32 +} + +func (p PoolLimits) Validate() error { + if p.MaxConns < 1 { + return fmt.Errorf("database maximum connections must be positive") + } + if p.MinConns < 0 || p.MinConns > p.MaxConns { + return fmt.Errorf("database minimum connections must be between zero and the maximum") + } + return nil +} diff --git a/internal/database/pool_limits_internal_test.go b/internal/database/pool_limits_internal_test.go new file mode 100644 index 000000000..0f5e37e0d --- /dev/null +++ b/internal/database/pool_limits_internal_test.go @@ -0,0 +1,37 @@ +package database + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestPoolLimitsRejectInvalidConfigurationBeforeConnecting(t *testing.T) { + for _, limits := range []PoolLimits{{0, 0}, {-1, 0}, {30, -1}, {30, 31}} { + _, err := NewPostgreSQLWithPoolLimits(context.Background(), "invalid connection URI", limits) + require.ErrorContains(t, err, "database") + } +} + +func TestExplicitPoolBudgetIsEnforced(t *testing.T) { + original := NewTestDB(t).(*PostgreSQL) + require.EqualValues(t, 60, original.pool.Config().MaxConns) + require.EqualValues(t, 10, original.pool.Config().MinConns) + db, err := NewPostgreSQLWithPoolLimits(context.Background(), original.pool.Config().ConnString(), PoolLimits{MaxConns: 30, MinConns: 5}) + require.NoError(t, err) + defer db.Close() + require.EqualValues(t, 30, db.pool.Config().MaxConns) + require.EqualValues(t, 5, db.pool.Config().MinConns) + for range 30 { + conn, err := db.pool.Acquire(context.Background()) + require.NoError(t, err) + defer conn.Release() + } + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + _, err = db.pool.Acquire(ctx) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.EqualValues(t, 30, db.pool.Stat().AcquiredConns()) +} diff --git a/internal/database/postgres.go b/internal/database/postgres.go index 00e2f2ab3..5b4144fa5 100644 --- a/internal/database/postgres.go +++ b/internal/database/postgres.go @@ -22,8 +22,9 @@ import ( // PostgreSQL is an implementation of the Database interface using PostgreSQL type PostgreSQL struct { - pool *pgxpool.Pool - poolMetrics metric.Registration + pool *pgxpool.Pool + poolMetrics metric.Registration + listingMetrics metric.Int64Counter } // Executor is an interface for executing queries (satisfied by both pgx.Tx and pgxpool.Pool) @@ -43,19 +44,24 @@ func (db *PostgreSQL) getExecutor(tx pgx.Tx) Executor { // NewPostgreSQL creates a new instance of the PostgreSQL database func NewPostgreSQL(ctx context.Context, connectionURI string) (*PostgreSQL, error) { + return NewPostgreSQLWithPoolLimits(ctx, connectionURI, PoolLimits{MaxConns: 60, MinConns: 10}) +} + +// NewPostgreSQLWithPoolLimits sets an explicit per-replica connection budget. +func NewPostgreSQLWithPoolLimits(ctx context.Context, connectionURI string, limits PoolLimits) (*PostgreSQL, error) { + if err := limits.Validate(); err != nil { + return nil, err + } // Parse connection config for pool settings config, err := pgxpool.ParseConfig(connectionURI) if err != nil { return nil, fmt.Errorf("failed to parse PostgreSQL config: %w", err) } - // Configure pool for stability-focused defaults. - // MaxConns was 30 per pod (60 total across 2 replicas) which saturated under - // the 2026-04-28 scraper bursts (15 req/s on /v0/servers caused queue blowup - // even though individual queries were fast). 60 per pod gives 120 total, - // leaving 80 of PG max_connections=200 for autovacuum/admin/headroom. - config.MaxConns = 60 // Handle scraper-burst concurrent load - config.MinConns = 10 // Keep connections warm for fast response + // Keep the aggregate budget bounded when changing the replica count. The + // default constructor retains the existing 60 maximum / 10 minimum. + config.MaxConns = limits.MaxConns + config.MinConns = limits.MinConns config.MaxConnIdleTime = 30 * time.Minute // Keep connections available for bursts config.MaxConnLifetime = 2 * time.Hour // Refresh connections regularly for stability dbMetrics, err := telemetry.NewDatabaseMetrics(otel.Meter("mcp_registry.database")) @@ -96,11 +102,17 @@ func NewPostgreSQL(ctx context.Context, connectionURI string) (*PostgreSQL, erro if err != nil { return nil, fmt.Errorf("observe database pool: %w", err) } + listingMetrics, err := otel.Meter("mcp_registry.database").Int64Counter("mcp_registry.db.listing_json.rows", metric.WithDescription("Listing rows served from prepared JSON or normalization fallback")) + if err != nil { + _ = registration.Unregister() + return nil, fmt.Errorf("initialize listing metrics: %w", err) + } initialized = true return &PostgreSQL{ - pool: pool, - poolMetrics: registration, + listingMetrics: listingMetrics, + pool: pool, + poolMetrics: registration, }, nil } @@ -208,7 +220,7 @@ func addCursorCondition(cursor string, argIndex int) (string, []any, int) { } // searchPageQuery retains the ordinary query for listings without a search. -func searchPageQuery(filter *ServerFilter, ordinary, whereClause string, limitArg int) string { +func searchPageQuery(filter *ServerFilter, ordinary, whereClause string, limitArg int, prepared ...bool) string { if filter == nil || filter.SubstringName == nil { return ordinary } @@ -219,6 +231,10 @@ func searchPageQuery(filter *ServerFilter, ordinary, whereClause string, limitAr // and can favor scanning most of the ordered index over a selective GIN // lookup. The outer LIMIT still defines the exact page before payload fetch. // PostgreSQL 17 can stop the ordered CTE scan after the requested rows. + payload := "s.value" + if len(prepared) > 0 && prepared[0] { + payload = listingPayload("s.") + } return fmt.Sprintf(` WITH candidates AS MATERIALIZED ( SELECT server_name, version FROM servers @@ -227,7 +243,7 @@ func searchPageQuery(filter *ServerFilter, ordinary, whereClause string, limitAr LIMIT GREATEST($%d, 8192) ) SELECT s.server_name, s.version, s.status, s.status_changed_at, s.status_message, - s.published_at, s.updated_at, s.is_latest, s.value + s.published_at, s.updated_at, s.is_latest, %s FROM ( SELECT server_name, version FROM candidates ORDER BY server_name, version @@ -235,7 +251,7 @@ func searchPageQuery(filter *ServerFilter, ordinary, whereClause string, limitAr ) page JOIN servers s USING (server_name, version) ORDER BY s.server_name, s.version - `, whereClause, limitArg, limitArg) + `, whereClause, limitArg, payload, limitArg) } func searchQueryArgs(filter *ServerFilter, args []any) []any { @@ -248,15 +264,37 @@ func searchQueryArgs(filter *ServerFilter, args []any) []any { return args } -func (db *PostgreSQL) ListServers( +func (db *PostgreSQL) ListServers(ctx context.Context, tx pgx.Tx, filter *ServerFilter, cursor string, limit int) (_ []*apiv0.ServerResponse, _ string, resultErr error) { + diagnostic := telemetry.ListingFromContext(ctx) + defer func() { diagnostic.DatabaseDone(resultErr) }() + page, err := db.listServerRows(ctx, tx, filter, cursor, limit, false) + if err != nil { + return nil, "", err + } + results, err := decodeListingPage(ctx, page, diagnostic) + if err != nil { + return nil, "", err + } + if limit <= 0 { + limit = 10 + } + nextCursor := "" + if len(results) > 0 && len(results) >= limit { + last := results[len(results)-1] + nextCursor = last.Server.Name + ":" + last.Server.Version + } + return results, nextCursor, nil +} + +func (db *PostgreSQL) listServerRows( ctx context.Context, tx pgx.Tx, filter *ServerFilter, cursor string, limit int, -) (_ []*apiv0.ServerResponse, _ string, resultErr error) { + prepared bool, +) ([]listingRow, error) { diagnostic := telemetry.ListingFromContext(ctx) - defer func() { diagnostic.DatabaseDone(resultErr) }() operation := "list" switch { case filter != nil && filter.SubstringName != nil: @@ -281,7 +319,7 @@ func (db *PostgreSQL) ListServers( } diagnostic.SetShape(shape) if ctx.Err() != nil { - return nil, "", ctx.Err() + return nil, ctx.Err() } // Build WHERE clause conditions @@ -301,77 +339,94 @@ func (db *PostgreSQL) ListServers( whereClause = "WHERE " + strings.Join(whereConditions, " AND ") } - // Query servers table with hybrid column/JSON data + // Select prepared bytes only for HTTP serialization; typed callers retain JSONB. + payload := "value" + if prepared { + payload = listingPayload("") + } query := fmt.Sprintf(` - SELECT server_name, version, status, status_changed_at, status_message, published_at, updated_at, is_latest, value + SELECT server_name, version, status, status_changed_at, status_message, published_at, updated_at, is_latest, %s FROM servers %s ORDER BY server_name, version LIMIT $%d - `, whereClause, argIndex) - query = searchPageQuery(filter, query, whereClause, argIndex) + `, payload, whereClause, argIndex) + query = searchPageQuery(filter, query, whereClause, argIndex, prepared) args = append(args, limit) args = searchQueryArgs(filter, args) rows, err := db.getExecutor(tx).Query(ctx, query, args...) if err != nil { - return nil, "", fmt.Errorf("failed to query servers: %w", err) + return nil, fmt.Errorf("failed to query servers: %w", err) } + return readListingRows(ctx, rows, diagnostic, prepared) +} + +type listingRow struct { + metadata apiv0.RegistryExtensions + value []byte + name, version string + prepared bool +} + +func readListingRows(ctx context.Context, rows pgx.Rows, diagnostic *telemetry.ListingDiagnostic, prepared bool) ([]listingRow, error) { defer rows.Close() - var results []*apiv0.ServerResponse + // Copy the bounded page before decoding JSON. Exhausting/closing pooled rows + // releases the connection, so application CPU work cannot keep it occupied. + // Within a caller-owned transaction, closing rows only frees the transaction + // for its next query; the caller still owns the transaction's connection. + var page []listingRow diagnostic.Phase("row_fetch") for rows.Next() { diagnostic.RowReceived() - var serverName, version, status string - var statusChangedAt, publishedAt, updatedAt time.Time - var statusMessage *string - var isLatest bool - var valueJSON []byte - - err := rows.Scan(&serverName, &version, &status, &statusChangedAt, &statusMessage, &publishedAt, &updatedAt, &isLatest, &valueJSON) - if err != nil { - return nil, "", fmt.Errorf("failed to scan server row: %w", err) + diagnostic.Phase("row_fetch") + if err := ctx.Err(); err != nil { + return nil, err } - - diagnostic.AddBytes(len(valueJSON)) - // Parse the ServerJSON from JSONB - var serverJSON apiv0.ServerJSON - if err := json.Unmarshal(valueJSON, &serverJSON); err != nil { - return nil, "", fmt.Errorf("failed to unmarshal server JSON: %w", err) + var status string + var row listingRow + meta := &row.metadata + // pgx scans JSONB/bytea into owned bytes, valid after Next and Close. + dest := []any{&row.name, &row.version, &status, &meta.StatusChangedAt, &meta.StatusMessage, + &meta.PublishedAt, &meta.UpdatedAt, &meta.IsLatest, &row.value} + if prepared { + dest = append(dest, &row.prepared) } - - // Build ServerResponse with separated metadata - serverResponse := &apiv0.ServerResponse{ - Server: serverJSON, - Meta: apiv0.ResponseMeta{ - Official: &apiv0.RegistryExtensions{ - Status: model.Status(status), - StatusChangedAt: statusChangedAt, - StatusMessage: statusMessage, - PublishedAt: publishedAt, - UpdatedAt: updatedAt, - IsLatest: isLatest, - }, - }, + if err := rows.Scan(dest...); err != nil { + return nil, fmt.Errorf("failed to scan server row: %w", err) } - - results = append(results, serverResponse) - diagnostic.Phase("row_fetch") + meta.Status = model.Status(status) + diagnostic.AddBytes(len(row.value)) + page = append(page, row) } - + rows.Close() if err := rows.Err(); err != nil { - return nil, "", fmt.Errorf("error iterating rows: %w", err) + return nil, fmt.Errorf("error iterating rows: %w", err) } - // Determine next cursor using compound serverName:version format - nextCursor := "" - if len(results) > 0 && len(results) >= limit { - lastResult := results[len(results)-1] - nextCursor = lastResult.Server.Name + ":" + lastResult.Server.Version + return page, nil +} + +func decodeListingPage(ctx context.Context, page []listingRow, diagnostic *telemetry.ListingDiagnostic) ([]*apiv0.ServerResponse, error) { + var results []*apiv0.ServerResponse + diagnostic.Phase("row_decode") + for i := range page { + if err := ctx.Err(); err != nil { + return nil, err + } + var serverJSON apiv0.ServerJSON + if err := json.Unmarshal(page[i].value, &serverJSON); err != nil { + return nil, fmt.Errorf("failed to unmarshal server JSON: %w", err) + } + page[i].value = nil // Drop each copied payload as its decoded form replaces it. + results = append(results, &apiv0.ServerResponse{ + Server: serverJSON, + Meta: apiv0.ResponseMeta{Official: &page[i].metadata}, + }) } - return results, nextCursor, nil + return results, nil } // GetServerByName retrieves the latest version of a server by server name @@ -609,10 +664,15 @@ func (db *PostgreSQL) CreateServer(ctx context.Context, tx pgx.Tx, serverJSON *a return nil, fmt.Errorf("failed to marshal server JSON: %w", err) } + normalized, err := normalizeListingJSON(valueJSON) + if err != nil { + return nil, err + } + // Insert the new server version using composite primary key insertQuery := ` - INSERT INTO servers (server_name, version, status, status_changed_at, status_message, published_at, updated_at, is_latest, value) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + INSERT INTO servers (server_name, version, status, status_changed_at, status_message, published_at, updated_at, is_latest, value, listing_value, listing_codec) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ` _, err = db.getExecutor(tx).Exec(ctx, insertQuery, @@ -625,6 +685,8 @@ func (db *PostgreSQL) CreateServer(ctx context.Context, tx pgx.Tx, serverJSON *a officialMeta.UpdatedAt, officialMeta.IsLatest, valueJSON, + normalized.json, + listingCodecVersion, ) if err != nil { return nil, fmt.Errorf("failed to insert server: %w", err) @@ -663,10 +725,15 @@ func (db *PostgreSQL) UpdateServer(ctx context.Context, tx pgx.Tx, serverName, v return nil, fmt.Errorf("failed to marshal updated server: %w", err) } - // Update only the JSON data (keep existing metadata columns) + normalized, err := normalizeListingJSON(valueJSON) + if err != nil { + return nil, err + } + + // Update payload and its derived response together. query := ` UPDATE servers - SET value = $1, updated_at = NOW() + SET value = $1, updated_at = NOW(), listing_value = $4, listing_codec = $5 WHERE server_name = $2 AND version = $3 RETURNING server_name, version, status, status_changed_at, status_message, published_at, updated_at, is_latest ` @@ -676,7 +743,7 @@ func (db *PostgreSQL) UpdateServer(ctx context.Context, tx pgx.Tx, serverName, v var statusMessage *string var isLatest bool - err = db.getExecutor(tx).QueryRow(ctx, query, valueJSON, serverName, version).Scan(&name, &vers, &status, &statusChangedAt, &statusMessage, &publishedAt, &updatedAt, &isLatest) + err = db.getExecutor(tx).QueryRow(ctx, query, valueJSON, serverName, version, normalized.json, listingCodecVersion).Scan(&name, &vers, &status, &statusChangedAt, &statusMessage, &publishedAt, &updatedAt, &isLatest) if err != nil { if errors.Is(err, pgx.ErrNoRows) { return nil, ErrNotFound diff --git a/internal/service/registry_service.go b/internal/service/registry_service.go index e65dce2ac..a581fe804 100644 --- a/internal/service/registry_service.go +++ b/internal/service/registry_service.go @@ -151,6 +151,28 @@ func (s *registryServiceImpl) ListServers(ctx context.Context, filter *database. return serverRecords, nextCursor, nil } +// ListServersJSON uses the database's normalized wire cache when available. +// Alternate database implementations retain the ordinary typed path. +func (s *registryServiceImpl) ListServersJSON(ctx context.Context, filter *database.ServerFilter, cursor string, limit int) (apiv0.ServerListResponse, error) { + if limit <= 0 { + limit = 30 + } + if db, ok := s.db.(interface { + ListServersJSON(context.Context, *database.ServerFilter, string, int) (apiv0.ServerListResponse, error) + }); ok { + return db.ListServersJSON(ctx, filter, cursor, limit) + } + servers, next, err := s.ListServers(ctx, filter, cursor, limit) + if err != nil { + return apiv0.ServerListResponse{}, err + } + values := make([]apiv0.ServerResponse, len(servers)) + for i, server := range servers { + values[i] = *server + } + return apiv0.ServerListResponse{Servers: values, Metadata: apiv0.Metadata{Count: len(values), NextCursor: next}}, nil +} + // GetServerByName retrieves the latest version of a server by its server name func (s *registryServiceImpl) GetServerByName(ctx context.Context, serverName string, includeDeleted bool) (*apiv0.ServerResponse, error) { serverRecord, err := s.db.GetServerByName(ctx, nil, serverName, includeDeleted) diff --git a/internal/telemetry/checkout_test.go b/internal/telemetry/checkout_test.go new file mode 100644 index 000000000..84968cc7d --- /dev/null +++ b/internal/telemetry/checkout_test.go @@ -0,0 +1,49 @@ +package telemetry_test + +import ( + "context" + "testing" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/require" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" + + "github.com/modelcontextprotocol/registry/internal/telemetry" +) + +func TestCheckoutTimingTracksOwnershipAndReuse(t *testing.T) { + reader := sdkmetric.NewManualReader() + provider := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + t.Cleanup(func() { require.NoError(t, provider.Shutdown(context.Background())) }) + m, err := telemetry.NewDatabaseMetrics(provider.Meter("test")) + require.NoError(t, err) + conn := &pgx.Conn{} + ctx := telemetry.WithDatabaseOperation(context.Background(), "list_page") + ctx = m.TraceAcquireStart(ctx, nil, pgxpool.TraceAcquireStartData{}) + m.TraceAcquireEnd(ctx, nil, pgxpool.TraceAcquireEndData{Conn: conn}) + queryCtx := m.TraceQueryStart(ctx, nil, pgx.TraceQueryStartData{}) + m.TraceQueryEnd(queryCtx, conn, pgx.TraceQueryEndData{}) + time.Sleep(20 * time.Millisecond) + _, exists := collectDBMetrics(t, reader)["mcp_registry.db.pool.checkout.duration"] + require.False(t, exists, "a still-owned connection has not completed a checkout") + m.TraceRelease(nil, pgxpool.TraceReleaseData{Conn: conn}) + m.TraceRelease(nil, pgxpool.TraceReleaseData{Conn: conn}) + data := collectDBMetrics(t, reader) + for _, name := range []string{"mcp_registry.db.pool.checkout.duration", "mcp_registry.db.pool.after_query.duration"} { + h := data[name].(metricdata.Histogram[float64]) + require.EqualValues(t, 1, h.DataPoints[0].Count) + require.GreaterOrEqual(t, h.DataPoints[0].Sum, .020) + require.Equal(t, 1, h.DataPoints[0].Attributes.Len()) + } + // Reusing the same connection without a query must not inherit its prior + // query-end timestamp. A failed acquisition must not produce a checkout. + m.TraceAcquireEnd(ctx, nil, pgxpool.TraceAcquireEndData{Conn: conn}) + m.TraceRelease(nil, pgxpool.TraceReleaseData{Conn: conn}) + m.TraceAcquireEnd(ctx, nil, pgxpool.TraceAcquireEndData{Err: context.Canceled}) + data = collectDBMetrics(t, reader) + require.EqualValues(t, 2, data["mcp_registry.db.pool.checkout.duration"].(metricdata.Histogram[float64]).DataPoints[0].Count) + require.EqualValues(t, 1, data["mcp_registry.db.pool.after_query.duration"].(metricdata.Histogram[float64]).DataPoints[0].Count) +} diff --git a/internal/telemetry/database.go b/internal/telemetry/database.go index 2c2ae66cd..9f8304bea 100644 --- a/internal/telemetry/database.go +++ b/internal/telemetry/database.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sync" "time" "github.com/jackc/pgx/v5" @@ -43,12 +44,22 @@ func databaseOperation(ctx context.Context) string { // timing starts after acquisition and ends when rows are closed or exhausted. // It includes network transfer and row consumption, not just PostgreSQL CPU. type DatabaseMetrics struct { - meter metric.Meter - acquireDuration metric.Float64Histogram - queryDuration metric.Float64Histogram - pending metric.Int64UpDownCounter + meter metric.Meter + acquireDuration metric.Float64Histogram + queryDuration metric.Float64Histogram + pending metric.Int64UpDownCounter + checkoutDuration metric.Float64Histogram + afterQueryDuration metric.Float64Histogram + loansMu sync.Mutex + loans map[*pgx.Conn]connectionLoan } +type connectionLoan struct { + started, queryEnded time.Time + operation string +} + +var _ pgxpool.ReleaseTracer = (*DatabaseMetrics)(nil) var _ pgx.QueryTracer = (*DatabaseMetrics)(nil) var _ pgxpool.AcquireTracer = (*DatabaseMetrics)(nil) @@ -71,7 +82,18 @@ func NewDatabaseMetrics(meter metric.Meter) (*DatabaseMetrics, error) { if err != nil { return nil, fmt.Errorf("create pending acquisition counter: %w", err) } - return &DatabaseMetrics{meter: meter, acquireDuration: acquire, queryDuration: query, pending: pending}, nil + checkout, err := meter.Float64Histogram(Namespace+".db.pool.checkout.duration", + metric.WithDescription("Time from successful acquisition to entry into Release; excludes pool wait and release bookkeeping"), buckets) + if err != nil { + return nil, fmt.Errorf("create checkout histogram: %w", err) + } + afterQuery, err := meter.Float64Histogram(Namespace+".db.pool.after_query.duration", + metric.WithDescription("Time from the final query tracer completion (including row consumption) to entry into Release; excludes checkouts without a query"), buckets) + if err != nil { + return nil, fmt.Errorf("create post-query histogram: %w", err) + } + return &DatabaseMetrics{meter: meter, acquireDuration: acquire, queryDuration: query, pending: pending, + checkoutDuration: checkout, afterQueryDuration: afterQuery, loans: make(map[*pgx.Conn]connectionLoan)}, nil } // ObservePool registers scrape-time pool statistics. Unregister before discarding @@ -107,6 +129,11 @@ func (m *DatabaseMetrics) TraceAcquireStart(ctx context.Context, pool *pgxpool.P } func (m *DatabaseMetrics) TraceAcquireEnd(ctx context.Context, _ *pgxpool.Pool, data pgxpool.TraceAcquireEndData) { + if data.Err == nil && data.Conn != nil { + m.loansMu.Lock() + m.loans[data.Conn] = connectionLoan{started: time.Now(), operation: databaseOperation(ctx)} + m.loansMu.Unlock() + } start, ok := ctx.Value(acquireStartKey{}).(time.Time) if !ok { return @@ -123,7 +150,13 @@ func (m *DatabaseMetrics) TraceQueryStart(ctx context.Context, conn *pgx.Conn, _ return context.WithValue(ctx, queryStartKey{}, time.Now()) } -func (m *DatabaseMetrics) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { +func (m *DatabaseMetrics) TraceQueryEnd(ctx context.Context, conn *pgx.Conn, data pgx.TraceQueryEndData) { + m.loansMu.Lock() + if loan, ok := m.loans[conn]; ok { + loan.queryEnded = time.Now() + m.loans[conn] = loan + } + m.loansMu.Unlock() start, ok := ctx.Value(queryStartKey{}).(time.Time) if !ok { return @@ -144,3 +177,22 @@ func databaseAttributes(ctx context.Context, err error) metric.MeasurementOption } return metric.WithAttributes(attribute.String("operation", databaseOperation(ctx)), attribute.String("outcome", outcome)) } + +// TraceRelease measures ownership, not PostgreSQL execution. A transaction may +// run several queries; after_query starts at the final query's tracer callback. +func (m *DatabaseMetrics) TraceRelease(_ *pgxpool.Pool, data pgxpool.TraceReleaseData) { + ended := time.Now() + m.loansMu.Lock() + loan, ok := m.loans[data.Conn] + delete(m.loans, data.Conn) + m.loansMu.Unlock() + if !ok { + return + } + ctx := context.Background() + attrs := metric.WithAttributes(attribute.String("operation", loan.operation)) + m.checkoutDuration.Record(ctx, ended.Sub(loan.started).Seconds(), attrs) + if !loan.queryEnded.IsZero() { + m.afterQueryDuration.Record(ctx, ended.Sub(loan.queryEnded).Seconds(), attrs) + } +} diff --git a/internal/telemetry/profiling.go b/internal/telemetry/profiling.go new file mode 100644 index 000000000..d7b378584 --- /dev/null +++ b/internal/telemetry/profiling.go @@ -0,0 +1,48 @@ +package telemetry + +import ( + "context" + "errors" + "fmt" + "log" + "net" + "net/http" + "net/http/pprof" + "strconv" + "time" +) + +// StartProfiling is opt-in and binds only to pod loopback. No profiling routes +// are registered on the application mux or exposed by its Service/Ingress. +func StartProfiling(enabled bool) (func(), error) { + if !enabled { + return func() {}, nil + } + var config net.ListenConfig + listener, err := config.Listen(context.Background(), "tcp", "127.0.0.1:6060") + if err != nil { + return nil, fmt.Errorf("listen for loopback profiling: %w", err) + } + server := &http.Server{Handler: profilingMux(), ReadHeaderTimeout: 5 * time.Second, WriteTimeout: 70 * time.Second} + go func() { + if err := server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Printf("Profiling server stopped: %v", err) + } + }() + return func() { _ = server.Close() }, nil +} + +func profilingMux() *http.ServeMux { + mux := http.NewServeMux() + mux.HandleFunc("GET /debug/pprof/profile", func(w http.ResponseWriter, r *http.Request) { + seconds, err := strconv.Atoi(r.URL.Query().Get("seconds")) + if err != nil || seconds < 1 || seconds > 60 { + http.Error(w, "seconds must be between 1 and 60", http.StatusBadRequest) + return + } + pprof.Profile(w, r) + }) + mux.Handle("GET /debug/pprof/goroutine", pprof.Handler("goroutine")) + mux.Handle("GET /debug/pprof/heap", pprof.Handler("heap")) + return mux +} diff --git a/internal/telemetry/profiling_internal_test.go b/internal/telemetry/profiling_internal_test.go new file mode 100644 index 000000000..c447dde1f --- /dev/null +++ b/internal/telemetry/profiling_internal_test.go @@ -0,0 +1,26 @@ +package telemetry + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestProfilingRoutesAreExplicitAndBounded(t *testing.T) { + mux := profilingMux() + for _, path := range []string{"/debug/pprof/profile", "/debug/pprof/profile?seconds=0", "/debug/pprof/profile?seconds=61"} { + w := httptest.NewRecorder() + mux.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + require.Equal(t, http.StatusBadRequest, w.Code) + } + for _, path := range []string{"/debug/pprof/cmdline", "/debug/pprof/symbol", "/debug/pprof/trace", "/"} { + w := httptest.NewRecorder() + mux.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + require.Equal(t, http.StatusNotFound, w.Code) + } + stop, err := StartProfiling(false) + require.NoError(t, err) + stop() +} diff --git a/pkg/api/v0/types.go b/pkg/api/v0/types.go index 67fe09402..f63437fbf 100644 --- a/pkg/api/v0/types.go +++ b/pkg/api/v0/types.go @@ -1,6 +1,7 @@ package v0 import ( + "encoding/json" "time" "github.com/modelcontextprotocol/registry/pkg/model" @@ -25,6 +26,9 @@ type ServerResponse struct { } type ServerListResponse struct { + // encoded contains immutable, normalized wire records for HTTP-only listing. + // Internal database/service operations continue to use typed ServerResponse. + encoded []EncodedServerRecord Servers []ServerResponse `json:"servers" doc:"List of server entries"` Metadata Metadata `json:"metadata" doc:"Pagination metadata"` } @@ -51,3 +55,57 @@ type Metadata struct { NextCursor string `json:"nextCursor,omitempty" doc:"Pagination cursor for retrieving the next page of results. Use this exact value in the cursor query parameter of your next request."` Count int `json:"count" doc:"Number of items in current page"` } + +// NewEncodedServerList creates a serialization-only listing. Each record must +// contain normalized ServerJSON bytes and current metadata. It takes ownership of +// records; callers must not mutate them after construction. +func NewEncodedServerList(records []EncodedServerRecord, metadata Metadata) ServerListResponse { + if records == nil { + records = make([]EncodedServerRecord, 0) + } + return ServerListResponse{encoded: records, Metadata: metadata} +} + +func (s ServerListResponse) MarshalJSON() ([]byte, error) { + if s.encoded == nil { + type plain ServerListResponse + return json.Marshal(plain(s)) + } + // Payloads were normalized with json.Marshal before storage. Metadata is + // freshly encoded here; append avoids copying every large payload through + // intermediate record and slice encoders. The outer encoder validates JSON. + capacity := 64 + for _, record := range s.encoded { + capacity += len(record.Server) + 384 + } + result := make([]byte, 0, capacity) + result = append(result, `{"servers":[`...) + for i, record := range s.encoded { + if i > 0 { + result = append(result, ',') + } + meta, err := json.Marshal(record.Meta) + if err != nil { + return nil, err + } + result = append(result, `{"server":`...) + result = append(result, record.Server...) + result = append(result, `,"_meta":`...) + result = append(result, meta...) + result = append(result, '}') + } + metadata, err := json.Marshal(s.Metadata) + if err != nil { + return nil, err + } + result = append(result, `],"metadata":`...) + result = append(result, metadata...) + return append(result, '}'), nil +} + +// EncodedServerRecord contains a normalized server payload and fresh metadata. +// It is used only to construct a serialization-only ServerListResponse. +type EncodedServerRecord struct { + Server json.RawMessage + Meta ResponseMeta +} diff --git a/tools/ingressbench/CONNECTION_CPU_VALIDATION.md b/tools/ingressbench/CONNECTION_CPU_VALIDATION.md new file mode 100644 index 000000000..1df229718 --- /dev/null +++ b/tools/ingressbench/CONNECTION_CPU_VALIDATION.md @@ -0,0 +1,140 @@ +# Connection lifetime and app CPU validation — 2026-10-28 + +**Do not promote this candidate as a complete burst-capacity fix.** A larger +application CPU reservation eliminated attempted-request failures in this run, +but 4.67% of the offered reads still could not be dispatched. The candidate also +retained three nodes at idle, missing the two-node target. + +## Changes and controls + +`ListServers` now consumes a bounded page into owned JSONB byte slices and closes +the rows before decoding JSON. This releases a pooled connection before that +application work. A caller-owned transaction still retains its own connection. +SQL, filtering, ordering and cursor behavior are unchanged. Forty-eight direct +origin API response hashes matched the original implementation before each run. + +The CPU comparison changed only the app CPU request from 250m to 1 CPU, using +the same connection-release image. Both runs retained two app replicas, pools +of max 60/min 10 per replica, a 1-GiB app memory limit, and no app CPU limit. +PostgreSQL retained its 2-CPU request, 3-CPU limit, 4-GiB memory limit, 512-MB +shared buffers and 200-connection ceiling. Ingress/node scaling settings were +unchanged from the preceding experiment. + +Each burst offered 4,500 HTTP/1.1 reads/s for ten minutes, including 36 uncached +incremental-sync reads/s and three searches/s, plus one independent publish/s. +The same binary, 4,000-key catalog, 80% hot-key selection, 1,024 read workers and +1,024 queued reads were used. Recovery offered 100 reads/s and one publish/s for +30 minutes. Its driver added bounded failure timestamps without changing traffic +scheduling or transport settings. Each completed run's own benchmark records +were removed before the next comparison; the base corpus contained 187,108 +versions and 52,366 server names. + +## Client results + +| Measurement | Connection release, 250m | Same image, 1 CPU | +| --- | ---: | ---: | +| Burst start, UTC | Oct 27 23:54:40 | Oct 28 00:42:40 | +| Offered reads | 2,700,000 | 2,700,000 | +| Reads dropped by the bounded generator | 471,814 (17.47%) | 125,976 (4.67%) | +| Attempted read failures | 269 | 0 | +| Acknowledged publishes | 597/600 | 600/600 | +| Publish timeouts | 3 | 0 | +| Publish p95 / p99 | 7.588 / 9.519 s | 1.350 / 1.924 s | +| Page p95 / p99 | 732 / 6,103 ms | 597 / 1,129 ms | +| Peak sampled pool waiters, total | 1,013 | 116 | +| Starting node count | 2 | 3 | +| Peak node / ingress count | 4 / 6 | 4 / 6 | +| Observed idle nodes | 2 | 3 | +| Recovery reads succeeded | 180,000/180,000 | 180,000/180,000 | +| Recovery publishes succeeded | 1,800/1,800 | 1,800/1,800 | +| Recovery dropped work / failures | 0 / 0 | 0 / 0 | +| Recovery publish p95 | 13.9 ms | 14.2 ms | + +Client latency includes scheduling delay through response-body completion. The +125,976 dropped reads were never dispatched because the bounded generator queue +was full; they are not HTTP failures, and must not be hidden behind the zero +attempted-request failure count. Two timed-out publishes in the 250m run committed +despite the client timing out: 599 records persisted, versus 597 acknowledgments. + +The predeclared experimental gate required no failed or dropped work, page p95 +at most one second and publish p95 at most two seconds, followed by clean recovery +and a repeat of a passing candidate. These are experiment criteria, not an +established production SLO. The 1-CPU run met the latency and attempted-request +criteria but failed the offered-load criterion. A passing repeat is therefore +not established. + +## What the evidence establishes + +The connection-only change was insufficient. Its dominant page statement +averaged 3.40 ms of PostgreSQL execution while pool queues grew and publishes +timed out. During a captured late-burst interval, the app sharing the database +node spent about 46% of wall time fully CPU-stalled according to its cgroup PSI +counter, versus about 13% for the other app. These samples are not a complete +burst average. + +In the 1-CPU run neither app shared the registry database node. The dominant +page statement averaged 3.00 ms, pool queues were much smaller, and client +publish latency improved. One app still experienced substantially more CPU +contention than the other: approximately 27% versus 8% full CPU-stall time over +the captured burst intervals. The settings improved the observed outcome; this +does not isolate the effect of CPU priority from the effect of placement or +the extra node already available at burst start. + +No increase in the database CPU-throttling counter was observed during either +burst, and no database endpoint loss or app/database restart was observed in +the successful burst snapshots. The 250m run recorded about 85 ms of additional +DB throttling late in recovery. These observations do not establish that larger +connection pools would help; the pool budget was deliberately unchanged. + +## Idle capacity, recovered telemetry and cleanup + +The 250m candidate naturally returned to two Ready nodes around 00:37:05 UTC, +approximately 32 minutes after its burst ended, without moving either app or +the registry database during the measured cycle. + +The 1-CPU rollout required a third node before load started. Scheduler events +reported insufficient CPU on the database node and the app topology-spread +constraint on the other node. The database node could not fit a 1-CPU app while +retaining its existing ingress pod and other reservations. It had 3,426m CPU +reserved out of 3,920m allocatable, leaving 494m: enough for the original 250m +app, but not a 1-CPU app. A node-pool minimum of two therefore does not establish +that this placement can return to two. + +Google Cloud reauthentication expired during recovery. Direct node snapshots +failed from 00:56:04 UTC, but the remote client benchmark completed at 01:23:09 +UTC. After reauthentication, the original result file was recovered and exactly +matched the JSON retained in SSH stdout. Historical application metrics were +backfilled: both app `up` series were healthy and every sampled pool-waiter gauge +was zero throughout recovery. The 60-second app scrape interval still limits +what these samples establish. Missing direct database cgroup and endpoint +snapshots from that interval cannot be recreated from those app metrics. + +GKE's retained autoscaler logs showed three nodes by 01:19:30 UTC. A live check at +02:38 UTC still found three Ready nodes and two ingress replicas. The autoscaler +explicitly reported `no.scale.down.node.no.place.to.move.pods` for the node +hosting the second app. Thus this is a confirmed placement obstacle, not merely +an unexpired scale-down timer. The 1-CPU candidate does not meet the two-node idle +requirement with its current reservations and placement constraints. + +Production was not changed. The temporary generator VM and its disk, temporary +rate-limit exemption and scoped benchmark publications have been removed. The +pre-experiment application image and 250m CPU request were restored, and all 48 +direct-origin response hashes still matched. Natural consolidation returned the +restored 250m configuration to two Ready nodes at approximately 02:59 UTC, with +both apps healthy on the original nodes. This occurred after reverting the +experimental settings; it does not turn the 1-CPU candidate's three-node idle +result into a pass. The candidate container image is retained for reproducibility. + +## Reproduction + +Application source: `3ea1fa2e974d97bff88cd0f04dc95fac5dea5f68`. +Image digest: `sha256:94f184cd38345b059f440756d5eaa22448dbc7d827490220d144bd9356713f94`. +Burst driver: `28ec057a098b8d01a5c8d7fb7fed6cd2d3d89dc2f35c201dff1173c2e6a00150`. +Recovery driver: `c747ec90766f5044632b08019d4fb5d32dbb36f10b73dff02821ba83263b0006`. +Catalog: `120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be`. + +Full internal/cmd race tests, database/telemetry tests, infrastructure Go tests +and root lint passed. GitHub checks passed at `3ce54f9`. Those checks establish +code validation, not a successful capacity test. See the adjacent +`results/2026-10-28-connection-cpu.json` for portable client measurements and +`README.md` for benchmark commands and measurement boundaries. diff --git a/tools/ingressbench/COORDINATED_CAPACITY.md b/tools/ingressbench/COORDINATED_CAPACITY.md new file mode 100644 index 000000000..279da1385 --- /dev/null +++ b/tools/ingressbench/COORDINATED_CAPACITY.md @@ -0,0 +1,212 @@ +# Coordinated CPU capacity candidate — 2026-10-28 + +**Failed the staging short delivery gate. Production is unchanged.** +After reauthentication, the candidate was applied in staging and tested with +four Ready apps, six Ready ingress replicas and four healthy node backends. +Of 810,000 scheduled reads, 7,418 (0.916%) were dropped before dispatch. +All 802,582 attempted requests succeeded, but their latency excludes dropped +work. The long burst, publish test and autoscaling recovery gate were not run. +The opt-in remains disabled; this result does not support promotion. + +Follow-up: the [production workload qualification](WORKLOAD_PROFILE.md) found +material API/page-size, cache-locality and duration differences in this stress +scenario. Its failed gate remains a failed gate, but it is not a validated +acceptance test for the actual production cycle. Qualify the workload baseline +before another capacity comparison. + +Latest follow-up: [the production read-trace comparison](WORKLOAD_COMPARISON.md) +measures the app improvement and the infrastructure transition separately, +with explicit cache-state and workload-coverage limitations. + +## Observed result + +The [saved result](evidence/coordinated-capacity-2026-10-28/summary.json) +and [raw driver output](evidence/coordinated-capacity-2026-10-28/coordinated-burst.json) +preserve the offered workload and limitations. + +| Measurement | Three-minute burst | Two-minute recovery | +| --- | ---: | ---: | +| Offered reads/s | 4,500 | 100 | +| Scheduled / dropped reads | 810,000 / 7,418 | 12,000 / 0 | +| Attempted request failures | 0 | 0 | +| Page p95 | 316 ms | 3.7 ms | +| Search p95 | 835 ms | 26.9 ms | +| Updated-listing p95 | 664 ms | 9.6 ms | + +The generator first completed 405,000 calibration requests without drops or +failures. During staging load its sixteen-vCPU VM averaged 17.8% CPU busy +(maximum 22.6%), with no observed network-device drops. This excludes neither +single-thread bottlenecks nor external network limitations. + +The app's connection hold time was short: listing checkout averaged 12–16 ms +across replicas, with less than 0.1 ms after query completion. However, listing +connection acquisition still averaged 29–135 ms. Every 15-second pending-queue +sample was zero: those snapshots missed waits captured by the histograms. +Acquisition includes pool and scheduler delay, so this does not establish that +a larger pool would help. App CPU full-stall rates were approximately 8.7–15.5% +over sampled intervals inside the burst; DB quota throttling remained zero. + +Placement also matters. The scheduler put apps 1/1/2/0 across four nodes; it did +not rebalance them when the fourth node arrived. The sole ingress on the DB +node used about 1.71 CPUs, versus 0.66–1.06 for the other controllers. That app +had the highest acquisition time and CPU stall rate. The previous four-app +test used hard one-app-per-node placement, and this test restarted PostgreSQL +before preparation, changing cache state. The apparent improvements are not +a controlled estimate of the CPU-request change alone. + +Drops recur roughly every 30 seconds. Both the generator's idle-connection +rotation and ingress cache TTL are 30 seconds; current aggregate timing cannot +separate their contribution. The next investigation should distinguish these +effects and the ingress hot node using the captured profiles and counters, +before proposing another capacity change. Do not interpret either quiet +recovery or lower attempted-request latency as a passing delivery gate. + +### Restoration + +The [verified cleanup](evidence/coordinated-capacity-2026-10-28/cleanup.json) +restored the exact original app specification (two replicas, 250m CPU request, +original image and pools), ingress HPA specification, DB resources (2-CPU +request / 3-CPU limit), and bulk-ingress whitelist. The app HPA was removed. +The generator VM and disk were confirmed absent. All 48 sampled response hashes +matched; the corpus remained 52,366 names / 187,108 versions with no benchmark +publications. The DB restarted during resource restoration and was Ready. + +At the final 20:21 UTC snapshot, staging still had four nodes and five +non-terminating ingress pods. Restored bounds allow 2–6 ingress replicas and +2–4 nodes; restoration does not prove this candidate returns naturally to two +nodes. No forced node deletion or long recovery test was performed. + +## Why this candidate + +The production review found CPU pressure in ingress, both apps and PostgreSQL +while both nodes were close to full CPU. Requests were approximately evenly +split across the apps. The 3-CPU DB quota was not exhausted: a limit is a ceiling, +not exclusive node capacity. Production requests 200m for PostgreSQL; the prior +staging tests requested 2 CPUs. This difference, and staging's candidate app +image, prevent treating staging as an exact control for today's production. + +The previous four-app and DB-isolation tests still failed their offered-load +gates. This candidate changes CPU allocation across the entire request path, +without increasing node size or the maximum of four nodes. It does not treat +app replica count or dedicated DB placement alone as a proven fix. + +| Setting | Production now | Candidate | +| --- | --- | --- | +| Nodes | 2 × e2-standard-4 | 2–4 × e2-standard-4 | +| App replicas | 2 | HPA 2–4 | +| App CPU request | 250m | 800m, no CPU limit | +| App HPA CPU target | None | 70% of request = 560m | +| Ingress replicas | 2 | HPA 2–6 | +| Ingress CPU request | 300m | 800m, no CPU limit | +| Ingress HPA CPU target | None | 70% of request = 560m | +| DB CPU request / limit | 200m / 3 CPUs | 1500m / 3 CPUs | +| App pool maximum / minimum | 60 / 10 each | 30 / 5 each | +| Total steady-state app pool maximum | 120 | 60–120 | + +Both HPAs allow two added pods per minute, use a ten-minute scale-down +stabilization window, then remove at most one pod per two minutes. CPU targets +are candidates for testing, not measured safe throughput thresholds. CPU-only +HPA can miss runnable work that cannot obtain CPU; pressure, queues and actual +delivery remain pass/fail evidence. Node provisioning and load-balancer health +checks add further delay. The existing 60-second ingress connection lifetime +and per-pod ingress metric scraping remain part of the burst profile. + +Placement uses the existing hard per-revision hostname spread with maxSkew 1. +Neither apps nor ingress are excluded from the DB node. A rollout may require +additional capacity. App rollouts use maxSurge 0 / maxUnavailable 1; terminating +pods can still overlap with replacements, so 120 is a steady-state pool budget, +not a hard global connection semaphore. Observe DB connections during rollouts. +Memory limits, DB buffers and max_connections=200 are unchanged. + +## Two-node budget + +The [original CPU packing calculation](evidence/coordinated-capacity-2026-10-28/cpu-budget.json) +uses the 15:51 UTC staging inventory. A fresh preflight rechecked system pod +requests, memory, volume locality, disruption budgets and autoscaler bounds. +Each node +has 3,920m allocatable CPU. Keeping the two singleton DB nodes, placing one app +and one ingress on each, and relocating the other movable workloads gives: + +| Idle node | Requested CPU | Unallocated requests | +| --- | ---: | ---: | +| Registry DB node | 3,726m | 194m | +| Monitoring DB node (fresh preflight) | 2,653m | 1,267m | + +With a 2-CPU DB request, the first placement would require 4,226m and would not +fit. The 1.5-CPU request is therefore a deliberate balance with the larger app +request, not an increase from the previous staging DB request. The DB can still +use up to its existing 3-CPU quota when CPU is available. It must demonstrate +acceptable latency and pressure under the benchmark. + +The four-node peak packing fits four apps and six ingress controllers in CPU +requests. These are constructive request budgets, **not** a Kubernetes scheduler +simulation or proof of runtime headroom. Natural return to two nodes must be +observed, not inferred from the configured minimum. Fresh inventory confirmed +staging had returned to two nodes after the preceding experiment; it does not +validate this candidate's scale-down behavior. The DB node has a tight request +budget. + +## Implementation guard + +`mcp-registry:coordinatedCapacity` defaults to false and is not enabled in any +stack file. The program rejects it outside staging or without all of: + +- `provider=gcp`, `nodeMachineType=e2-standard-4`, `burstAutoscaling=true`; +- `registryCPURequest=800m`, `postgresCPURequest=1500m`, `postgresCPULimit=3`. + +When enabled it registers the app HPA, gives it ownership of deployment replica +count, sets app pools to 30/5 and changes ingress's CPU target from 88% to 70%. +Use the prepared-listing candidate app image; current production 1.8.6 does not +contain these app changes. Do not merge/promote simply to test the opt-in. + +## Bounded validation sequence + +1. Reauthenticate and refresh all live staging state; back up the exact original + specs privately. Verify the footprint and render/preview the proposed changes. + Applying the DB request may restart the singleton DB; stage only while test + traffic is stopped and spare schedulable capacity is available. +2. Use the previously calibrated sixteen-vCPU external generator, identical + binary/catalog and request policy. Capture scheduler and request-phase timing; + verify no calibration/delivery regression before interpreting staging drops. +3. Short gate with four Ready apps, six Ready ingress replicas and healthy LB + backends: two minutes at 100 reads/s, three minutes at 4,500 reads/s, then two + minutes at 100 reads/s. Keep the earlier 4,000 keys, 80% hot mix, 36 updated + listings/s and three searches/s. Require zero drops, zero attempt failures, + page p95 ≤1 second, healthy DB, and drained recovery queues. +4. If the short gate fails, capture the limiting stage, stop, restore original + specs and delete the generator. Do not run the long test or tune multiple + candidates during the same comparison. +5. Only if it passes: enable the intended 2–4 / 2–6 HPA bounds, let staging settle + to two nodes and two replicas of each workload without forcing eviction, and + then run the same ten-minute burst with one publish/s followed by thirty + minutes of recovery. Record the rise from idle, pending pods, CPU/pressure, + LB readiness, all offered requests and publishes, and natural scale-down. + Require zero dropped work/failures, page p95 ≤1 second, publish p95 ≤2 seconds, + drained queues and two-node recovery. Inability to settle to two nodes is + itself a failed idle-footprint gate. + +Persist origin and ingress counters separately, per-replica metrics, cache +outcomes from the benchmark response headers, and direct cAdvisor pressure +samples throughout. Ingress metric series must be nonempty; missing telemetry +is a collection failure, not zero load. Cache-outcome headers and direct pressure +captures are test evidence; this candidate does not yet install permanent +cache-outcome or CPU-pressure time series in production. + +After the test, verify response equivalence, remove test publications and the +owned generator/disk, restore original app/ingress/DB settings and whitelist, +and record cleanup and actual node count. Do not claim a successful rollback +or two-node return from a successful patch command alone. + +## Local checks + +The Pulumi program builds, deployment-module tests pass, and repository +`make lint` passes. Mocked-resource tests check the HPA's target, scaling bounds, +CPU-only metric and recovery window; configuration tests reject production and +incomplete CPU budgets. A separate full deployment-module lint invocation also +reports existing module lint debt; new changes are checked separately against +the preceding commit. Kubernetes server dry-runs passed before the staging +changes. The live short test failed as reported above; a Pulumi apply was not +used for this temporary comparison. + +HPA behavior follows the [Kubernetes v2 API](https://kubernetes.io/docs/reference/kubernetes-api/autoscaling/horizontal-pod-autoscaler-v2/) +and [GKE CPU utilization semantics](https://docs.cloud.google.com/kubernetes-engine/docs/concepts/horizontalpodautoscaler). diff --git a/tools/ingressbench/DELIVERY_VALIDATION.md b/tools/ingressbench/DELIVERY_VALIDATION.md new file mode 100644 index 000000000..6d58cad46 --- /dev/null +++ b/tools/ingressbench/DELIVERY_VALIDATION.md @@ -0,0 +1,149 @@ +# Generator calibration and DB isolation — 2026-10-28 + +**The generator needed more headroom, and real app CPU contention remains. +Neither staging layout passed the short gate. Production was unchanged.** +The conditional ten-minute burst with publishes and thirty-minute recovery was +not run. No app autoscaler or permanent dedicated DB node was introduced. + +## Resolve the measurement uncertainty first + +The instrumented driver now records scheduler, worker-queue, connection, TLS, +first-byte and response-transfer timings by request kind and bounded cache +outcome. Scheduled-to-completion latency remains available, so client delays +are not silently excluded. CPU profiles are optional and remain local. + +An explicit calibration mode serves 64-KiB JSON responses over loopback HTTPS, +using the same request scheduler, workers and 30-second idle-connection rotation. +It cannot publish or collect a staging catalog. Calibration used 4,500 reads/s +for 90 seconds, 1,024 workers and the saved catalog. + +| Generator host | Scheduled requests | Dropped before HTTP | Attempt failures | Page p95 | +| --- | ---: | ---: | ---: | ---: | +| Four vCPUs, fixture and client together | 405,000 | 25,918 (6.40%) | 0 | 63.9 ms | +| Sixteen vCPUs, fixture and client together | 405,000 | 0 | 0 | 1.55 ms | + +The small host stalled at startup and around connection rotations even without +the registry. Its CPU profile includes both client and fixture work; TLS and +runtime contention are visible. The larger temporary generator passed with the +same connection policy. This establishes a local delivery limitation and a +passing local control, not an independent external-network capacity guarantee. +It does not prove that all losses in earlier staging tests came from the driver. +The instrumented binary was refactored into helpers for lint between calibrations; +request generation behavior was unchanged. Exact binary hashes are recorded. + +Both subsequent staging arms used the same sixteen-vCPU generator and identical +instrumented binary. The generator stayed outside the tested node pool and was +deleted afterward. Staging node types and counts were not increased for this +generator change. + +## Two bounded staging tests + +Both used the existing prepared-JSON app image from source `43e17e0`, four apps +at 400m CPU request each, no app CPU limit, pools of 30 maximum / 5 minimum each, +six ingress controllers and four nodes. PostgreSQL retained its three-CPU quota. +The total app pool budget remained 120 maximum / 20 minimum connections. + +Each test had two minutes at 100 reads/s, three minutes at 4,500 reads/s and two +minutes at 100 reads/s. The burst included 36 updated listings/s and three +searches/s within the total, HTTP/1.1, 4,000 keys and an 80% hot-key mix. Neither +short test published data. + +The first layout spread one app per node, sharing nodes with ingress and the DB. +The second left PostgreSQL on its existing node and excluded apps and ingress +from that node. Six ingress replicas were spread two per workload node; four +apps occupied those three nodes. Temporary affinity and three-domain spread +constraints held the layout at four nodes. With local ingress traffic policy, +the three workload-node load-balancer backends were healthy and the DB node was +intentionally excluded. PostgreSQL was not restarted or moved. + +| Measurement | Shared nodes | DB isolated, same four nodes | +| --- | ---: | ---: | +| Scheduled reads | 810,000 | 810,000 | +| Reads dropped before HTTP | 8,641 (1.067%) | 23,426 (2.892%) | +| Attempt failures | 9 timeouts | 0 | +| Page p95, scheduled to completed | 328 ms | 414 ms | +| Search p95, scheduled to completed | 3,369 ms | 1,635 ms | +| Updated-listing p95, scheduled to completed | 3,935 ms | 1,291 ms | +| Largest sampled pool queue on one app | 758 | 460 | +| Highest measured app CPU-stalled fraction | 39.1% | 30.7% | +| Dominant DB statement mean | 1.72 ms | 0.63 ms | +| Short recovery: drops / attempt failures | 0 / 0 | 0 / 0 | + +Each layout was tested once. Isolation also changes traffic distribution and +concentrates apps/ingress onto three nodes. These results describe that complete +placement change, not a pure measurement of PostgreSQL isolation alone. +Comparison against older four-vCPU-generator runs additionally involves changed +node/pod placements and must not assign all differences to generator CPU. + +## What the additional measurements establish + +On the larger generator, mean CPU busy was 18.4% / 21.7% across sixteen vCPUs; +mean received traffic was about 2.12 / 2.07 Gbit/s. No interface drops were +recorded. Scheduler timing was much tighter than in the earlier runs. This +still does not rule out every network or client limitation, but the server-side +queue and CPU-pressure evidence establishes a real remaining server problem. + +In the shared layout, the most affected app was on a node **without PostgreSQL**. +Its mean page pool-acquisition time was 1,632 ms and checkout time 128 ms, while +the healthiest peer averaged about 1 ms and 5 ms. The affected app spent 39% of +its bounded cAdvisor interval fully CPU-stalled. Node CPU samples reached 90% +on that node and 97% on the DB node; interval averages can hide shorter peaks. + +Client phases distinguish the slow path: shared-layout page cache hits averaged +about 59 ms of HTTP processing, while page misses averaged 788 ms, including +759 ms from request-written to first response byte. Updated-listing misses +averaged 575 ms to first byte. Connection acquisition and body transfer were +much smaller on average. First-byte time includes ingress, networking and +upstream queueing; it is not a pure application execution timer. Nine attempts +timed out: four pages and five updated listings. + +DB isolation made the dominant observed SQL statement faster, but two app +replicas still reached sampled queues of 357 and 460 and approximately 31% CPU +stalling. Faster SQL therefore did not remove the application-side bottleneck. +The statement comparison covers 82,727 / 86,114 calls in bounded top-25 snapshot +deltas that include preparation and collection padding. That statement recorded +no shared-block reads or temporary-block writes. Neither captured run increased +the DB quota-throttling counter, and neither restarted an app or the database. + +Both short recoveries completed all 12,000 reads without drops or failures, and +the final sampled app pool queues were zero. All 140 direct capture operations +per arm succeeded. These are short recovery observations, not full publishing, +autoscaling or idle-footprint validations. + +## Decision + +Keep the PR in draft. Four app replicas alone and dedicated DB placement alone +have not met the offered-load requirement. Do not increase the database pool +or promote either layout on these results. Do not run the conditional long test +after a failed short gate. + +The next design change should address CPU allocation and placement for apps +sharing busy nodes with ingress. Raising an app CPU request requires checking +actual scheduling budgets and the intended idle node count; it is not equivalent +to having a dedicated core. Additional replica counts or a DB-only node do not +by themselves establish that CPU will be available where requests land. The +specific resource/placement change still needs validation; this report does not +claim a proven production fix. + +## Validation and cleanup + +`go test -race ./tools/ingressbench` and `make lint` passed. Tests verify timing +separation, connection reuse, missing-event handling, fixture size, bounded +cache labels and rejection of publishing/catalog collection in calibration +mode. [CI at driver source `08077e4`](https://github.com/modelcontextprotocol/registry/actions/runs/40715012264) +passed, as did CodeQL. The app image itself was unchanged. + +The temporary generator VM and disk were removed. The original staging app +image/resources, two replicas, app and ingress affinity/spread constraints, +rollout strategies, ingress HPA specification and rate-limit annotation were +restored. All 48 sampled response hashes matched the pre-test responses. No test +publications were created. At the final 15:51 UTC snapshot, four nodes and six +ingress replicas remained; ingress HPA bounds were restored to 2–6 and node-pool +bounds remained 2–4. Natural return to two nodes has not been demonstrated. +Restoration verification is recorded alongside the portable results. + +Portable measurements are in +[evidence/delivery-validation-2026-10-28](evidence/delivery-validation-2026-10-28). +Raw client results, profiles and cluster samples remain in the local +`delivery-validation-2026-10-28` evidence directory. Private deployment snapshots +and raw profiles are not uploaded to the repository. diff --git a/tools/ingressbench/PROFILE_VALIDATION.md b/tools/ingressbench/PROFILE_VALIDATION.md new file mode 100644 index 000000000..e534e7d2f --- /dev/null +++ b/tools/ingressbench/PROFILE_VALIDATION.md @@ -0,0 +1,144 @@ +# Listing CPU diagnosis — 2026-10-28 + +The subsequent [fixed-budget two-versus-four app comparison](REPLICA_CAPACITY_VALIDATION.md) +tests the replica/placement hypothesis below. It improves origin latency but +does not pass the complete offered-load gate. + +**The tested candidates do not pass the promotion gate. Production was not +changed.** The short runs identify expensive listing JSON work and continued +application CPU contention. They do not establish a complete production fix. +No ten-minute burst/publish or thirty-minute recovery validation was run for +these candidates because the short workload still dropped offered reads. + +## Controlled workload and results + +Each arm started with four Ready nodes, six Ready ingress controllers and all +four load-balancer backends healthy. The two apps remained on the same original +two nodes, one sharing the registry database. Pools stayed at 60 maximum / 10 +minimum per replica; the DB quota stayed at three CPUs. Each arm used two minutes +at 100 reads/s, three minutes at 4,500 reads/s (including 36 updated listings/s and +three searches/s), and two minutes at 100 reads/s. These diagnostic runs contained +no publishes. The same four-vCPU generator, HTTP/1.1 driver, 4,000-key catalog and +80% hot-key mix were used. The catalog SHA-256 is +`120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be`. + +| Candidate | App CPU request | Offered reads | Generator-dropped reads | Attempt failures | Page p95 | +| --- | --- | ---: | ---: | ---: | ---: | +| Connection-release baseline + profiling | 250m | 810,000 | 123,015 (15.19%) | 95 | 1,236 ms | +| Bounded in-memory normalization cache | 250m | 810,000 | 257,289 (31.76%) | 9,517 | 1,059 ms | +| Prepared JSON stored with each record | 250m | 810,000 | 56,843 (7.02%) | 2 | 711 ms | +| Same prepared JSON image | 400m | 810,000 | 43,120 (5.32%) | 0 | 729 ms | + +Dropped reads never reached an HTTP attempt. Zero attempt errors therefore does +not mean the last arm handled all offered work. Latency percentiles describe +attempted requests and cannot hide the missing work. Each short recovery had +12,000 offered reads with no drops or attempt failures; that is not the full +recovery or natural two-node validation. + +The baseline source is commit `2a70d5d`, the rejected memory-cache prototype is +`9e27b14`, and the prepared JSON implementation is `6e77bb2`. The last two arms +used the identical image digest +`sha256:58671aecb4b2e66de0814567a241ca8b4a0fd7749678521bc0658f0b8240c5c2`. +The 400m request was a temporary staging patch, not a production configuration +change. Each arm is one observation, not a repeated capacity estimate. + +## What the evidence establishes + +CPU profiles identified typed JSON decoding and rebuilding the listing response +as substantial application work. In one baseline capture, JSON unmarshal used +33% of sampled CPU and the Huma response-writing subtree used about 30%; these +are different paths, not percentages to add to nested child functions. The new +checkout tracer showed little time between the final pgx query-end callback and +release entry. Query end includes row consumption: it is not the timestamp at +which PostgreSQL finished executing SQL. + +The small normalization cache was ineffective for this workload: origin traffic +largely consisted of records absent from its 8,192-entry cache. Misses added +normalization work. It was rejected and removed from the final source. + +Preparing normalized JSON during writes removed that read-time roundtrip without +relying on key reuse. Across both replicas, measured process CPU per origin +completion fell from approximately **4.69 ms to 3.10 ms (34%)** in the 250m +comparison. This is a counter-based interval estimate, not an isolated CPU cost +for a particular query. All 187,108 staging records were backfilled, all 48 direct +API comparisons matched, and the observed listing row counters reported no +fallbacks during the prepared bursts. The prepared payloads total 155,017,950 bytes +before PostgreSQL storage/compression overhead. Source JSON and official metadata +were not modified by the backfill. Ordinary VACUUM/ANALYZE finished before testing. + +The prepared 250m arm still reached 893 sampled pool waiters on the database node; +the other app had none. In the 400m arm the affected replica switched to the other +original node: it reached 900 waiters and spent approximately 39% of the captured +cAdvisor interval fully CPU-stalled. Its peer had no sampled waiters. Origin +completion counts were nearly equal (40,863 and 40,893), so work was reaching both +replicas. A busy app can queue despite a healthy peer and spare cluster capacity. + +Both app nodes also hosted busy ingress controllers. Their captured ingress CPU +averages in the 400m burst were approximately 1.74 cores across two controllers on +one node and 1.42 cores on the database node's controller. These exclude system +pods and kernel overhead and use slightly different sample intervals. CPU +contention is directly observed; the precise contribution of every neighboring +workload is not isolated by these runs. + +For the 400m run, the dominant statement present in both bounded top-25 database +snapshots averaged **1.54 ms over 79,884 calls**, with no shared-block reads or +temporary-block writes in that delta. The interval includes preparation and +collection padding, and the bounded snapshots do not represent every statement. +The captured DB throttling counter did not increase during either prepared run; +the database stayed Ready with no app/DB restarts. This supports application CPU +contention and pool congestion rather than a demonstrated need for more database +connections. It does not prove that every slow request has the same cause. + +The 400m generator had 58% mean / 67% peak aggregate CPU busy, approximately +2.03 Gbit/s mean received traffic, and no recorded interface drops. This argues +against aggregate generator CPU exhaustion; it does not exclude a single-thread, +scheduling or network limitation. The extra generator telemetry was collected +only for this arm. + +## Decision and remaining work + +Keep the PR in draft and do not promote these settings on the basis of this +result. Preparing JSON is a measured improvement, not the complete solution. +A larger connection pool and repeated full-length runs of this configuration +are not justified by the evidence. + +The next capacity decision should address how origin-serving app CPU is shared +with ingress, rather than another database index. A bounded app-replica/placement +control can establish whether distributing origin work onto spare CPU removes +queues. If that control passes, app autoscaling needs a fixed total database +connection budget, a signal that detects queued demand even when CPU-starved, +and validation of both the scale-up delay and return to two idle nodes. Those +changes have not been implemented or proven here. A remaining JSON reformatting +pass also appears in profiles, but removing it is not yet a demonstrated solution +to the node contention. + +The 400m request fits the observed original-node scheduling budget, while 500m +would exceed the database node's then-current allocatable CPU by about 6m. That +arithmetic does not prove natural two-node recovery or adequate runtime capacity. +The candidate was restored before a full recovery gate, so no such claim is made. + +## Validation and cleanup + +- `go test -race ./internal/... ./cmd/... ./pkg/... ./tools/prepare-listing-json` + and lint passed. [CI at the measured implementation](https://github.com/modelcontextprotocol/registry/actions/runs/40680509055) + passed. +- Tests cover typed/prepared API and OpenAPI equivalence, current metadata, + changed payloads, large publisher numbers, old-writer invalidation, unknown + codec fallback, conditional backfill rejection after a concurrent change, + legacy identity mismatch fallback, empty listings and cancellation. +- Migration 018 adds nullable derived payload/codec columns. New writes update + source and derived data atomically; an older writer changing source JSON + invalidates the derived bytes. A codec bump is required if wire normalization + changes. Backfill is an explicit, bounded maintenance tool, not startup work. +- The original staging app image, resource requests, affinity and HPA spec were + restored. Temporary loopback profiling, the generator IP's rate-limit + exemption, generator VM/disk and local test PostgreSQL were removed/stopped. +- The additive migration and derived payloads remain in staging; the original + app ignores them. The corpus remains 187,108 versions / 52,366 names, with no + benchmark publications from these diagnostic runs. All 48 response checks + also matched after restoration, and the DB postmaster start was unchanged. +- Portable aggregate evidence is in + [evidence/app-profile-2026-10-28](evidence/app-profile-2026-10-28). Raw profiles, + per-pod samples, client results and capture status remain in the local + `app-profile-diagnosis-2026-10-28` evidence directory. Raw profiles were not + uploaded to the repository. diff --git a/tools/ingressbench/README.md b/tools/ingressbench/README.md new file mode 100644 index 000000000..2f3f78988 --- /dev/null +++ b/tools/ingressbench/README.md @@ -0,0 +1,202 @@ +# Public-ingress staging benchmark + +The [production workload audit](WORKLOAD_PROFILE.md) found that the earlier +stress workload does not reproduce the observed production cycle closely enough +to serve as its acceptance test. Read that qualification before choosing rates +or interpreting capacity results. + +This driver targets only `https://staging.registry.modelcontextprotocol.io` and +requires `-staging`. Run it on a separate load-generator host, outside the tested +node pool. It exercises HTTPS, ingress caches, and the application together. HTTP/1.1 is +explicitly selected by default; `-http2` allows HTTP/2. Keep the protocol fixed +within a comparison and verify the response protocol counts in the output. +The existing `tools/readbench` remains the query-focused benchmark. + +Before using a generator for a capacity verdict, run `-staging -calibrate` with +the intended rate, workers and saved catalog. This uses only a loopback HTTPS +fixture with a 64-KiB JSON response and exercises the same scheduling, +connection rotation and body consumption. It cannot publish or collect a +staging catalog. The fixture shares generator CPU, so a failure demonstrates a +limit in this local client/server setup; a pass does not certify the external +network. `Calibration` in the result prevents confusing this with a registry +test. Optional `-cpu-profile=generator.pb.gz` writes a local Go CPU profile. + +The [fixed-budget replica comparison](REPLICA_CAPACITY_VALIDATION.md) records +the earlier short test and its load-generator limitations. It is not a passing +production promotion or autoscaling gate. + +The subsequent [generator calibration and DB-isolation control](DELIVERY_VALIDATION.md) +resolves part of that measurement uncertainty and confirms remaining app CPU +contention. Neither tested layout passed the short offered-load gate. + +The [coordinated CPU capacity candidate](COORDINATED_CAPACITY.md) records the +subsequent staging test: 7,418 of 810,000 offered reads were dropped, despite +lower attempted-request latency. Its short delivery gate failed, so the long +autoscaling test was not run and the opt-in remains disabled. + +`-page-workload measured-pages.json` replaces the legacy page-selection rule +with explicit API version, page size, version filter and anonymous key-frequency +classes. Each class requires enough **distinct** matching staging catalog keys; +the driver fails before load if the catalog is too small. It permutes ranks +across matching keys independently of catalog ordering, then repeats a +deterministically shuffled frequency schedule. This models aggregate frequency, +not chronological crawl sessions or exact production cursor positions. Search +and updated-listing generation remain separately configured by the existing +flags; they are not calibrated automatically by a page profile. + +The profile format is `{"classes":[{"api_version":"v0.1","limit":100, +"version_filter":"","key_counts":[7,2,1]}]}`. This example gives three distinct +staging URLs frequencies 7/2/1. Use measured counts for actual calibration. +Results record the profile SHA-256 and bounded `RequestShapes` counts to verify +the attempted API/limit/version/cursor mix. `HotPercent=-1` means the measured +profile replaced that legacy parameter. The old 4,000-key catalog is insufficient +for the measured broad-crawl profile; do not silently reduce its diversity. + +`-connection-rotation=0` disables synchronized idle-connection churn. The default +remains 30 seconds to reproduce older tests, and results record the setting. +Neither value is evidence of production connection lifetimes. Hold transport +policy fixed within a comparison and label any sensitivity test explicitly. + +`-read-trace trace.json` schedules an ordered read trace against the private +`-keys` catalog. The format is `{"duration_ms":1000,"requests":[{"at_ms":10, +"key":0,"kind":"page"}]}`. It preserves repeated keys and relative arrival +spacing, overriding generated read rates, page selection, search and update +fixtures. It accepts only supported relative public GET read paths, rejects +publishing/page-profile combinations, and bounds duration, count and per-second +arrivals. Results record the trace hash and use `-1` for overridden rate fields. +Keep catalogs containing production search parameters private; do not commit +them or raw access logs. Public result artifacts contain hashes and aggregate +counts. Log completion time minus request duration estimates arrival time; it +does not recover client TCP sessions. A trace also needs an explicitly matched +initial cache state and a corpus-coverage check before it can qualify a workload. + +For read-only traffic profiling, `collect_profile_logs.py --start +--seconds 60 --out profile.json` reads production Cloud Logging with the current +gcloud identity. Use `--project mcp-registry-staging` for staging comparisons. +It does not change either environment. It paginates up to 200,000 +records and marks incomplete coverage on errors or truncation; such slices do +not receive a rate estimate. `profile_logs.py` separates zero-request-length +background-work records from client records and saves only bounded aggregates +and anonymous frequency ranks. It does not save URLs, search text, IPs, user +agents or request IDs. Complete pagination does not independently prove logging +ingestion completeness. Run parser/coverage tests with +`python3 -m unittest discover -s tools/ingressbench -p 'test_profile_logs.py'`. + +```sh +go build -o ingressbench ./tools/ingressbench +./ingressbench -staging -catalog -keys keys.json +./ingressbench -staging -keys keys.json -rate 4500 -updated-rate 36 \ + -search-rate 3 -hot-percent 80 -workers 1024 -duration 10m -token-file publish-token \ + -out result.json +``` + +Catalog collection walks four real cursor streams (two API versions, limits 50 +and 100), up to 4,000 keys. It must be completed before timing. Preserve the same +catalog for comparisons; its SHA-256 is included in results. The hot fraction +uses 128 keys; remaining page requests traverse the full catalog. Search uses +five terms and varied valid limits. Incremental sync varies timestamps across +one hour, one day, and one week, producing uncached requests. Set its rate to +match observed origin demand. Rates are scheduled, not measured completions. + +An optional short-lived staging JWT must permit publishing only +`io.benchmark/autoscale-*`. Each publish has a unique name and remote URL. +Publishing is independently scheduled at one per second with 16 workers and a +bounded queue; slow writes do not reduce the offered write rate silently. +Keep the token in a private file, never a command-line argument or report. +Only the benchmark's own staging publications should be removed afterward. + +The default read concurrency and queue are each bounded at 512. Dropped work, +HTTP status zero (transport or body-read failure), HTTP failures, up to 100 timestamped failure samples (with an omitted count), cache response +headers, bytes and latency are recorded. Latencies run from intended start to +body completion, including scheduling/queue delay, and include failed requests. +`Scheduled` and `Dropped` count reads; `ScheduledPublishes` and +`DroppedPublishes` count publishes; `Completed` includes both. Elapsed time +includes draining outstanding work. Idle connections rotate every 30 seconds +so new ingress replicas can receive new connections. + +`Windows` adds UTC-aligned ten-second read/publish counters. Scheduled work is +attributed to its intended time, drops to the actual enqueue attempt, dispatches +to the start of request processing, and completions/failures to body completion. +Partial first/last windows and delayed completions can cross interval boundaries; +compare counts over the whole run as well as individual windows. Scheduler lag +measures intended time to enqueue attempt (including dropped attempts); queue +delay measures enqueue to request processing, including worker preparation; HTTP +duration measures request processing through body completion. Queue/HTTP timings +are grouped by dispatch window. Their fixed histograms report `P95UpperMS`, a +bucket upper bound, while `MeanMS` and `MaximumMS` use observed durations. Overall +`LatencyMS` retains its existing exact sample percentiles. No URLs, bodies, or +credentials are included in these windows. Use the same instrumented binary in +both arms of a comparison. Window `FailureClasses` counts every failure by its +bounded transport-error class or HTTP status, even after the first 100 individual +failure samples have been retained. This preserves transition timing for later +error classes without retaining client data. + +`RequestPhases` separates attempted requests by request kind and bounded cache +outcome. It records scheduler delay, worker queue delay, connection acquisition, +TLS handshake, request-written to first response byte, first byte to body +completion, and total HTTP processing. Connection acquisition includes dial, +TLS and transport waiting; these phases overlap and must not be added together. +First-byte time includes network and upstream queuing, not just server CPU. +Counts distinguish absent events from measured zero durations. TLS hooks can +include speculative connections that are not ultimately used by a request, so +their count can exceed the number of new connections actually obtained. No +addresses, URLs, headers or credentials are retained in these aggregates. + +Temporarily exempt only the generator IP from staging ingress rate limits when +measuring aggregate multi-client capacity. Save and restore the prior annotation; +never weaken production rate limits. The generator otherwise represents one +source address and is subject to the per-controller client limit. + +Capture generator CPU/network utilization alongside cluster metrics. A saturated +generator or load-balancer connection imbalance is not evidence of database +capacity. Preserve timestamps, query mix, cache outcomes, per-pod CPU, DB CPU +pressure/throttling, pool queues, readiness, ingress HPA decisions and node +counts. Observe cold-cache scaling and recovery as well as the warm steady state. +A run that cannot reproduce the incident is calibration, not proof of a fix. + +### Targeted application diagnosis + +For a short staging diagnosis, set `MCP_REGISTRY_ENABLE_PPROF=true` on the +experimental application deployment. This opt-in listener binds to +`127.0.0.1:6060`; access it with a pod-specific `kubectl port-forward`, without +adding a Service or Ingress port. CPU captures use +`/debug/pprof/profile?seconds=30` (allowed range: 1–60 seconds). Heap and goroutine +profiles are available at `/debug/pprof/heap` and `/debug/pprof/goroutine`. +Remove the setting when restoring the deployment. Profile files remain local +investigation artifacts; do not publish them without reviewing their contents. + +The pool checkout histogram measures successful acquisition through entry into +`Release`, excluding acquisition wait and release bookkeeping. The post-query +histogram measures the interval from the final pgx query-end callback to that +release callback. Query end includes row consumption/close, so this is **not** +server-side SQL completion time. Checkouts with no traced query contribute only +to checkout duration. Both histograms use bounded operation labels and contain +no SQL, arguments, connection identifiers or credentials. + +The listing HTTP path uses versioned, normalized JSON prepared when a server is +created or updated. PostgreSQL still selects every result and supplies fresh +registry metadata. Migration 018 adds nullable derived columns; an older binary's +payload update invalidates them through a trigger. Missing or unknown codec +versions fall back to the ordinary typed normalization. Bump `listingCodecVersion` +whenever ServerJSON wire normalization changes. The HTTP router enables this +serialization-only path with schema-link response transforms disabled; other +service/database consumers retain typed records. The listing JSON row counter +labels the source as `prepared` or `fallback`. + +Existing rows can be prepared using `go run ./tools/prepare-listing-json` with +`DATABASE_URL` supplied securely through the environment, after migration 018. +The tool defaults to 500-row transactions with a 100 ms pause, a five-second +statement timeout and a one-second lock timeout. It uses keyset batches, copies +rows before normalizing, and conditionally updates only unchanged JSONB. It never +changes source JSON or official timestamps. A concurrent update can leave a row +for another pass; rerunning is safe. Identity mismatches remain on the fallback +path and are counted. Check prepared/fallback coverage before interpreting a +benchmark. No automatic full-table backfill runs during application startup. + +For a controlled app-replica comparison, `MCP_REGISTRY_DATABASE_MAX_CONNECTIONS` +and `MCP_REGISTRY_DATABASE_MIN_CONNECTIONS` set each app's pool budget (defaults +60/10). Invalid limits fail startup before connection retries. Two apps at 60/10 +and four apps at 30/5 both permit 120 maximum and 20 minimum connections in total. +Lower per-app limits and finish that rollout before increasing replicas. Include +surge pods, monitoring and administration in the PostgreSQL connection budget; +the settings do not enforce a deployment-wide limit automatically. diff --git a/tools/ingressbench/REPLICA_CAPACITY_VALIDATION.md b/tools/ingressbench/REPLICA_CAPACITY_VALIDATION.md new file mode 100644 index 000000000..32075a855 --- /dev/null +++ b/tools/ingressbench/REPLICA_CAPACITY_VALIDATION.md @@ -0,0 +1,137 @@ +# Fixed-budget app replica comparison — 2026-10-28 + +Follow-up [generator calibration and DB isolation](DELIVERY_VALIDATION.md) +establish local generator limitations and continued app CPU contention with a +larger generator. The older results below remain bounded observations. + +**Four apps substantially improved origin latency and sampled pool queues, but +the short offered-load gate did not pass. Do not promote or treat this as an +autoscaling validation.** Production was unchanged. The conditional ten-minute +burst with publishes and thirty-minute recovery was not run. + +## Controlled comparison + +Both arms used commit `43e17e0993e1afed08950da75b461a1e4966fab3`, prepared listing +JSON, the same image digest +`sha256:60fb8b7d7931a257542848460f7d1c76a99f16c46ddad795249fb5c25b1c9b0e`, +400m CPU request per app, no app CPU limit, four Ready nodes and the same six +ingress controller UIDs and placements. The DB quota remained three CPUs. +The control placed two apps on the original two nodes; the candidate placed one +app on each of the four nodes. Total app CPU requests therefore rose from 800m +to 1,600m: this tests the proposed replica/placement package, not replica count +in isolation. + +The new pool configuration kept the aggregate budget fixed: two apps at 60/10 +maximum/minimum connections versus four at 30/5, both totaling 120/20. Live +per-app pool maximum gauges were checked before each arm. Scaling transitions +were sequenced to avoid exceeding that budget; a future deployment must also +budget for rolling-update surge replicas. + +Each arm used two minutes at 100 reads/s, three minutes at 4,500 reads/s and two +minutes at 100 reads/s. The burst included 36 updated listings/s and three +searches/s within the total, 4,000 cache keys, an 80% hot-key mix, HTTP/1.1, +1,024 workers and a 1,024-job queue. There were no publishes. Generator VM, +binary, catalog and connection rotation policy were unchanged. Hashes and the +gate recorded before the bursts are in [experiment.json](evidence/replica-capacity-2026-10-28/experiment.json). + +| Measurement | Two apps, 60/10 each | Four apps, 30/5 each | +| --- | ---: | ---: | +| Scheduled reads | 810,000 | 810,000 | +| Reads dropped before an HTTP attempt | 50,653 (6.253%) | 48,899 (6.037%) | +| Attempt failures | 0 | 0 | +| Page p95, scheduled arrival to body completion | 688 ms | 621 ms | +| Search p95, scheduled arrival to body completion | 2,702 ms | 804 ms | +| Updated-listing p95, scheduled arrival to body completion | 3,360 ms | 829 ms | +| Largest sampled pool queue on one app | 905 | 3 | +| Ingress upstream-response p95 bucket | >2,500–5,000 ms | >100–250 ms | +| Ingress total-request p99 bucket | >1,000–2,500 ms | >250–500 ms | +| Short recovery: dropped reads / attempt failures | 0 / 0 | 0 / 0 | + +The client percentiles include generator scheduler and worker-queue delay. +Ingress histograms independently support an origin-serving improvement; they +cover the bulk-read ingress and use a scrape-based 180-second counter increase +ending at burst completion. They are bucket bounds, not exact percentiles or +matched per-request traces. Upstream timing counts origin responses; total +request timing includes cached responses. Pool gauges were sampled every 15 +seconds and can miss shorter queues. Each arm is a single observation. + +## What improved, and what remains unresolved + +Origin completions were balanced across replicas: 41,547/41,562 in the two-app +counter interval and 20,389–20,395 per app in the four-app interval. The four-app +arm observed 5.81 million prepared listing rows and no fallback rows. Keeping +120 total maximum connections while reducing sampled queues strongly argues +against increasing the DB pool as the next intervention. + +CPU contention was reduced, not eliminated. The two apps spent about 28% of +their bounded cAdvisor intervals fully CPU-stalled. The four apps ranged from +3.3% to 22.8%; the highest remained on the database node. Its mean page-pool +acquisition time was still 80 ms, versus roughly 1–5 ms on its peers, even though +its periodic pending gauge samples were zero. Do not read sparse gauge samples +as proof of no waits. + +The dominant statement in the bounded database deltas averaged 1.59 ms over +81,657 calls in the control and 1.82 ms over 80,475 calls with four apps. These +snapshot intervals include preparation and collection padding and only cover +statements present in both top-25 snapshots. No shared-block reads or temporary +block writes appeared in that statement's delta. DB quota-throttling counters +did not increase within either captured run, and app/DB restarts stayed at zero. +These observations do not demonstrate a database-capacity problem. + +The residual dropped work cannot yet be assigned entirely to the server or +entirely to the generator. Both drivers received about 2 Gbit/s; aggregate CPU +busy averaged 61.5% and 65.0%, with no recorded interface drops. However, maximum +scheduler lag was 828 ms and 490 ms. The scheduler catches up to absolute +deadlines after a pause and drops jobs when its bounded queue is full. Slow +server responses can also keep those workers busy. Aggregate CPU headroom and +interface counters do not exclude single-thread, scheduling or network limits. + +In the four-app run, seven complete ten-second windows scheduled 45,000 reads +with no drops, while other windows lost work. The existing measurements lack +client connection/DNS/TLS/first-byte timing and latency broken down by cache +outcome, so they cannot isolate why the remaining intermittent stalls occur. +There is no evidence here for a claimed fixed 2-Gbit/s network cap. + +## Decision + +The fixed-budget replica change is a measured mitigation for origin contention, +not a proven complete solution at the intended arrival rate. Stop increasing +replicas or pool sizes based on these short runs. Keep the PR in draft. + +Before another capacity verdict, validate that the load generator can deliver +the intended arrival pattern independently of the registry, and separate +scheduler/queue, connection establishment, first-byte and body-transfer timing +by cache outcome. A split-generator control would help distinguish one client's +limit from aggregate server capacity, but must preserve the total request mix +and key distribution. These are next diagnostic requirements, not completed +tests or permission to generate production load. + +Only after a credible short offered-load pass should the full burst/publish and +recovery gate run. Dynamic app scaling, scale-up delay and return to two nodes +remain separate unproven properties; no app HPA was implemented here. + +## Validation and restoration + +- Race tests (`go test -race ./internal/... ./cmd/... ./pkg/...`) and lint passed. + [CI for the measured source](https://github.com/modelcontextprotocol/registry/actions/runs/40704622588) + passed. Pool tests exercise defaults, explicit 30/5 limits, invalid settings + and the deadline of a 31st acquisition with 30 connections held. +- All 70 two-app and 140 four-app direct capture operations succeeded; bounded + runtime commands succeeded and database endpoint samples remained Ready. +- The original app image, 250m requests, two replicas, affinity, ingress HPA + specification and rate-limit annotation were restored. Profiling was removed. + The generator VM and disk were deleted and local test PostgreSQL stopped. +- All 48 API comparisons matched after restoration. The database retained + 187,108 versions / 52,366 names, with the same postmaster start time. No test + publications were created. Additive prepared-JSON data from the previous + experiment remains in staging; the original app ignores it. +- At 14:50 UTC, ingress had decreased from six to four replicas and the cluster + autoscaler identified a removal candidate; four nodes still awaited normal + cooldown. Node-pool bounds remained 2–4. Restoration is not proof of the + candidate's natural two-node recovery. + +Portable aggregates are in +[evidence/replica-capacity-2026-10-28](evidence/replica-capacity-2026-10-28). +Raw client results, direct metrics, CPU profiles, cAdvisor samples and historical +queries remain in the local `replica-capacity-validation-2026-10-28` directory. +Raw profiles and private deployment snapshots are not included in the PR. diff --git a/tools/ingressbench/VALIDATION.md b/tools/ingressbench/VALIDATION.md new file mode 100644 index 000000000..a7f3019e7 --- /dev/null +++ b/tools/ingressbench/VALIDATION.md @@ -0,0 +1,95 @@ +# Staging autoscaling experiment — 2026-10-27 + +Follow-up: [connection lifetime and app CPU comparison](CONNECTION_CPU_VALIDATION.md) +records the next two burst/recovery tests, including their remaining capacity +and idle-node limitations. + +**Neither autoscaling candidate passed. Do not promote these settings as a +production performance fix.** Database CPU protection helped, but client-visible +publish timeouts and application pool queues remained. + +All three runs used the same application image, benchmark binary and saved +cursor catalog, starting from two Ready `e2-standard-4` nodes and two ingress +replicas. Each offered ten minutes of 4,500 reads/s, including 36 uncached +incremental-sync reads/s and three varied searches/s, plus one independent +publish/s. The read concurrency and queue were each bounded at 1,024. Actual +traffic was lower when that queue filled; dropped work is reported separately +from attempted-request failures. All runs explicitly used HTTP/1.1. + +| Measurement | Fixed capacity | Ingress/node autoscaling | Autoscaling + 60s client connections | +| --- | ---: | ---: | ---: | +| Burst start, UTC | 21:07:43 | 21:43:20 | 22:25:39 | +| Offered reads | 2,700,000 | 2,700,000 | 2,700,000 | +| Dropped reads at generator | 666,282 (24.68%) | 631,279 (23.38%) | 976,541 (36.17%) | +| Read transport/body failures | 13 | 625 | 435 | +| Acknowledged publishes | 600/600 | 593/600 | 596/600 | +| Publish client timeouts | 0 | 7 | 4 | +| Publish p95, client measurement | 2.06 s | 8.98 s | 8.44 s | +| Page p95 / p99, client measurement | 0.93 / 1.44 s | 0.82 / 6.65 s | 1.34 / 7.49 s | +| Peak sampled pool waiters, total | 294 | 1,259 | 915 | +| Peak sampled DB CPU pressure, some avg10 | 85.55% | 30.05% | 19.88% | +| Largest observed node / ingress count | 2 / 2 | 4 / 6 | 3 / 5 | + +Client latency runs from intended dispatch to body completion, including queue +delay and failures. The server histogram p95 is a different measurement. HTTP +status 200 alone does not count a truncated response as successful. In the first +autoscaling run, 597 publish records persisted despite only 593 acknowledgements: +four timed-out requests committed. They remain ambiguous client outcomes. + +The fixed-capacity run reproduced node CPU saturation, database CPU contention, +database readiness failures and application queues. Both candidates increased +the database CPU request from 200m to 2 CPUs, retaining the 3-CPU quota, 4-GiB +memory limit, 512-MB shared buffers and 200-connection ceiling. App replicas +remained at two, with each pool retaining max 60/min 10. Neither candidate's +sampled database endpoints became unready during its burst. + +The second candidate exposed a scaling blind spot: with five ingress pods the +HPA measured 648m average CPU against its 704m target and held at five, while +the database node was at 101% of allocatable CPU. Another sample showed the two +apps receiving approximately 155 and 151 requests/s, but with zero and 875 pool +waiters respectively. The queued app shared the database node. Equal request +counts therefore did not imply equal available processing capacity. Query mix +and per-request cost still need consideration when comparing replicas. + +Database statement deltas during the first candidate showed approximately +3.9 ms mean server execution for the dominant page query. Sampled slow-request +diagnostics attributed 86–90% of elapsed time to acquiring a pool connection. +These are different populations and timing boundaries: application query timing +also includes consuming rows. This supports investigating app CPU contention and +connection hold time; it does not prove that a larger pool would solve the issue. + +The first candidate scaled naturally back to two nodes at 22:25:27 UTC, about +32 minutes after its burst ended. Database pods stayed on their existing nodes. +HPA stabilization, gradual pod removal and node draining all contributed to the +delay. Minimum node count is not an immediate post-burst cost guarantee. + +## Reproduction and limits + +Use the command and capture instructions in this directory's README. The fixed +baseline had earlier cache-warming calibration; the candidates started after +rollout/idle and added cold caches while scaling. Staging used the same main +catalog throughout, with additional small benchmark publications between runs; +it was not restored to a byte-identical database snapshot each time. Fifty +fresh-connection version probes were added during the first candidate to verify +new-backend reachability. The second candidate did not include those probes. + +The generator remained outside the tested node pool. Its sampled utilization in +the fixed run was about 47% CPU on average, peaking near 53%; the bounded queue +still limited delivered traffic as responses slowed. These tests reproduce a +failure mode; they are not a replay of every production client or a guarantee at +4,500 delivered requests/s. Earlier HTTP/2 calibration had transport failures and +is not used as this comparison's baseline. A separate 100-request/s HTTP/2 smoke +test with the 60-second setting completed 9,000 requests without errors; that +does not validate high-load HTTP/2 behavior. + +The comparison binary SHA-256 was +`28ec057a098b8d01a5c8d7fb7fed6cd2d3d89dc2f35c201dff1173c2e6a00150`. +The catalog SHA-256 was +`120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be`. +Timestamped raw measurements are retained with the investigation artifacts. + +The next candidate should address application CPU availability and bounded read +concurrency while retaining a database connection budget across the maximum app +replica count. It must repeat the same burst and protect publishing during +scale-up. Raising a pool limit or merely lowering the ingress CPU target is not +established as a solution by this evidence. diff --git a/tools/ingressbench/WARM_CAPACITY_VALIDATION.md b/tools/ingressbench/WARM_CAPACITY_VALIDATION.md new file mode 100644 index 000000000..5ff7e9ffd --- /dev/null +++ b/tools/ingressbench/WARM_CAPACITY_VALIDATION.md @@ -0,0 +1,202 @@ +# Ready capacity versus autoscaling — 2026-10-28 + +**The current candidate is not a complete burst-capacity fix.** Both arms failed +with the same application image, CPU reservation, connection pools and offered +traffic. Having four nodes and six ingress replicas ready before the burst did +not remove sustained queueing. Both arms returned naturally to two nodes, but +the autoscaling arm also had one connection refusal during recovery. + +## Controlled comparison + +Both arms used two connection-release application replicas with a 250m CPU +request, no CPU limit, and max60/min10 connections per replica. The application +pod UIDs and placement on the original two nodes were identical. PostgreSQL +retained its 2-CPU request, 3-CPU quota, 4-GiB memory budget, 512-MB shared buffers +and 200-connection ceiling. Neither database was moved or restarted. + +The warm arm began with four Ready nodes, six Ready ingress replicas and four +healthy load-balancer backends. The autoscaling arm began with two of each and +allowed the existing ingress HPA (2–6) and node autoscaler (2–4) to respond. Both +used an 800m ingress CPU request and the same HPA policies. The warm arm's minimum +was returned to two immediately after its burst. No manual node deletion was +used to demonstrate idle consolidation. + +Each arm had five minutes of 100 read requests/s preparation with no publishes, +then ten minutes of 4,500 reads/s including 36 uncached incremental-sync reads/s +and three varied searches/s, plus one publish/s. Recovery offered 100 reads/s +and one publish/s for 30 minutes. The burst used 1,024 read workers and a +1,024-entry queue; preparation/recovery used 128 each. HTTP/1.1, the 4,000-key +catalog, 80% selection from 128 hot keys, cache settings and transport behavior +were unchanged. Temporary rate-limit exemption covered only the generator IP. + +The same four-vCPU generator VM and instrumented binary were used for all six +phases. Forty-eight direct-origin response hashes matched before both arms. +The first arm's 2,400 test publications were removed before the second, restoring +187,108 versions and 52,366 names. This comparison deliberately retained 250m: +the preceding 1-CPU candidate could not consolidate onto two nodes. It does not +isolate CPU priority from placement in that earlier experiment. + +## Burst results + +| Measurement | Ready capacity | Autoscaling from two nodes | +| --- | ---: | ---: | +| Burst start, UTC | 08:38:24 | 09:28:36 | +| Offered reads | 2,700,000 | 2,700,000 | +| Generator-dropped reads | 362,437 (13.42%) | 497,703 (18.43%) | +| Attempted read failures | 223 | 639 | +| Acknowledged publishes | 600/600 | 591/600 | +| Publish p95 / p99 | 6.30 / 6.83 s | 7.98 / 10.00 s | +| Page p95 / p99 | 650 / 5,565 ms | 660 / 6,102 ms | +| Read scheduler lag, mean | 0.39 ms | 0.16 ms | +| Read worker queue, mean | 224 ms | 243 ms | +| Read HTTP duration, mean | 261 ms | 278 ms | +| Generator CPU, mean / peak | 51% / 59% | 52% / 66% | +| Generator interface drops | 0 | 0 | +| Peak sampled pool waiters | 927 | 895 | + +Dropped reads never reached HTTP dispatch and are separate from attempted +request failures. Client percentiles include intended-schedule-to-body-completion +time and failed attempts. Window timing counts reconcile with total scheduled, +dropped, dispatched and completed work. Aggregate generator CPU alone cannot +exclude every single-thread or network constraint, but sub-millisecond mean +scheduler lag and large downstream waits do not support a scheduling-capacity +explanation for the sustained losses. + +Both arms fail the experimental gate: no failed or dropped work, page p95 at +most one second and publish p95 at most two seconds, followed by clean recovery, +natural two-node idle and a passing repeat. A passing repeat was not attempted +because neither candidate passed the burst gate. + +## What this establishes + +The warm arm dropped reads throughout the burst, including its final minute. +The autoscaling arm reached four healthy load-balancer backends and six Ready +ingress endpoints at 09:31:43 UTC, approximately 3.1 minutes after burst start. +It continued dropping reads after that point. This documents a capacity-readiness delay in the autoscaling arm, but it cannot +by itself explain the failure of the warm arm. + +The application sharing the registry database node spent about 46% of the +captured warm-burst interval fully CPU-stalled, versus 13% for the other app. +In the autoscaling burst the corresponding values were about 45% and 19%. +These are cgroup CPU-pressure counter deltas over roughly 9.4–9.8 minutes, +not whole-run averages or percentages of the app's configured CPU request. +The database node was near full CPU while other nodes had spare capacity. + +Origin arrivals were approximately 200 requests/s per app during the middle +and late warm burst. Equal request distribution therefore did not provide equal +processing capacity. The warm arm's peak sampled pool queue was 927, concentrated +on the app sharing the database node. Slow diagnostics showed multi-second pool +acquisition waits; a concurrent database snapshot had 118 idle connections and +three active sessions, including the diagnostic query. + +The dominant page statement averaged 3.43 ms and 3.36 ms in the respective +bounded PostgreSQL snapshots bracketing preparation/burst and early recovery. +Those are SQL execution means, not end-to-end request latency or worst-case +query times. The sampled statement sets had no block reads or temporary-block +writes, and the statistics-reset and postmaster-start values were unchanged. +No database quota-throttling increment was observed during either burst. +Together, these measurements support application CPU contention with severe +pool congestion as a sustained bottleneck. They are consistent with slow +application-side connection turnover, but do not directly measure the interval +from SQL completion to connection return. They do not establish that every +individual failure has the same cause. + +## Connection resets remain a separate uncertainty + +The warm arm had timeouts but no connection resets. The autoscaling arm recorded +304 read connection resets and one publish connection reset, in addition to +335 read timeouts and eight publish timeouts. Its largest failure windows were +near new load-balancer backends becoming healthy. The original ingress pods had +no observed readiness loss or restart during the burst. This is correlation, +not proof that backend admission caused the resets; no packet capture establishes +their origin in this comparison. + +The first 100 retained individual failures were early timeouts, so the bounded +sample omitted individual reset timestamps. Complete per-window failure totals +remain available. A subsequent benchmark-only change adds complete per-window +error-class counts for future tests. It was not substituted into either arm. + +The autoscaling burst left 595 published records despite only 591 successful +acknowledgments. Thus at least four unsuccessful client attempts committed; +a failed client response must not be counted automatically as a failed database +write. + +## Recovery and cleanup + +| Measurement | Ready capacity | Autoscaling from two nodes | +| --- | ---: | ---: | +| Recovery window, UTC | 08:48:31–09:18:31 | 09:38:42–10:08:42 | +| Offered reads / publishes | 180,000 / 1,800 | 180,000 / 1,800 | +| Generator-dropped work | 0 | 0 | +| Read failures / publish failures | 0 / 0 | 1 / 0 | +| Recovery page / publish p95 | 8.06 / 12.98 ms | 8.14 / 13.95 ms | +| Two-node idle confirmed, UTC | 09:22:18 | 10:10:54 | + +Queues drained in both arms. The autoscaling recovery had one connection refusal +at 09:52:59 UTC, near an ingress endpoint becoming terminating while its node +still appeared healthy to the load balancer. This is another scaling-transition +failure to investigate; the sampled observations do not identify the rejecting +backend. Thus the autoscaling recovery does not pass the zero-error gate. + +Both arms consolidated naturally to two Ready nodes with the same application +pods and placement, before restoring the pre-experiment deployment. The last +spare node disappeared after each 30-minute client recovery finished. Continued +idle observations establish consolidation, but are not additional measured +recovery traffic. This result applies to the 250m candidate, not the preceding +1-CPU configuration that remained on three nodes. + +No application or registry-database restart, registry-database endpoint loss, +or snapshot command failure was observed in either arm. The load-balancer +health collector had one failed query after the final autoscale node was deleted; +that sample is missing telemetry, not a successful health check. Database CPU +throttling increased by 0.215 seconds over the warm observation and 0.340 seconds +over the autoscaling observation, outside their bursts. Empty alert/5xx metric +series are not evidence that every request succeeded; the client recorded +transport failures. + +Cleanup is complete. The original staging app image and 250m CPU request, HPA +specification and rate-limit annotation were restored. The generator VM and its +boot disk were deleted. Scoped cleanup removed 2,400 warm-arm publications and +2,395 autoscaling-arm publications; the corpus returned to 187,108 versions and +52,366 names with no benchmark records. All 48 direct-origin response hashes +still matched. At 10:15:42 UTC the restored deployment had two healthy app replicas +on two Ready nodes. The registry database postmaster start remained unchanged. +Production was not modified. + +## Interpretation and next test + +Do not promote ingress/node autoscaling alone as the solution, and do not treat +larger connection pools as a demonstrated remedy. The next application-capacity +experiment should address CPU contention and work placement while retaining a +bounded aggregate connection budget and proving the two-node idle state. A +smaller CPU reservation increase that fits the baseline is a useful isolated +control; application scaling/placement should be tested separately if it cannot +meet the gate. The reset mechanism also needs complete error-class timing and +packet-level evidence during scale transitions before claiming a clean rollout. + +This is one controlled pair, not proof of an exact production traffic mix or a +10× capacity margin. Ready capacity changes cache readiness and connection +distribution as well as node startup timing. `updated_since` values advance with +wall clock in both arms. Pool gauges are scraped every 60 seconds and direct +snapshots every 30 seconds; brief events can be missed. Ingress counters can +include traffic beyond the generator, so client counters define offered and +completed benchmark work. + +## Reproduction and validation + +Application source: `3ea1fa2e974d97bff88cd0f04dc95fac5dea5f68`. +Image: `sha256:94f184cd38345b059f440756d5eaa22448dbc7d827490220d144bd9356713f94`. +Measured driver source: `4eb0c10810904e47696ea853b09b1348e68f38b5`. +Measured binary: `96693b313ae702e76943ef2242e3de5c471ff3d0f0011b6999a51b88dfeee1a2`. +Catalog: `120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be`. + +Benchmark race tests, lint and GitHub CI passed at `4eb0c10`. The subsequent +per-window error-class addition passed those checks at `b14b0ec`. No production +configuration or deployment was changed. See `README.md` for driver invocation +and measurement semantics, and `CONNECTION_CPU_VALIDATION.md` for the preceding +CPU/connection-lifetime comparison. + +The portable measurements are in +[`results/2026-10-28-warm-capacity.json`](results/2026-10-28-warm-capacity.json). +They retain phase totals, timing conservation, generator/pressure summaries, +database deltas, integrity checks and cleanup verification. diff --git a/tools/ingressbench/WORKLOAD_COMPARISON.md b/tools/ingressbench/WORKLOAD_COMPARISON.md new file mode 100644 index 000000000..d6b9fd1fe --- /dev/null +++ b/tools/ingressbench/WORKLOAD_COMPARISON.md @@ -0,0 +1,147 @@ +# Production read-trace comparison — 2026-10-29 + +The prepared-listing app reduced measured CPU use and startup delivery losses. +The coordinated infrastructure candidate scaled from two to four nodes, but +neither candidate passed the short delivery gate. Production was not changed. +This is a four-minute broad-crawl comparison, not a completed production-cycle +or publish validation. A separate thirty-minute low-load recovery completed. + +[Aggregate comparison](evidence/workload-comparison-2026-10-29/broad-comparison.json), +[read-mix qualification](evidence/workload-comparison-2026-10-29/baseline-trace-assessment.json), +and [source coverage](evidence/workload-comparison-2026-10-29/read-trace-summary.json) +are checked in. Request catalogs, search text and raw access logs are excluded. + +## Controlled comparison + +The same ordered trace offered 274,116 public GET reads over four minutes, +including 270,766 page reads, 2,235 detail reads, 974 searches and 141 updated +listings. It preserves relative arrivals and reuse of 34,542 raw cache keys +from the production interval 18:27–18:31 UTC on October 28. Catalog and trace +hashes match across all arms; private request paths and search parameters are +not published. Forced generator connection rotation was disabled throughout. + +| Setting | Production-aligned baseline | App only | App + coordinated infra | +| --- | --- | --- | --- | +| App image | v1.8.6 (production digest verified) | Prepared-listing candidate | Same candidate | +| App replicas / CPU request each | 2 / 250m | 2 / 250m | HPA 2–4 / 800m | +| Ingress replicas / CPU request each | 2 / 300m | 2 / 300m | HPA 2–6 / 800m | +| Ingress memory request each | 90Mi | 90Mi | 512Mi | +| PostgreSQL CPU request / limit | 200m / 3 CPUs | 200m / 3 CPUs | 1500m / 3 CPUs | +| App pool max/min each | 60/10 | 60/10 | 30/5 | +| Nodes | 2 | 2 | Autoscaling 2–4 | + +The infrastructure arm also restores the candidate's 60-second ingress +connection lifetime. Its changes are a package; this experiment cannot assign +an individual benefit to CPU reservations, pool sizes or HPA settings. + +All arms start with two Ready app and ingress replicas and empty ingress +caches after a normal rollout. No old terminating ingress pods remained in +the saved start snapshots. PostgreSQL was warm for baseline/app; changing its +CPU request restarted it before the infra arm, followed by two minutes of +100-read/s conditioning. That does not establish identical DB cache state. + +## Delivery results + +| Measurement | Baseline | App only | App + infra | +| --- | ---: | ---: | ---: | +| Offered | 274,116 | 274,116 | 274,116 | +| Attempted | 268,352 | 271,813 | 272,191 | +| Dropped before dispatch | 5,764 (2.10%) | 2,303 (0.84%) | 1,925 (0.70%) | +| Transport failures | 48 | 0 | 7 | +| Page p95, attempted requests | 3,322 ms | 1,597 ms | 1,667 ms | +| Last window containing drops ends by | 78.1 s | 43.9 s | 83.1 s | + +The generator used 1,024 read workers and a bounded 1,024-job queue. +Drops are generator-side queue rejections before dispatch, not HTTP server +errors; they show unmet offered load under this concurrency policy. Attempted +latency excludes that work. +The final infra minute had no drops and ten-second-window mean HTTP durations +of roughly 7–24 ms. This supports a transition problem rather than proving +that sustained demand exceeds the scaled cluster's capacity. Empty caches and +an abrupt replay start make this harsher than entering the middle of a warm +production cycle; these results do not isolate cold cache as the sole cause. + +The baseline and app arms each returned 28 detail 404s: those objects were +absent in the older staging corpus. Infra returned 22 detail 404s and 42 detail +429s. Only the bulk-read ingress exempted the generator from rate limits, so +those 429s are a single-source test artifact, reported separately. Infra also +had six page body connection resets and one body timeout. Their cause is not +established by the aggregate result. + +App-only sampled CPU use fell from about 2.13 to 1.47 cores across both apps +(approximately 31%), despite attempting more requests. CPU full-stall rates +also fell, from 17–30% to 10–18% across the apps. These are sampled intervals +within one run, not repeated-run confidence intervals. The maximum sampled +page pool-wait gauge fell from 397 to 189, and rolling page acquisition p95 +from 4.72 s to 1.21 s. Histograms use rolling windows and app scrapes are 60 +seconds apart, so these are not exact phase boundaries. Forty-eight saved public +response fixtures matched their reference hashes. No publish traffic was run. + +## What matches production, and what remains unproven + +The initial frequency-based approximation failed cache/origin-mix checks. +Using the ordered trace corrected that mismatch: all twelve selected final- +minute checks passed the declared tolerances. Cache outcome shares were within +five percentage points; wire response sizes, key diversity and time-aligned +origin page/search/update rates were within 20%. This qualifies that minute's +read mix. It does not retroactively qualify the startup cache state or the full +rise, peak and recovery of the production cycle. + +Only successful source reads were replayed. The source contains about 1.8% +HTTP/2 reads; the generator used HTTP/1.1. Client TCP sessions, retries and +connection lifetimes cannot be recovered from these logs. Arrival times are +estimated from completion timestamps minus request durations. A separate +1,227,778-read, ten-minute peak trace was collected but was not executed after +the short gate failed. Write contention and the longer peak remain untested. + +The next acceptance run should start with the preceding observed traffic to +establish cache state, then continue into the selected peak, with corpus and +rate-limit mismatches resolved before comparing configurations. Keep the +empty-cache case as a separate resilience test. Repeating an abrupt cold start +and calling it the full production incident would not improve confidence. +Do not increase the connection pool or promote the infrastructure package on +these results alone. + +## Recovery and restoration + +The infrastructure arm completed **180,000/180,000 recovery reads at 100/s** +without drops, transport failures or non-200 responses. Page p95 was 11.38 ms. +The final five minutes had zero sampled pending-acquisition and in-flight +counts; these are 60-second app scrapes, not proof of zero instantaneous work. + +HPA first reported two app replicas after 12.8 minutes and two ingress replicas +after 15.8 minutes. Nodes scaled from two to four during the trace and back to +three by the final candidate snapshot at 23:33:44 UTC. The autoscaler was +Healthy with bounds 2–4 and one remaining scale-down candidate. **Return to +two nodes within thirty minutes was not demonstrated.** No forced eviction or +node removal was used to manufacture a pass. The final candidate snapshot +precedes restoration; later observations of the restored configuration must +not be credited to this candidate. + +PostgreSQL recorded only two throttled periods and 24.1 ms of throttled time +across the candidate observation, including recovery. The measured VM CPU +averages were approximately 10.0%, 9.4% and 8.4% for baseline, app and infra, +with zero network-device error/drop increments. Aggregate utilization does +not exclude single-thread or external-network limits. All collected node +cAdvisor snapshots succeeded and contained telemetry. + +[Recovery results](evidence/workload-comparison-2026-10-29/recovery-comparison.json) +and the [final verdict](evidence/workload-comparison-2026-10-29/final-summary.json) +retain the delivery and footprint gates separately. + +[Restoration verification](evidence/workload-comparison-2026-10-29/cleanup-summary.json) +confirmed the original app and ingress Deployment specs, ingress HPA/ConfigMap, +DB resources (2-CPU request / 3-CPU limit) and bulk whitelist. The experimental +app HPA was removed; the primary ingress was unchanged. App, ingress and DB +were Ready. All 48 response hashes matched, the corpus remained 52,366 names / +187,108 versions, and no test publications existed. The owned generator VM and +boot disk were verified absent, and the temporary publish token was removed. + +The remaining spare node began removal at 23:34:04 UTC, overlapping the start +of restoration, so that is not an isolated two-node candidate observation. +During restoration an ingress pod triggered another scale-up: scheduler events +reported insufficient CPU and topology-spread constraints while a node was +unschedulable. The restored snapshot at 23:39 UTC had four Ready nodes within +the unchanged 2–4 bounds. This is separate rollout-induced capacity, not the +candidate's recovery result. Natural consolidation remains pending; restoring +configuration does not mean immediate restoration of idle compute cost. diff --git a/tools/ingressbench/WORKLOAD_PROFILE.md b/tools/ingressbench/WORKLOAD_PROFILE.md new file mode 100644 index 000000000..3819130f6 --- /dev/null +++ b/tools/ingressbench/WORKLOAD_PROFILE.md @@ -0,0 +1,128 @@ +# Production workload qualification — 2026-10-28 + +**The earlier 4,500-read/s test is a stress scenario, not a qualified replay of +the production incident.** It reproduced CPU contention, but material workload +differences prevent choosing the final capacity configuration from it alone. +Production and staging configuration were not changed during this audit. No +generator VM was created. + +Follow-up: [the ordered read-trace comparison](WORKLOAD_COMPARISON.md) qualifies +a final-minute broad-crawl read mix and compares the app and infrastructure +candidates. Its cold-start limitations prevent treating it as a full-cycle pass. + +## Evidence + +The [portable profile](evidence/workload-profile-2026-10-28/profile-summary.json) +combines four hours of historical application metrics (16:30–20:30 UTC) with +17 complete, one-minute Cloud Logging queries spaced 15 minutes apart. These +queries returned 962,375 records, including 959,671 parsed access records and +870,686 client records. The 2,704 unparsed records are retained as a coverage +limitation; they are not assumed to be successful requests. + +Each query was paginated to completion. These are samples, not continuous +ingress history or a request-arrival trace. Rates below count completions within +the selected minute. Zero-request-length records, used as a proxy for NGINX +background subrequests, are counted separately. Neither application request +rate nor all ingress log records should be equated with client demand. + +| Sample start (UTC) | Client requests/s | Client cache HIT | Top 128 page URI share | Distinct page URIs/min | +| --- | ---: | ---: | ---: | ---: | +| 16:30 | 308 | 30.0% | 6.0% | 9,524 | +| 17:00 | 1,783 | 93.9% | 98.5% | 1,547 | +| 17:45 | 1,984 | 87.2% | 17.9% | 8,560 | +| 18:30 | 911 | 69.3% | 13.6% | 11,833 | +| 19:00 | 478 | 40.6% | 10.9% | 12,231 | +| 19:30 | 187 | 17.0% | 12.4% | 7,310 | +| 20:30 | 147 | 15.6% | 9.0% | 5,059 | + +The cycle changes locality as well as rate. A short sample can be particularly +misleading: the first ten seconds at 17:00 contained 165 client records, whereas +the full minute contained 106,975. An isolated quiet slice is not a baseline. + +Application metrics independently show CPU saturation during the busy period: +at 17:45 both nodes were around 98–100% CPU, and v0.1 listing p95 was about +405 ms. At 20:30 they were around 25–35%, listing p95 about 86 ms, and the sampled +pending-acquisition gauge was zero. These are minute snapshots and rolling +histogram estimates, not proof that every intervening request recovered. + +## What the old test got wrong + +The catalog is ordered by API version and limit. Selecting 80% of page requests +from its first 128 entries made **85.02% of offered pages use `/v0` with limit +50**, with only 4.99% using `/v0.1` with limit 100. At 18:30, **92.9% of production +page requests used `/v0.1` with limit 100**. During the main observed wave that +share was roughly 93–99%. The test therefore exercised a different response +shape, not merely a higher rate of the same work. + +Other differences remain material: + +- The production minute at 18:30 contained 11,833 distinct page URIs, versus a + total test catalog of 4,000. Its top-128 share was 13.6%, compared with the + deliberate 80% hot selection in the test. Production locality varies sharply + across the cycle, so replacing 80% with one other fixed percentage is also + insufficient. +- That minute included 8.4 client searches/s and about 0.93 incremental reads/s. + The old test offered 3 and 36 respectively. Origin DB query rates are a + separate measurement because caches and background refreshes intervene. +- About 97.7% of client records in that minute used HTTP/1.1, but HTTP/2's share + changes during recovery. The logs do not expose TCP connection IDs or reuse + lifetimes. Synchronized 30-second generator rotation is an assumption. +- The old short gate jumps to a constant rate for three minutes. Production's + rise, sustained work and recovery span hours. Warm four-app/six-ingress tests + also do not reproduce scale-up from the production two-node baseline. +- Staging's recent DB requests, ingress requests and candidate image differ + from production. These are deliberate candidate changes, but require a + production-aligned baseline for causal comparison. + +## Implemented correction + +The driver now accepts explicit page classes and anonymous per-key frequencies, +validates distinct catalog capacity, records the selected profile hash and +attempted request shapes, and supports disabling forced connection rotation. +The legacy mode remains available for reproducibility and is labeled stress. + +The [measured broad-crawl page model](evidence/workload-profile-2026-10-28/measured-pages.json) +contains 53,463 page observations from the 18:30 minute, excluding 16 unsupported +page requests. A read-only staging catalog query mapped its 11,346 normalized +key ranks to distinct valid staging cursors. Query ordering/default limits are +normalized, explaining part of the difference from 11,833 raw URI cache keys. +This model is not a chronological client-session replay. It does not preserve +exact cursor positions, first-page frequency or response-size distribution. + +A local HTTPS fixture completed 300/300 requests with no drops or failures. +The small live staging check scheduled 200 reads at 10/s with four workers: +199 were attempted and returned HTTP 200; one was dropped by the bounded local +queue. Mean decoded body size was about 102 KB. This verifies usable requests, +**not** a capacity or fidelity pass; production log bytes are wire sizes and +cannot be equated with decoded response bytes. No publish requests were made. + +Go race tests, parser/privacy/pagination tests, and repository lint pass. Tests +specifically cover catalog-order bias, insufficient distinct keys, filter +isolation, page sequencing across search/update slots, bounded result labels, +background/client separation and incomplete pagination. + +## Gate before another capacity comparison + +First calibrate a baseline using the production release and resource settings. +Compare both client demand and work reaching the app/DB. Proposed calibration +tolerances are engineering criteria, not an established production SLO: + +1. Match API version, limit/filter and protocol shares within five percentage + points for each phase; include observed searches, updates, details and writes. +2. Match distinct keys/minute and response-size p50/p95 within 20%, and top-128 + popularity/cache-outcome shares within five percentage points. Measure both + wire and decoded bytes before using their distributions interchangeably. +3. Match origin page/search/update query rates within 20% over comparable + rolling windows. Reproduce CPU pressure and acquisition-delay symptoms; + matching total client RPS alone is insufficient. +4. Preserve the real duration and changing locality, or explicitly label a + shortened experiment as stress. Account for initial cache state and the + actual transition from idle capacity. +5. Treat unmeasured connection reuse/retries and search selectivity explicitly. + A bounded sensitivity comparison can test assumptions, but cannot turn them + into observed production facts. + +Only after the baseline passes workload qualification should the candidate run +against the same workload and recovery criteria. The full baseline/candidate +capacity comparison has **not** run as part of this audit. No production +promotion is justified by the fixture or low-rate staging check. diff --git a/tools/ingressbench/collect_profile_logs.py b/tools/ingressbench/collect_profile_logs.py new file mode 100644 index 000000000..f00d7b635 --- /dev/null +++ b/tools/ingressbench/collect_profile_logs.py @@ -0,0 +1,78 @@ +"""Read bounded production Cloud Logging slices and save only aggregate profiles.""" +import argparse +import datetime +import json +import pathlib +import subprocess +import urllib.request + +from profile_logs import Profile, parse_timestamp + + +def collect(start, seconds, token, fetch=None, max_records=200000, project='mcp-registry-prod'): + if not 1 <= seconds <= 300 or not 1 <= max_records <= 200000: + raise ValueError('duration must be 1–300 seconds and record cap 1–200000') + if project not in ('mcp-registry-prod', 'mcp-registry-staging'): + raise ValueError('unsupported registry project') + at = parse_timestamp(start) + end = (at + datetime.timedelta(seconds=seconds)).isoformat() + body = { + 'resourceNames': ['projects/' + project], + 'filter': f'resource.type="k8s_container" AND resource.labels.namespace_name="ingress-nginx" AND timestamp>="{at.isoformat()}" AND timestamp<"{end}"', + 'orderBy': 'timestamp asc', 'pageSize': min(10000, max_records), + } + if fetch is None: + def fetch(query): + request = urllib.request.Request( + 'https://logging.googleapis.com/v2/entries:list', data=json.dumps(query).encode(), + headers={'Authorization': 'Bearer ' + token, 'Content-Type': 'application/json'}) + with urllib.request.urlopen(request, timeout=45) as response: + return json.load(response) + profile = Profile() + count = pages = 0 + complete, error = False, None + try: + while count < max_records: + body['pageSize'] = min(10000, max_records-count) + data = fetch(body) + rows = data.get('entries', []) + count += len(rows) + pages += 1 + for row in rows: + profile.add(row.get('textPayload', row.get('jsonPayload', {}).get('message', '')), + parse_timestamp(row['timestamp']).timestamp(), + row.get('resource', {}).get('labels', {}).get('pod_name', 'unknown')) + if not data.get('nextPageToken'): + complete = True + break + body['pageToken'] = data['nextPageToken'] + if pages >= 1000: + raise RuntimeError('pagination bound exceeded') + except Exception as exc: # Preserve partial coverage; never print raw log data. + error = type(exc).__name__ + output = {'start': at.isoformat(), 'end': end, 'seconds': seconds, 'entries': count, + 'pages': pages, 'query_complete': complete, 'record_cap': max_records, + 'error_type': error, **profile.result()} + for group in output['populations'].values(): + group['records_per_second'] = group['count']/seconds if complete else None + return output + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--start', required=True, help='RFC3339 timestamp with timezone') + parser.add_argument('--seconds', type=int, default=60) + parser.add_argument('--out', type=pathlib.Path, required=True) + parser.add_argument('--project', choices=['mcp-registry-prod', 'mcp-registry-staging'], + default='mcp-registry-prod') + args = parser.parse_args() + token = subprocess.check_output(['gcloud', 'auth', 'print-access-token'], text=True, timeout=30).strip() + result = collect(args.start, args.seconds, token, project=args.project) + args.out.write_text(json.dumps(result, indent=2) + '\n') + print(json.dumps({k: result[k] for k in ['query_complete', 'entries', 'parsed', 'unparsed', 'error_type']})) + if not result['query_complete']: + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/tools/ingressbench/evidence/app-profile-2026-10-28/cache-check-checkout-summary.json b/tools/ingressbench/evidence/app-profile-2026-10-28/cache-check-checkout-summary.json new file mode 100644 index 000000000..faaffaa64 --- /dev/null +++ b/tools/ingressbench/evidence/app-profile-2026-10-28/cache-check-checkout-summary.json @@ -0,0 +1,256 @@ +{ + "mcp-registry-7fc9585957-6rq6t": { + "first": "burst-000-mcp-registry-7fc9585957-6rq6t.prom", + "last": "burst-180-mcp-registry-7fc9585957-6rq6t.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 29899.0, + "sum": 425.49390079799286, + "mean_ms": 14.231041198635168 + }, + "mcp_registry_db_pool_acquire_duration/list": { + "count": 37.0, + "sum": 1.1831182200000006, + "mean_ms": 31.976168108108123 + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 2095.0, + "sum": 10.428127551000028, + "mean_ms": 4.977626515990466 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 171.0, + "sum": 1.7546932549999994, + "mean_ms": 10.261364064327482 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 29845.0, + "sum": 312.9376773679973, + "mean_ms": 10.485430637225575 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 37.0, + "sum": 0.3945361289999999, + "mean_ms": 10.663138621621618 + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 2095.0, + "sum": 20.28154378100003, + "mean_ms": 9.680927819093093 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 171.0, + "sum": 8.446093573000008, + "mean_ms": 49.39236007602344 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 29845.0, + "sum": 1.4232190949999868, + "mean_ms": 0.04768701943374055 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 37.0, + "sum": 0.0008786980000000003, + "mean_ms": 0.023748594594594604 + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 2095.0, + "sum": 0.07606957599999996, + "mean_ms": 0.03631006014319807 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 171.0, + "sum": 0.010726379000000001, + "mean_ms": 0.06272736257309942 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_page": { + "count": 29845.0, + "sum": 310.13403425299856, + "mean_ms": 10.39149050939851 + }, + "mcp_registry_db_query_duration/list": { + "count": 37.0, + "sum": 0.3922005189999997, + "mean_ms": 10.600014027027019 + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 2095.0, + "sum": 20.138220753000017, + "mean_ms": 9.612515872553708 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 171.0, + "sum": 8.437472827999994, + "mean_ms": 49.341946362573054 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 5.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 55.0 + } + }, + "cache": { + "hit": 182891.0, + "miss": 2088894.0, + "hit_fraction": 0.08050541754611462, + "mcp_registry_db_listing_json_cache_entries": 8192.0, + "mcp_registry_db_listing_json_cache_payload_bytes": 6806144.0 + } + }, + "mcp-registry-7fc9585957-r6gck": { + "first": "burst-000-mcp-registry-7fc9585957-r6gck.prom", + "last": "burst-180-mcp-registry-7fc9585957-r6gck.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 29911.0, + "sum": 2194.2550947149966, + "mean_ms": 73.35946958359789 + }, + "mcp_registry_db_pool_acquire_duration/list": { + "count": 43.0, + "sum": 7.017721797, + "mean_ms": 163.2028324883721 + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 2221.0, + "sum": 108.9760119389998, + "mean_ms": 49.06619177802783 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 188.0, + "sum": 6.797710042000006, + "mean_ms": 36.158032138297905 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 29874.0, + "sum": 2707.6365600479967, + "mean_ms": 90.6352199252861 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 43.0, + "sum": 7.391689142000002, + "mean_ms": 171.89974748837213 + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 2220.0, + "sum": 200.93862396700013, + "mean_ms": 90.51289367882889 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 188.0, + "sum": 25.235101978999996, + "mean_ms": 134.22926584574466 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 29874.0, + "sum": 7.364668155000007, + "mean_ms": 0.2465243407310707 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 43.0, + "sum": 0.012715377, + "mean_ms": 0.29570644186046513 + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 2220.0, + "sum": 0.5013231059999995, + "mean_ms": 0.2258212189189187 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 188.0, + "sum": 0.022854259000000012, + "mean_ms": 0.12156520744680858 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_page": { + "count": 29874.0, + "sum": 2681.161433420023, + "mean_ms": 89.74899355359253 + }, + "mcp_registry_db_query_duration/list": { + "count": 43.0, + "sum": 6.981532712000001, + "mean_ms": 162.36122586046514 + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 2220.0, + "sum": 199.24890840000003, + "mean_ms": 89.75176054054056 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 188.0, + "sum": 25.176720596999985, + "mean_ms": 133.91872657978715 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 29.0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 1.0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 58.0, + "{'state': 'constructing'}": 3.0, + "{'state': 'idle'}": 56.0 + } + }, + "cache": { + "hit": 287038.0, + "miss": 1922144.0, + "hit_fraction": 0.1299295395309214, + "mcp_registry_db_listing_json_cache_entries": 8192.0, + "mcp_registry_db_listing_json_cache_payload_bytes": 7030096.0 + } + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/app-profile-2026-10-28/comparison.json b/tools/ingressbench/evidence/app-profile-2026-10-28/comparison.json new file mode 100644 index 000000000..b6fab4799 --- /dev/null +++ b/tools/ingressbench/evidence/app-profile-2026-10-28/comparison.json @@ -0,0 +1,223 @@ +{ + "production_changed": false, + "short_diagnostic_only": true, + "arms": { + "diagnosis": { + "scheduled_reads": 810000, + "attempts_finished": 686985, + "dropped": 123015, + "drop_percent": 15.187037037037038, + "attempt_failures": 95, + "errors": { + "page:body:timeout": 1, + "page:request:timeout": 69, + "search:request:timeout": 2, + "updated:request:timeout": 23 + }, + "latency_ms": { + "page_p50": 251.070017, + "page_p95": 1235.930531, + "page_p99": 5599.785674, + "search_p50": 401.166649, + "search_p95": 5816.341109, + "search_p99": 6783.554524, + "updated_p50": 3875.411338, + "updated_p95": 5861.290088, + "updated_p99": 7410.209708 + }, + "start": "2026-10-28T10:45:57.055163008Z", + "end": "2026-10-28T10:48:59.351126811Z", + "replicas": { + "mcp-registry-75dc47cb8c-js56r": { + "observed_origin_completions": 35135.0, + "process_cpu_seconds": 164.95, + "cpu_ms_per_origin_completion": 4.6947488259570225, + "peak_sampled_pool_waiters": 899.0, + "peak_heap_inuse_bytes": 123002880.0, + "payload_sources": null, + "first_sample": "burst-000-mcp-registry-75dc47cb8c-js56r.prom", + "last_sample": "burst-180-mcp-registry-75dc47cb8c-js56r.prom" + }, + "mcp-registry-75dc47cb8c-ksg8g": { + "observed_origin_completions": 36121.0, + "process_cpu_seconds": 168.98, + "cpu_ms_per_origin_completion": 4.678165056338418, + "peak_sampled_pool_waiters": 0.0, + "peak_heap_inuse_bytes": 38084608.0, + "payload_sources": null, + "first_sample": "burst-000-mcp-registry-75dc47cb8c-ksg8g.prom", + "last_sample": "burst-180-mcp-registry-75dc47cb8c-ksg8g.prom" + } + }, + "short_recovery": { + "reads": 12000, + "dropped": 0, + "failures": 0 + } + }, + "cache-check": { + "scheduled_reads": 810000, + "attempts_finished": 552711, + "dropped": 257289, + "drop_percent": 31.764074074074074, + "attempt_failures": 9517, + "errors": { + "page:body:timeout": 17, + "page:request:timeout": 9253, + "search:request:timeout": 40, + "updated:body:timeout": 1, + "updated:request:timeout": 206 + }, + "latency_ms": { + "page_p50": 336.21588, + "page_p95": 1059.356022, + "page_p99": 10311.673549, + "search_p50": 470.867973, + "search_p95": 10329.207561, + "search_p99": 10405.934062, + "updated_p50": 441.863005, + "updated_p95": 9911.322031, + "updated_p99": 10368.672263 + }, + "start": "2026-10-28T11:08:41.967713271Z", + "end": "2026-10-28T11:11:45.223509111Z", + "replicas": { + "mcp-registry-7fc9585957-6rq6t": { + "observed_origin_completions": 32278.0, + "process_cpu_seconds": 196.92, + "cpu_ms_per_origin_completion": 6.100749736662743, + "peak_sampled_pool_waiters": 0.0, + "peak_heap_inuse_bytes": 60989440.0, + "payload_sources": null, + "first_sample": "burst-000-mcp-registry-7fc9585957-6rq6t.prom", + "last_sample": "burst-180-mcp-registry-7fc9585957-6rq6t.prom" + }, + "mcp-registry-7fc9585957-r6gck": { + "observed_origin_completions": 30656.0, + "process_cpu_seconds": 182.85, + "cpu_ms_per_origin_completion": 5.964574634655532, + "peak_sampled_pool_waiters": 30.0, + "peak_heap_inuse_bytes": 451141632.0, + "payload_sources": null, + "first_sample": "burst-000-mcp-registry-7fc9585957-r6gck.prom", + "last_sample": "burst-180-mcp-registry-7fc9585957-r6gck.prom" + } + }, + "short_recovery": { + "reads": 12000, + "dropped": 0, + "failures": 0 + } + }, + "prepared-check": { + "scheduled_reads": 810000, + "attempts_finished": 753157, + "dropped": 56843, + "drop_percent": 7.0176543209876545, + "attempt_failures": 2, + "errors": { + "updated:request:timeout": 2 + }, + "latency_ms": { + "page_p50": 8.232611, + "page_p95": 710.60077, + "page_p99": 4256.339053, + "search_p50": 109.884778, + "search_p95": 5350.352014, + "search_p99": 6420.12033, + "updated_p50": 153.105197, + "updated_p95": 5150.545888, + "updated_p99": 6082.472277 + }, + "start": "2026-10-28T11:36:28.479166749Z", + "end": "2026-10-28T11:39:30.299524647Z", + "replicas": { + "mcp-registry-5f767c4845-4v89g": { + "observed_origin_completions": 39343.0, + "process_cpu_seconds": 119.07000000000001, + "cpu_ms_per_origin_completion": 3.0264595989121323, + "peak_sampled_pool_waiters": 893.0, + "peak_heap_inuse_bytes": 114507776.0, + "payload_sources": { + "prepared": 2805814.0, + "fallback": 0.0 + }, + "first_sample": "burst-000-mcp-registry-5f767c4845-4v89g.prom", + "last_sample": "burst-180-mcp-registry-5f767c4845-4v89g.prom" + }, + "mcp-registry-5f767c4845-vddtl": { + "observed_origin_completions": 40304.0, + "process_cpu_seconds": 127.87, + "cpu_ms_per_origin_completion": 3.1726379515680825, + "peak_sampled_pool_waiters": 0.0, + "peak_heap_inuse_bytes": 49455104.0, + "payload_sources": { + "prepared": 2872981.0, + "fallback": 0.0 + }, + "first_sample": "burst-000-mcp-registry-5f767c4845-vddtl.prom", + "last_sample": "burst-180-mcp-registry-5f767c4845-vddtl.prom" + } + }, + "short_recovery": { + "reads": 12000, + "dropped": 0, + "failures": 0 + } + }, + "prepared-cpu-check": { + "scheduled_reads": 810000, + "attempts_finished": 766880, + "dropped": 43120, + "drop_percent": 5.323456790123457, + "attempt_failures": 0, + "errors": {}, + "latency_ms": { + "page_p50": 7.961928, + "page_p95": 728.886813, + "page_p99": 3935.847246, + "search_p50": 170.103934, + "search_p95": 4321.247371, + "search_p99": 5482.600454, + "updated_p50": 262.484453, + "updated_p95": 4794.609, + "updated_p99": 5468.749362 + }, + "start": "2026-10-28T11:45:52.423835392Z", + "end": "2026-10-28T11:48:52.516379847Z", + "replicas": { + "mcp-registry-685dff56b7-48qz9": { + "observed_origin_completions": 40863.0, + "process_cpu_seconds": 149.49, + "cpu_ms_per_origin_completion": 3.658321709125615, + "peak_sampled_pool_waiters": 900.0, + "peak_heap_inuse_bytes": 118947840.0, + "payload_sources": { + "prepared": 2915345.0, + "fallback": 0.0 + }, + "first_sample": "burst-000-mcp-registry-685dff56b7-48qz9.prom", + "last_sample": "burst-180-mcp-registry-685dff56b7-48qz9.prom" + }, + "mcp-registry-685dff56b7-tzs8j": { + "observed_origin_completions": 40893.0, + "process_cpu_seconds": 121.4, + "cpu_ms_per_origin_completion": 2.9687232533685473, + "peak_sampled_pool_waiters": 0.0, + "peak_heap_inuse_bytes": 60710912.0, + "payload_sources": { + "prepared": 2907534.0, + "fallback": 0.0 + }, + "first_sample": "burst-000-mcp-registry-685dff56b7-tzs8j.prom", + "last_sample": "burst-180-mcp-registry-685dff56b7-tzs8j.prom" + } + }, + "short_recovery": { + "reads": 12000, + "dropped": 0, + "failures": 0 + } + } + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/app-profile-2026-10-28/diagnosis-checkout-summary.json b/tools/ingressbench/evidence/app-profile-2026-10-28/diagnosis-checkout-summary.json new file mode 100644 index 000000000..96cbfe688 --- /dev/null +++ b/tools/ingressbench/evidence/app-profile-2026-10-28/diagnosis-checkout-summary.json @@ -0,0 +1,256 @@ +{ + "mcp-registry-75dc47cb8c-js56r": { + "first": "burst-000-mcp-registry-75dc47cb8c-js56r.prom", + "last": "burst-180-mcp-registry-75dc47cb8c-js56r.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 32383.0, + "sum": 139923.75845917864, + "mean_ms": 4320.901660104952 + }, + "mcp_registry_db_pool_acquire_duration/list": { + "count": 42.0, + "sum": 177.077460854, + "mean_ms": 4216.130020333333 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 2573.0, + "sum": 11343.563675226987, + "mean_ms": 4408.691673232409 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 213.0, + "sum": 929.6411971990007, + "mean_ms": 4364.512662906106 + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 32336.0, + "sum": 6967.861922144939, + "mean_ms": 215.4831123869662 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 42.0, + "sum": 14.246622057999996, + "mean_ms": 339.20528709523796 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 2573.0, + "sum": 527.2341940569994, + "mean_ms": 204.910296951807 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 213.0, + "sum": 53.80341299699996, + "mean_ms": 252.59818308450687 + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 32336.0, + "sum": 44.09452008399998, + "mean_ms": 1.3636355790450267 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 42.0, + "sum": 0.0006797800000000003, + "mean_ms": 0.016185238095238103 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 2573.0, + "sum": 2.6879455869999966, + "mean_ms": 1.0446737609794001 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 213.0, + "sum": 0.2885354659999997, + "mean_ms": 1.3546266009389658 + }, + "mcp_registry_db_query_duration/list_page": { + "count": 32336.0, + "sum": 6388.073811950991, + "mean_ms": 197.55300012218552 + }, + "mcp_registry_db_query_duration/list": { + "count": 42.0, + "sum": 13.88802664, + "mean_ms": 330.66730095238097 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 2573.0, + "sum": 482.04232194399975, + "mean_ms": 187.34641350330344 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 213.0, + "sum": 51.560183335999994, + "mean_ms": 242.06658843192486 + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 3.0, + "{'operation': 'list_page'}": 843.0, + "{'operation': 'list_search'}": 9.0, + "{'operation': 'list_updated'}": 77.0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 58.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 19.0 + } + }, + "cache": { + "hit": 0, + "miss": 0, + "hit_fraction": null, + "mcp_registry_db_listing_json_cache_entries": 0, + "mcp_registry_db_listing_json_cache_payload_bytes": 0 + } + }, + "mcp-registry-75dc47cb8c-ksg8g": { + "first": "burst-000-mcp-registry-75dc47cb8c-ksg8g.prom", + "last": "burst-180-mcp-registry-75dc47cb8c-ksg8g.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 33096.0, + "sum": 647.7390540120152, + "mean_ms": 19.571520848803942 + }, + "mcp_registry_db_pool_acquire_duration/list": { + "count": 39.0, + "sum": 0.6372074540000005, + "mean_ms": 16.33865266666668 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 2767.0, + "sum": 20.754347321999983, + "mean_ms": 7.500667626310077 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 220.0, + "sum": 1.852707417, + "mean_ms": 8.421397350000001 + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 33096.0, + "sum": 435.9225932980025, + "mean_ms": 13.171458584058572 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 39.0, + "sum": 0.5308828280000001, + "mean_ms": 13.612380205128208 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 2767.0, + "sum": 34.24139123700006, + "mean_ms": 12.37491551752803 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 219.0, + "sum": 9.030233360999997, + "mean_ms": 41.23394228767122 + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 33096.0, + "sum": 1.6756086680000077, + "mean_ms": 0.050628736644912006 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 39.0, + "sum": 0.05231855299999999, + "mean_ms": 1.3415013589743587 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 2767.0, + "sum": 0.12623753300000023, + "mean_ms": 0.04562252728586926 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 219.0, + "sum": 0.010610954000000002, + "mean_ms": 0.04845184474885846 + }, + "mcp_registry_db_query_duration/list_page": { + "count": 33096.0, + "sum": 427.2969633130019, + "mean_ms": 12.910834037738757 + }, + "mcp_registry_db_query_duration/list": { + "count": 39.0, + "sum": 0.477987415, + "mean_ms": 12.256087564102565 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 2767.0, + "sum": 33.890723538000024, + "mean_ms": 12.248183425370447 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 219.0, + "sum": 9.013212399, + "mean_ms": 41.156221 + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 3.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 60.0 + } + }, + "cache": { + "hit": 0, + "miss": 0, + "hit_fraction": null, + "mcp_registry_db_listing_json_cache_entries": 0, + "mcp_registry_db_listing_json_cache_payload_bytes": 0 + } + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-backfill.json b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-backfill.json new file mode 100644 index 000000000..f0bf41f26 --- /dev/null +++ b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-backfill.json @@ -0,0 +1,8 @@ +{ + "completed_at": 1793187208.031768, + "exit_code": 0, + "batches": 376, + "scanned": 187108, + "prepared": 187108, + "skipped_identity": 0 +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-check-app-pressure-summary.json b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-check-app-pressure-summary.json new file mode 100644 index 000000000..23d04535a --- /dev/null +++ b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-check-app-pressure-summary.json @@ -0,0 +1,50 @@ +[ + { + "pod": "mcp-registry-5f767c4845-vddtl", + "metric": "container_cpu_usage_seconds_total", + "seconds": 153.43299984931946, + "rate": 0.6979223055350761, + "start": 1793187403.756, + "end": 1793187557.189 + }, + { + "pod": "mcp-registry-5f767c4845-vddtl", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 153.43299984931946, + "rate": 0.1898387636858108, + "start": 1793187403.756, + "end": 1793187557.189 + }, + { + "pod": "mcp-registry-5f767c4845-vddtl", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 153.43299984931946, + "rate": 0.26651577587714986, + "start": 1793187403.756, + "end": 1793187557.189 + }, + { + "pod": "mcp-registry-5f767c4845-4v89g", + "metric": "container_cpu_usage_seconds_total", + "seconds": 141.85899996757507, + "rate": 0.6674637493683337, + "start": 1793187402.493, + "end": 1793187544.352 + }, + { + "pod": "mcp-registry-5f767c4845-4v89g", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 141.85899996757507, + "rate": 0.31388750104101804, + "start": 1793187402.493, + "end": 1793187544.352 + }, + { + "pod": "mcp-registry-5f767c4845-4v89g", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 141.85899996757507, + "rate": 0.41932248227885793, + "start": 1793187402.493, + "end": 1793187544.352 + } +] \ No newline at end of file diff --git a/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-check-checkout-summary.json b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-check-checkout-summary.json new file mode 100644 index 000000000..c4dde1241 --- /dev/null +++ b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-check-checkout-summary.json @@ -0,0 +1,250 @@ +{ + "mcp-registry-5f767c4845-4v89g": { + "first": "burst-000-mcp-registry-5f767c4845-4v89g.prom", + "last": "burst-180-mcp-registry-5f767c4845-4v89g.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list": { + "count": 43.0, + "sum": 49.774483958999994, + "mean_ms": 1157.5461385813953 + }, + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 36261.0, + "sum": 54186.1863280828, + "mean_ms": 1494.337892724492 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 248.0, + "sum": 400.04572328100016, + "mean_ms": 1613.0875938750007 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 2842.0, + "sum": 4338.391311590002, + "mean_ms": 1526.527555098523 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 43.0, + "sum": 6.991384839, + "mean_ms": 162.59034509302327 + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 36233.0, + "sum": 4269.480231568994, + "mean_ms": 117.83402510333106 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 248.0, + "sum": 34.29042988900002, + "mean_ms": 138.26786245564526 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 2839.0, + "sum": 272.31096460500027, + "mean_ms": 95.91791638076798 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 43.0, + "sum": 0.0008035549999999999, + "mean_ms": 0.018687325581395343 + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 36233.0, + "sum": 26.085540906000176, + "mean_ms": 0.7199387548919542 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 248.0, + "sum": 0.017891243, + "mean_ms": 0.07214210887096775 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 2839.0, + "sum": 5.6125384649999965, + "mean_ms": 1.9769420447340602 + }, + "mcp_registry_db_query_duration/list": { + "count": 43.0, + "sum": 5.519831676999999, + "mean_ms": 128.3681785348837 + }, + "mcp_registry_db_query_duration/list_page": { + "count": 36233.0, + "sum": 3635.5562619770267, + "mean_ms": 100.33826241208365 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 248.0, + "sum": 30.205716500000005, + "mean_ms": 121.79724395161293 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 2839.0, + "sum": 212.60283943900026, + "mean_ms": 74.88652322613606 + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 3.0, + "{'operation': 'list_page'}": 834.0, + "{'operation': 'list_search'}": 9.0, + "{'operation': 'list_updated'}": 77.0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 60.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 59.0 + } + }, + "payload_sources": { + "prepared": 2805814.0, + "fallback": 0.0 + } + }, + "mcp-registry-5f767c4845-vddtl": { + "first": "burst-000-mcp-registry-5f767c4845-vddtl.prom", + "last": "burst-180-mcp-registry-5f767c4845-vddtl.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list": { + "count": 48.0, + "sum": 0.001594517, + "mean_ms": 0.033219104166666666 + }, + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 37049.0, + "sum": 444.1121059719944, + "mean_ms": 11.987155010175561 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 236.0, + "sum": 1.085973621, + "mean_ms": 4.601583139830508 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 2983.0, + "sum": 14.77997897899997, + "mean_ms": 4.954736499832373 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 48.0, + "sum": 1.0871106739999994, + "mean_ms": 22.648139041666653 + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 37039.0, + "sum": 655.0482190649992, + "mean_ms": 17.685364590431686 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 236.0, + "sum": 9.186436356, + "mean_ms": 38.92557777966102 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 2983.0, + "sum": 38.49288359700007, + "mean_ms": 12.904084343613835 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 48.0, + "sum": 0.0011242029999999998, + "mean_ms": 0.02342089583333333 + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 37039.0, + "sum": 3.9117999779999395, + "mean_ms": 0.10561300191689676 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 236.0, + "sum": 0.009487944000000012, + "mean_ms": 0.04020315254237293 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 2983.0, + "sum": 1.3622614230000007, + "mean_ms": 0.4566749658062356 + }, + "mcp_registry_db_query_duration/list": { + "count": 48.0, + "sum": 1.0851089080000003, + "mean_ms": 22.606435583333337 + }, + "mcp_registry_db_query_duration/list_page": { + "count": 37039.0, + "sum": 639.3003720039995, + "mean_ms": 17.260195253759537 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 236.0, + "sum": 9.173180537999999, + "mean_ms": 38.86940905932203 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 2983.0, + "sum": 36.16434850799997, + "mean_ms": 12.123482570566532 + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 10.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 60.0 + } + }, + "payload_sources": { + "prepared": 2872981.0, + "fallback": 0.0 + } + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-check-container-cpu-summary.json b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-check-container-cpu-summary.json new file mode 100644 index 000000000..e99febf19 --- /dev/null +++ b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-check-container-cpu-summary.json @@ -0,0 +1,77 @@ +{ + "scope": "Bounded burst intervals per cAdvisor series, default/ingress namespaces only; intervals are not exactly identical. System pods and node kernel overhead are excluded.", + "rows": [ + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "pod": "ingress-nginx-controller-8587f46b84-m9sss", + "container": "controller", + "start": 1793187404.216, + "end": 1793187556.127, + "cpu_cores_mean": 0.7676151825065424 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "pod": "ingress-nginx-controller-8587f46b84-q78zh", + "container": "controller", + "start": 1793187406.852, + "end": 1793187543.717, + "cpu_cores_mean": 0.6866729039086061 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "pod": "mcp-registry-5f767c4845-vddtl", + "container": "mcp-registry", + "start": 1793187403.756, + "end": 1793187557.189, + "cpu_cores_mean": 0.6979223055350761 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-jb9n", + "pod": "ingress-nginx-controller-8587f46b84-2t69w", + "container": "controller", + "start": 1793187400.213, + "end": 1793187544.021, + "cpu_cores_mean": 0.5463551894353937 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-jb9n", + "pod": "ingress-nginx-controller-8587f46b84-84rdr", + "container": "controller", + "start": 1793187396.998, + "end": 1793187553.072, + "cpu_cores_mean": 0.5855333491141419 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "pod": "ingress-nginx-controller-8587f46b84-rr9nr", + "container": "controller", + "start": 1793187392.779, + "end": 1793187541.727, + "cpu_cores_mean": 1.4996833194715615 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "pod": "mcp-registry-5f767c4845-4v89g", + "container": "mcp-registry", + "start": 1793187402.493, + "end": 1793187544.352, + "cpu_cores_mean": 0.6674637493683337 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "pod": "registry-pg-1", + "container": "postgres", + "start": 1793187397.465, + "end": 1793187555.336, + "cpu_cores_mean": 0.7161023111472128 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-sdtz", + "pod": "ingress-nginx-controller-8587f46b84-hmt7n", + "container": "controller", + "start": 1793187393.476, + "end": 1793187556.2, + "cpu_cores_mean": 1.106968228567 + } + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-check-runtime-integrity.json b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-check-runtime-integrity.json new file mode 100644 index 000000000..e614c7b7d --- /dev/null +++ b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-check-runtime-integrity.json @@ -0,0 +1,25 @@ +{ + "as_of": 1793187727.645783, + "samples": 16, + "commands_failed": [], + "app_restarts": { + "mcp-registry-5f767c4845-4v89g/mcp-registry": 0, + "mcp-registry-5f767c4845-vddtl/mcp-registry": 0, + "registry-pg-1/postgres": 0 + }, + "node_count_timeline": [ + { + "at": 1793187255.700844, + "nodes": 4 + } + ], + "db_throttled_usec_range": [ + 869130, + 869130 + ], + "db_endpoint_samples": 86, + "db_endpoint_errors_or_empty": [], + "db_endpoint_not_ready": [], + "cadvisor_failures_or_empty": [], + "lb_collection_failures": [] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-coverage.json b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-coverage.json new file mode 100644 index 000000000..875d88932 --- /dev/null +++ b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-coverage.json @@ -0,0 +1,10 @@ +{ + "at": "2026-10-28T11:33:34.798187+00:00", + "rows": 187108, + "prepared": 187108, + "jsonb_text_bytes": 162038018, + "prepared_bytes": 155017950, + "table_bytes": 349216768, + "indexes_bytes": 212328448, + "postmaster_start": "2026-10-27T21:24:59.430872+00:00" +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-app-pressure-summary.json b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-app-pressure-summary.json new file mode 100644 index 000000000..d8cfa3a99 --- /dev/null +++ b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-app-pressure-summary.json @@ -0,0 +1,50 @@ +[ + { + "pod": "mcp-registry-685dff56b7-48qz9", + "metric": "container_cpu_usage_seconds_total", + "seconds": 153.62700009346008, + "rate": 0.8265537237773973, + "start": 1793187957.843, + "end": 1793188111.47 + }, + { + "pod": "mcp-registry-685dff56b7-48qz9", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 153.62700009346008, + "rate": 0.3890580234180099, + "start": 1793187957.843, + "end": 1793188111.47 + }, + { + "pod": "mcp-registry-685dff56b7-48qz9", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 153.62700009346008, + "rate": 0.5358288643918152, + "start": 1793187957.843, + "end": 1793188111.47 + }, + { + "pod": "mcp-registry-685dff56b7-tzs8j", + "metric": "container_cpu_usage_seconds_total", + "seconds": 144.22499990463257, + "rate": 0.6666630408291069, + "start": 1793187969.154, + "end": 1793188113.379 + }, + { + "pod": "mcp-registry-685dff56b7-tzs8j", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 144.22499990463257, + "rate": 0.2408143874013926, + "start": 1793187969.154, + "end": 1793188113.379 + }, + { + "pod": "mcp-registry-685dff56b7-tzs8j", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 144.22499990463257, + "rate": 0.3396559821972066, + "start": 1793187969.154, + "end": 1793188113.379 + } +] \ No newline at end of file diff --git a/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-checkout-summary.json b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-checkout-summary.json new file mode 100644 index 000000000..2a1dc28fe --- /dev/null +++ b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-checkout-summary.json @@ -0,0 +1,250 @@ +{ + "mcp-registry-685dff56b7-48qz9": { + "first": "burst-000-mcp-registry-685dff56b7-48qz9.prom", + "last": "burst-180-mcp-registry-685dff56b7-48qz9.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list": { + "count": 41.0, + "sum": 72.97298986000001, + "mean_ms": 1779.82902097561 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 243.0, + "sum": 384.3209296120004, + "mean_ms": 1581.567611572018 + }, + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 37681.0, + "sum": 62661.115775340266, + "mean_ms": 1662.9366464621498 + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 2935.0, + "sum": 4832.284966661008, + "mean_ms": 1646.4344009066467 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 41.0, + "sum": 8.057877312000002, + "mean_ms": 196.53359297560982 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 243.0, + "sum": 48.028122104, + "mean_ms": 197.64659302057612 + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 37663.0, + "sum": 5836.756948023934, + "mean_ms": 154.97323495271047 + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 2934.0, + "sum": 381.9355693750002, + "mean_ms": 130.17572235003416 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 41.0, + "sum": 0.089748763, + "mean_ms": 2.188994219512195 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 243.0, + "sum": 0.2942551669999999, + "mean_ms": 1.2109266131687237 + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 37663.0, + "sum": 47.16015979500002, + "mean_ms": 1.25216153240581 + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 2934.0, + "sum": 6.348170307999989, + "mean_ms": 2.1636572283571875 + }, + "mcp_registry_db_query_duration/list": { + "count": 41.0, + "sum": 7.894403977000001, + "mean_ms": 192.5464384634147 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_search": { + "count": 243.0, + "sum": 43.66806680699998, + "mean_ms": 179.70397862962955 + }, + "mcp_registry_db_query_duration/list_page": { + "count": 37663.0, + "sum": 5185.536685331962, + "mean_ms": 137.68251826280334 + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 2934.0, + "sum": 330.1328185629995, + "mean_ms": 112.51970639502368 + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 1.0, + "{'operation': 'list_page'}": 846.0, + "{'operation': 'list_search'}": 6.0, + "{'operation': 'list_updated'}": 71.0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 58.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 59.0 + } + }, + "payload_sources": { + "prepared": 2915345.0, + "fallback": 0.0 + } + }, + "mcp-registry-685dff56b7-tzs8j": { + "first": "burst-000-mcp-registry-685dff56b7-tzs8j.prom", + "last": "burst-180-mcp-registry-685dff56b7-tzs8j.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list": { + "count": 52.0, + "sum": 6.310755440999998, + "mean_ms": 121.36068155769226 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 257.0, + "sum": 6.355719186999995, + "mean_ms": 24.730424852140057 + }, + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 37474.0, + "sum": 2306.797697421041, + "mean_ms": 61.557284982148715 + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 3110.0, + "sum": 99.84304416799989, + "mean_ms": 32.10387272282955 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 52.0, + "sum": 2.3995811619999987, + "mean_ms": 46.14579157692305 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 257.0, + "sum": 15.509352962999994, + "mean_ms": 60.34767689883266 + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 37474.0, + "sum": 1088.9586607270146, + "mean_ms": 29.059045224075746 + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 3110.0, + "sum": 61.605811719000066, + "mean_ms": 19.808942674919635 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 52.0, + "sum": 0.0008391569999999998, + "mean_ms": 0.016137634615384613 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 257.0, + "sum": 0.014421249000000021, + "mean_ms": 0.05611380933852148 + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 37474.0, + "sum": 6.924113302000028, + "mean_ms": 0.18477112936969706 + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 3110.0, + "sum": 0.8421470720000007, + "mean_ms": 0.27078683987138286 + }, + "mcp_registry_db_query_duration/list": { + "count": 52.0, + "sum": 2.162753165, + "mean_ms": 41.591407019230765 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_search": { + "count": 257.0, + "sum": 15.390850828, + "mean_ms": 59.88657909727626 + }, + "mcp_registry_db_query_duration/list_page": { + "count": 37474.0, + "sum": 1023.3793692590028, + "mean_ms": 27.30905078878697 + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 3110.0, + "sum": 59.100986604999996, + "mean_ms": 19.003532670418007 + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 39.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 60.0 + } + }, + "payload_sources": { + "prepared": 2907534.0, + "fallback": 0.0 + } + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-container-cpu-summary.json b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-container-cpu-summary.json new file mode 100644 index 000000000..328c40086 --- /dev/null +++ b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-container-cpu-summary.json @@ -0,0 +1,77 @@ +{ + "scope": "Bounded burst intervals per cAdvisor series, default/ingress namespaces only; intervals are not exactly identical. System pods and node kernel overhead are excluded.", + "rows": [ + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "pod": "ingress-nginx-controller-8587f46b84-m9sss", + "container": "controller", + "start": 1793187965.646, + "end": 1793188113.993, + "cpu_cores_mean": 0.907919188720838 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "pod": "ingress-nginx-controller-8587f46b84-q78zh", + "container": "controller", + "start": 1793187964.617, + "end": 1793188113.931, + "cpu_cores_mean": 0.8301688394273478 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "pod": "mcp-registry-685dff56b7-48qz9", + "container": "mcp-registry", + "start": 1793187957.843, + "end": 1793188111.47, + "cpu_cores_mean": 0.8265537237773973 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-jb9n", + "pod": "ingress-nginx-controller-8587f46b84-2t69w", + "container": "controller", + "start": 1793187960.937, + "end": 1793188118.739, + "cpu_cores_mean": 0.5458778721119613 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-jb9n", + "pod": "ingress-nginx-controller-8587f46b84-84rdr", + "container": "controller", + "start": 1793187968.502, + "end": 1793188117.902, + "cpu_cores_mean": 0.5644004824681268 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "pod": "ingress-nginx-controller-8587f46b84-rr9nr", + "container": "controller", + "start": 1793187965.933, + "end": 1793188110.733, + "cpu_cores_mean": 1.4194825004674445 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "pod": "mcp-registry-685dff56b7-tzs8j", + "container": "mcp-registry", + "start": 1793187969.154, + "end": 1793188113.379, + "cpu_cores_mean": 0.6666630408291069 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "pod": "registry-pg-1", + "container": "postgres", + "start": 1793187971.666, + "end": 1793188116.235, + "cpu_cores_mean": 0.7133892120435082 + }, + { + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-sdtz", + "pod": "ingress-nginx-controller-8587f46b84-hmt7n", + "container": "controller", + "start": 1793187967.405, + "end": 1793188107.671, + "cpu_cores_mean": 1.2138303007189315 + } + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-db-burst-statement-deltas.json b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-db-burst-statement-deltas.json new file mode 100644 index 000000000..71ec5f777 --- /dev/null +++ b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-db-burst-statement-deltas.json @@ -0,0 +1,17 @@ +{ + "before": "diagnostics-1793187820.json", + "after": "diagnostics-1793188148.json", + "window_seconds": 328.9415910243988, + "reset_unchanged": true, + "scope": "Only statements present in both bounded top-25 snapshots; interval brackets burst with collection padding", + "rows": [ + { + "queryid": 308564448692527448, + "calls": 79884, + "mean_ms": 1.5357204145260692, + "total_exec_ms": 122679.48959400051, + "block_reads": 0, + "temp_blocks_written": 0 + } + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-generator-summary.json b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-generator-summary.json new file mode 100644 index 000000000..6c0109a73 --- /dev/null +++ b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-generator-summary.json @@ -0,0 +1,219 @@ +{ + "sample_intervals": 17, + "scope": "Whole four-vCPU generator from /proc/stat and non-loopback network devices; includes monitoring overhead. CPU busy includes steal, reported separately. Does not alone rule out single-thread or network constraints.", + "intervals": [ + { + "at": 1793187965.3098376, + "cpu_busy_percent": 48.43096234309623, + "cpu_steal_percent": 0.10460251046025104, + "cpu_iowait_percent": 0.02615062761506276, + "cpu_softirq_percent": 15.42887029288703, + "received_gbps": 1.6929860905123988, + "transmitted_gbps": 0.021505235998962702, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793187975.3164468, + "cpu_busy_percent": 47.577672459189046, + "cpu_steal_percent": 0.13164823591363875, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 14.823591363875725, + "received_gbps": 1.7457258050510644, + "transmitted_gbps": 0.020390688183447327, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793187985.323938, + "cpu_busy_percent": 55.33351336753983, + "cpu_steal_percent": 0.10802052389954091, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 15.825006751282745, + "received_gbps": 2.1039461848002374, + "transmitted_gbps": 0.024811958684462503, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793187995.3315475, + "cpu_busy_percent": 58.25039287585123, + "cpu_steal_percent": 0.10476689366160294, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 18.517548454688317, + "received_gbps": 2.1650894506782015, + "transmitted_gbps": 0.025451647499584288, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793188005.3392386, + "cpu_busy_percent": 58.23725339482449, + "cpu_steal_percent": 0.07686395080707148, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 17.448116833205226, + "received_gbps": 2.114330425024324, + "transmitted_gbps": 0.02497318955794149, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793188015.3472104, + "cpu_busy_percent": 65.77743902439025, + "cpu_steal_percent": 0.07621951219512195, + "cpu_iowait_percent": 0.02540650406504065, + "cpu_softirq_percent": 16.666666666666668, + "received_gbps": 2.0341557735026012, + "transmitted_gbps": 0.028664751537677818, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793188025.3546264, + "cpu_busy_percent": 58.04375804375805, + "cpu_steal_percent": 0.10296010296010295, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 16.344916344916346, + "received_gbps": 2.1340850691993074, + "transmitted_gbps": 0.025632589246431012, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793188035.3633037, + "cpu_busy_percent": 56.71641791044776, + "cpu_steal_percent": 0.07720020586721564, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 16.881111682964487, + "received_gbps": 2.119102314376219, + "transmitted_gbps": 0.024406018301957613, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793188045.3709314, + "cpu_busy_percent": 65.87383660806618, + "cpu_steal_percent": 0.07755946225439504, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 15.046535677352637, + "received_gbps": 1.95991323036766, + "transmitted_gbps": 0.028167885509856487, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793188055.3780813, + "cpu_busy_percent": 53.23316340220016, + "cpu_steal_percent": 0.10732492621411323, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 16.31338878454521, + "received_gbps": 2.028581655748846, + "transmitted_gbps": 0.02494473028056717, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793188065.3861265, + "cpu_busy_percent": 50.64321344184825, + "cpu_steal_percent": 0.10501443948542925, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 16.907324757154107, + "received_gbps": 1.9135035168140258, + "transmitted_gbps": 0.022001711990297112, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793188075.3954966, + "cpu_busy_percent": 64.52017038336257, + "cpu_steal_percent": 0.12528188423953895, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 17.38912553244801, + "received_gbps": 2.100221093432269, + "transmitted_gbps": 0.027671641027249623, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793188085.4046028, + "cpu_busy_percent": 57.9446437724244, + "cpu_steal_percent": 0.12813941568426448, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 16.01742696053306, + "received_gbps": 2.129391066592682, + "transmitted_gbps": 0.024552101665329038, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793188095.410813, + "cpu_busy_percent": 67.4189364461738, + "cpu_steal_percent": 0.10376134889753567, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 16.60181582360571, + "received_gbps": 1.948257238917701, + "transmitted_gbps": 0.029266594887123017, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793188105.417789, + "cpu_busy_percent": 63.40479192938209, + "cpu_steal_percent": 0.15132408575031525, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 16.77175283732661, + "received_gbps": 2.036918116489781, + "transmitted_gbps": 0.027231082298692668, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793188115.4249136, + "cpu_busy_percent": 56.347080878414566, + "cpu_steal_percent": 0.16068559185859668, + "cpu_iowait_percent": 0.02678093197643278, + "cpu_softirq_percent": 16.47027316550616, + "received_gbps": 2.103577875780444, + "transmitted_gbps": 0.024966514601217932, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793188125.4338727, + "cpu_busy_percent": 57.44456177402323, + "cpu_steal_percent": 0.13199577613516367, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 17.68743400211193, + "received_gbps": 2.1654591204754396, + "transmitted_gbps": 0.024825402785236816, + "receive_drops": 0, + "transmit_drops": 0 + } + ], + "cpu_busy_percent": { + "mean": 57.95281223852895, + "max": 67.4189364461738 + }, + "cpu_steal_percent": { + "mean": 0.11019816860493517, + "max": 0.16068559185859668 + }, + "cpu_iowait_percent": { + "mean": 0.004608121391560953, + "max": 0.02678093197643278 + }, + "cpu_softirq_percent": { + "mean": 16.53770034888647, + "max": 18.517548454688317 + }, + "received_gbps": { + "mean": 2.0291320016331293, + "max": 2.1654591204754396 + }, + "transmitted_gbps": { + "mean": 0.025262573179766743, + "max": 0.029266594887123017 + }, + "receive_drops": 0, + "transmit_drops": 0 +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-runtime-integrity.json b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-runtime-integrity.json new file mode 100644 index 000000000..ecfa7cd57 --- /dev/null +++ b/tools/ingressbench/evidence/app-profile-2026-10-28/prepared-cpu-check-runtime-integrity.json @@ -0,0 +1,25 @@ +{ + "as_of": 1793188422.761034, + "samples": 16, + "commands_failed": [], + "app_restarts": { + "mcp-registry-685dff56b7-48qz9/mcp-registry": 0, + "mcp-registry-685dff56b7-tzs8j/mcp-registry": 0, + "registry-pg-1/postgres": 0 + }, + "node_count_timeline": [ + { + "at": 1793187820.076055, + "nodes": 4 + } + ], + "db_throttled_usec_range": [ + 886768, + 886768 + ], + "db_endpoint_samples": 86, + "db_endpoint_errors_or_empty": [], + "db_endpoint_not_ready": [], + "cadvisor_failures_or_empty": [], + "lb_collection_failures": [] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/app-pressure-summary.json b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/app-pressure-summary.json new file mode 100644 index 000000000..77b3fb0e2 --- /dev/null +++ b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/app-pressure-summary.json @@ -0,0 +1,98 @@ +[ + { + "pod": "mcp-registry-57d9d88679-52rg4", + "metric": "container_cpu_usage_seconds_total", + "seconds": 177.7170000076294, + "rate": 0.4621973980906368, + "start": 1793218057.983, + "end": 1793218235.7 + }, + { + "pod": "mcp-registry-57d9d88679-52rg4", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 177.7170000076294, + "rate": 0.15492261853856432, + "start": 1793218057.983, + "end": 1793218235.7 + }, + { + "pod": "mcp-registry-57d9d88679-52rg4", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 177.7170000076294, + "rate": 0.25454387029973485, + "start": 1793218057.983, + "end": 1793218235.7 + }, + { + "pod": "mcp-registry-57d9d88679-vfcxl", + "metric": "container_cpu_usage_seconds_total", + "seconds": 176.40199995040894, + "rate": 0.4234080170349387, + "start": 1793218056.57, + "end": 1793218232.972 + }, + { + "pod": "mcp-registry-57d9d88679-bzq4l", + "metric": "container_cpu_usage_seconds_total", + "seconds": 153.7590000629425, + "rate": 0.43019241782869694, + "start": 1793218057.166, + "end": 1793218210.925 + }, + { + "pod": "mcp-registry-57d9d88679-vfcxl", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 176.40199995040894, + "rate": 0.09445222279046714, + "start": 1793218056.57, + "end": 1793218232.972 + }, + { + "pod": "mcp-registry-57d9d88679-bzq4l", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 153.7590000629425, + "rate": 0.09256245159095655, + "start": 1793218057.166, + "end": 1793218210.925 + }, + { + "pod": "mcp-registry-57d9d88679-vfcxl", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 176.40199995040894, + "rate": 0.14315949369678024, + "start": 1793218056.57, + "end": 1793218232.972 + }, + { + "pod": "mcp-registry-57d9d88679-bzq4l", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 153.7590000629425, + "rate": 0.1405596744980965, + "start": 1793218057.166, + "end": 1793218210.925 + }, + { + "pod": "mcp-registry-57d9d88679-sjl52", + "metric": "container_cpu_usage_seconds_total", + "seconds": 125.23699998855591, + "rate": 0.45432400971916703, + "start": 1793218088.328, + "end": 1793218213.565 + }, + { + "pod": "mcp-registry-57d9d88679-sjl52", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 125.23699998855591, + "rate": 0.087224398548338, + "start": 1793218088.328, + "end": 1793218213.565 + }, + { + "pod": "mcp-registry-57d9d88679-sjl52", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 125.23699998855591, + "rate": 0.1408256266248123, + "start": 1793218088.328, + "end": 1793218213.565 + } +] \ No newline at end of file diff --git a/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/burst-gate.json b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/burst-gate.json new file mode 100644 index 000000000..0256710a9 --- /dev/null +++ b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/burst-gate.json @@ -0,0 +1,5 @@ +{ + "failed": true, + "full": false, + "shortened_recovery_after_failure": false +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/calibration.json b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/calibration.json new file mode 100644 index 000000000..ac2f3f719 --- /dev/null +++ b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/calibration.json @@ -0,0 +1,748 @@ +{ + "Calibration": true, + "RequestPhases": { + "page:HIT": { + "Attempts": 401490, + "NewConnections": 44, + "ReusedConnections": 401446, + "SchedulerLag": { + "Count": 401490, + "MeanMS": 0.22996455564023746, + "P95UpperMS": 1, + "MaximumMS": 55.17366 + }, + "QueueDelay": { + "Count": 401490, + "MeanMS": 0.04600408297590017, + "P95UpperMS": 0.5, + "MaximumMS": 39.421176 + }, + "ConnectionWait": { + "Count": 401490, + "MeanMS": 0.29622787783255594, + "P95UpperMS": 0.1, + "MaximumMS": 338.356314 + }, + "TLS": { + "Count": 347, + "MeanMS": 112.96794453602304, + "P95UpperMS": 500, + "MaximumMS": 379.708841 + }, + "FirstByte": { + "Count": 401490, + "MeanMS": 0.42713882190339497, + "P95UpperMS": 0.5, + "MaximumMS": 410.044744 + }, + "Body": { + "Count": 401490, + "MeanMS": 0.17294890748960576, + "P95UpperMS": 0.5, + "MaximumMS": 367.273187 + }, + "HTTP": { + "Count": 401490, + "MeanMS": 1.2300994108146983, + "P95UpperMS": 0.5, + "MaximumMS": 747.943929 + } + }, + "search:HIT": { + "Attempts": 270, + "NewConnections": 1, + "ReusedConnections": 269, + "SchedulerLag": { + "Count": 270, + "MeanMS": 0.32463417777777787, + "P95UpperMS": 1, + "MaximumMS": 30.326402 + }, + "QueueDelay": { + "Count": 270, + "MeanMS": 1.1163395185185192, + "P95UpperMS": 0.5, + "MaximumMS": 176.65662 + }, + "ConnectionWait": { + "Count": 270, + "MeanMS": 0.2274629296296295, + "P95UpperMS": 0.1, + "MaximumMS": 33.253349 + }, + "TLS": { + "Count": 1, + "MeanMS": 32.800994, + "P95UpperMS": 50, + "MaximumMS": 32.800994 + }, + "FirstByte": { + "Count": 270, + "MeanMS": 0.1951380888888889, + "P95UpperMS": 0.5, + "MaximumMS": 9.264996 + }, + "Body": { + "Count": 270, + "MeanMS": 0.1034079481481482, + "P95UpperMS": 0.5, + "MaximumMS": 0.986923 + }, + "HTTP": { + "Count": 270, + "MeanMS": 0.5650045999999997, + "P95UpperMS": 0.5, + "MaximumMS": 34.385388 + } + }, + "updated:HIT": { + "Attempts": 3240, + "NewConnections": 0, + "ReusedConnections": 3240, + "SchedulerLag": { + "Count": 3240, + "MeanMS": 0.23362741172839516, + "P95UpperMS": 2, + "MaximumMS": 50.933417 + }, + "QueueDelay": { + "Count": 3240, + "MeanMS": 0.1096629354938268, + "P95UpperMS": 0.5, + "MaximumMS": 186.334021 + }, + "ConnectionWait": { + "Count": 3240, + "MeanMS": 0.25702168024691385, + "P95UpperMS": 0.1, + "MaximumMS": 92.866368 + }, + "TLS": { + "Count": 4, + "MeanMS": 144.77077975, + "P95UpperMS": 500, + "MaximumMS": 225.540554 + }, + "FirstByte": { + "Count": 3240, + "MeanMS": 0.39996171141975567, + "P95UpperMS": 0.5, + "MaximumMS": 180.315982 + }, + "Body": { + "Count": 3240, + "MeanMS": 0.34331726296296333, + "P95UpperMS": 0.5, + "MaximumMS": 366.639093 + }, + "HTTP": { + "Count": 3240, + "MeanMS": 1.1452019333333325, + "P95UpperMS": 0.5, + "MaximumMS": 526.472371 + } + } + }, + "StartedAt": "2026-10-28T19:58:02.560258316Z", + "FinishedAt": "2026-10-28T19:59:32.561104489Z", + "CatalogSHA256": "120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be", + "ReadRate": 4500, + "UpdatedRate": 36, + "SearchRate": 3, + "HotPercent": 80, + "Scheduled": 405000, + "ScheduledPublishes": 0, + "Completed": 405000, + "Dropped": 0, + "DroppedPublishes": 0, + "Bytes": 26542080000, + "Status": { + "page:200": 401490, + "search:200": 270, + "updated:200": 3240 + }, + "Cache": { + "HIT": 405000 + }, + "Errors": {}, + "Protocols": { + "HTTP/1.1": 405000 + }, + "ErrorDetails": {}, + "LatencyMS": { + "page_p50": 0.309116, + "page_p95": 1.4322, + "page_p99": 2.428981, + "search_p50": 0.326879, + "search_p95": 1.462516, + "search_p99": 67.132465, + "updated_p50": 0.31745, + "updated_p95": 1.44725, + "updated_p99": 1.889981 + }, + "ElapsedSeconds": 90.000846042, + "Failures": null, + "OmittedFailures": 0, + "Windows": [ + { + "StartedAt": "2026-10-28T19:58:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 33479, + "Dropped": 0, + "Dispatched": 33478, + "Completed": 33478, + "Failures": 0, + "SchedulerLag": { + "Count": 33478, + "MeanMS": 1.0779281501881681, + "P95UpperMS": 2, + "MaximumMS": 55.17366 + }, + "QueueDelay": { + "Count": 33478, + "MeanMS": 0.34425650878187253, + "P95UpperMS": 1, + "MaximumMS": 186.334021 + }, + "HTTPDuration": { + "Count": 33478, + "MeanMS": 11.381366567238164, + "P95UpperMS": 100, + "MaximumMS": 747.943929 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T19:58:10Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45001, + "Completed": 45000, + "Failures": 0, + "SchedulerLag": { + "Count": 45001, + "MeanMS": 0.18862627843825852, + "P95UpperMS": 2, + "MaximumMS": 4.717959 + }, + "QueueDelay": { + "Count": 45001, + "MeanMS": 0.02182147696717918, + "P95UpperMS": 0.5, + "MaximumMS": 0.911863 + }, + "HTTPDuration": { + "Count": 45001, + "MeanMS": 0.32112115339659003, + "P95UpperMS": 0.5, + "MaximumMS": 24.337408 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T19:58:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 44998, + "Completed": 44999, + "Failures": 0, + "SchedulerLag": { + "Count": 44998, + "MeanMS": 0.14986084925996665, + "P95UpperMS": 1, + "MaximumMS": 1.926188 + }, + "QueueDelay": { + "Count": 44998, + "MeanMS": 0.019852251122272725, + "P95UpperMS": 0.5, + "MaximumMS": 2.273629 + }, + "HTTPDuration": { + "Count": 44998, + "MeanMS": 0.29555208120360876, + "P95UpperMS": 0.5, + "MaximumMS": 5.734464 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T19:58:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 44998, + "Completed": 44997, + "Failures": 0, + "SchedulerLag": { + "Count": 45002, + "MeanMS": 0.15932558992933507, + "P95UpperMS": 1, + "MaximumMS": 6.444069 + }, + "QueueDelay": { + "Count": 44998, + "MeanMS": 0.024078025445576436, + "P95UpperMS": 0.5, + "MaximumMS": 9.019504 + }, + "HTTPDuration": { + "Count": 44998, + "MeanMS": 0.3990117717898601, + "P95UpperMS": 0.5, + "MaximumMS": 53.907007 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T19:58:40Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45000, + "Completed": 45001, + "Failures": 0, + "SchedulerLag": { + "Count": 44996, + "MeanMS": 0.15217630993866058, + "P95UpperMS": 1, + "MaximumMS": 2.342529 + }, + "QueueDelay": { + "Count": 45000, + "MeanMS": 0.019667756444445128, + "P95UpperMS": 0.5, + "MaximumMS": 2.625167 + }, + "HTTPDuration": { + "Count": 45000, + "MeanMS": 0.2934834888444427, + "P95UpperMS": 0.5, + "MaximumMS": 4.305302 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T19:58:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45004, + "Completed": 45003, + "Failures": 0, + "SchedulerLag": { + "Count": 45004, + "MeanMS": 0.1389170924806686, + "P95UpperMS": 1, + "MaximumMS": 1.251191 + }, + "QueueDelay": { + "Count": 45004, + "MeanMS": 0.018886398897876334, + "P95UpperMS": 0.5, + "MaximumMS": 0.978465 + }, + "HTTPDuration": { + "Count": 45004, + "MeanMS": 0.3015269344724923, + "P95UpperMS": 0.5, + "MaximumMS": 4.363098 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T19:59:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 44997, + "Completed": 44998, + "Failures": 0, + "SchedulerLag": { + "Count": 44997, + "MeanMS": 0.14478993292886197, + "P95UpperMS": 1, + "MaximumMS": 1.990875 + }, + "QueueDelay": { + "Count": 44997, + "MeanMS": 0.019506986843567893, + "P95UpperMS": 0.5, + "MaximumMS": 1.635128 + }, + "HTTPDuration": { + "Count": 44997, + "MeanMS": 0.3047035829277475, + "P95UpperMS": 0.5, + "MaximumMS": 6.826555 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T19:59:10Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45003, + "Completed": 45002, + "Failures": 0, + "SchedulerLag": { + "Count": 45003, + "MeanMS": 0.1460213479545802, + "P95UpperMS": 1, + "MaximumMS": 1.212579 + }, + "QueueDelay": { + "Count": 45003, + "MeanMS": 0.019933657045086556, + "P95UpperMS": 0.5, + "MaximumMS": 0.812342 + }, + "HTTPDuration": { + "Count": 45003, + "MeanMS": 0.30058144559251365, + "P95UpperMS": 0.5, + "MaximumMS": 3.775121 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T19:59:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 44997, + "Completed": 44998, + "Failures": 0, + "SchedulerLag": { + "Count": 44997, + "MeanMS": 0.15061702433495475, + "P95UpperMS": 1, + "MaximumMS": 1.659743 + }, + "QueueDelay": { + "Count": 44997, + "MeanMS": 0.020106356045959322, + "P95UpperMS": 0.5, + "MaximumMS": 1.587554 + }, + "HTTPDuration": { + "Count": 44997, + "MeanMS": 0.3012109433073299, + "P95UpperMS": 0.5, + "MaximumMS": 6.077487 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T19:59:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 11521, + "Dropped": 0, + "Dispatched": 11524, + "Completed": 11524, + "Failures": 0, + "SchedulerLag": { + "Count": 11524, + "MeanMS": 0.14937863372092938, + "P95UpperMS": 1, + "MaximumMS": 1.180099 + }, + "QueueDelay": { + "Count": 11524, + "MeanMS": 0.01983235768830254, + "P95UpperMS": 0.5, + "MaximumMS": 0.760641 + }, + "HTTPDuration": { + "Count": 11524, + "MeanMS": 0.2982812257028805, + "P95UpperMS": 0.5, + "MaximumMS": 4.778751 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + } + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/checkout-summary.json b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/checkout-summary.json new file mode 100644 index 000000000..67edec74c --- /dev/null +++ b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/checkout-summary.json @@ -0,0 +1,498 @@ +{ + "mcp-registry-57d9d88679-52rg4": { + "first": "burst-000-mcp-registry-57d9d88679-52rg4.prom", + "last": "burst-180-mcp-registry-57d9d88679-52rg4.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 19832.0, + "sum": 2671.182599758981, + "mean_ms": 134.69053044367593 + }, + "mcp_registry_db_pool_acquire_duration/list": { + "count": 33.0, + "sum": 2.5535394880000015, + "mean_ms": 77.37998448484853 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 146.0, + "sum": 8.752614036000002, + "mean_ms": 59.94941120547947 + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 1644.0, + "sum": 133.9088617309998, + "mean_ms": 81.45307891180038 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 19832.0, + "sum": 317.3671917570001, + "mean_ms": 16.002782964753937 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 33.0, + "sum": 0.34554335599999997, + "mean_ms": 10.471010787878786 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 146.0, + "sum": 10.119083722, + "mean_ms": 69.30879261643835 + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 1644.0, + "sum": 18.084914997999995, + "mean_ms": 11.000556568126518 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 19832.0, + "sum": 1.5336187330000004, + "mean_ms": 0.0773305129588544 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 33.0, + "sum": 0.0027124119999999995, + "mean_ms": 0.08219430303030302 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 146.0, + "sum": 0.004654085000000001, + "mean_ms": 0.03187729452054795 + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 1644.0, + "sum": 0.2870319899999992, + "mean_ms": 0.17459366788321115 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_page": { + "count": 19832.0, + "sum": 302.85224179599925, + "mean_ms": 15.27088754517947 + }, + "mcp_registry_db_query_duration/list": { + "count": 33.0, + "sum": 0.34177790199999997, + "mean_ms": 10.35690612121212 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 146.0, + "sum": 9.988774406000008, + "mean_ms": 68.41626305479457 + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 1644.0, + "sum": 17.224061519000028, + "mean_ms": 10.476923065085176 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 2.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 30.0 + } + }, + "payload_sources": { + "prepared": 1538987.0, + "fallback": 0.0 + } + }, + "mcp-registry-57d9d88679-bzq4l": { + "first": "burst-000-mcp-registry-57d9d88679-bzq4l.prom", + "last": "burst-180-mcp-registry-57d9d88679-bzq4l.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 19962.0, + "sum": 1302.3936863810318, + "mean_ms": 65.24364724882436 + }, + "mcp_registry_db_pool_acquire_duration/list": { + "count": 20.0, + "sum": 1.7993743130000002, + "mean_ms": 89.96871565000001 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 120.0, + "sum": 4.168257690999997, + "mean_ms": 34.73548075833331 + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 1570.0, + "sum": 48.28134439399995, + "mean_ms": 30.752448658598695 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 19961.0, + "sum": 256.2248084679975, + "mean_ms": 12.83627115214656 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 20.0, + "sum": 0.253953905, + "mean_ms": 12.697695249999999 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 120.0, + "sum": 8.616490450000002, + "mean_ms": 71.80408708333336 + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 1569.0, + "sum": 15.901171703000005, + "mean_ms": 10.134589995538564 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 19961.0, + "sum": 0.7858065599999962, + "mean_ms": 0.039367093832974105 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 20.0, + "sum": 0.000724497, + "mean_ms": 0.03622485 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 120.0, + "sum": 0.010087658999999999, + "mean_ms": 0.084063825 + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 1569.0, + "sum": 0.058483850999999976, + "mean_ms": 0.03727460229445505 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_page": { + "count": 19961.0, + "sum": 253.00418001799972, + "mean_ms": 12.674925104854452 + }, + "mcp_registry_db_query_duration/list": { + "count": 20.0, + "sum": 0.25274347199999997, + "mean_ms": 12.637173599999999 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 120.0, + "sum": 8.604589283000003, + "mean_ms": 71.7049106916667 + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 1569.0, + "sum": 15.767338592000018, + "mean_ms": 10.049291645634174 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 4.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 30.0 + } + }, + "payload_sources": { + "prepared": 1548275.0, + "fallback": 0.0 + } + }, + "mcp-registry-57d9d88679-sjl52": { + "first": "burst-000-mcp-registry-57d9d88679-sjl52.prom", + "last": "burst-180-mcp-registry-57d9d88679-sjl52.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 19979.0, + "sum": 577.1493973780033, + "mean_ms": 28.887802061064285 + }, + "mcp_registry_db_pool_acquire_duration/list": { + "count": 21.0, + "sum": 1.575979518, + "mean_ms": 75.04664371428572 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 128.0, + "sum": 3.356164981000001, + "mean_ms": 26.220038914062506 + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 1524.0, + "sum": 22.921456378000062, + "mean_ms": 15.04032570734912 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 19979.0, + "sum": 255.0566578960003, + "mean_ms": 12.766237444116337 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 21.0, + "sum": 0.4122976100000001, + "mean_ms": 19.633219523809526 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 128.0, + "sum": 7.195824071, + "mean_ms": 56.2173755546875 + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 1524.0, + "sum": 15.797473160999983, + "mean_ms": 10.365796037401564 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 19979.0, + "sum": 0.9531120689999997, + "mean_ms": 0.04770569442915059 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 21.0, + "sum": 0.000548634, + "mean_ms": 0.02612542857142857 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 128.0, + "sum": 0.006231186, + "mean_ms": 0.048681140624999994 + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 1524.0, + "sum": 0.12462113899999966, + "mean_ms": 0.08177240091863494 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_page": { + "count": 19979.0, + "sum": 251.72876724999912, + "mean_ms": 12.599668013914567 + }, + "mcp_registry_db_query_duration/list": { + "count": 21.0, + "sum": 0.40919799, + "mean_ms": 19.48561857142857 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 128.0, + "sum": 7.185343052000003, + "mean_ms": 56.13549259375002 + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 1524.0, + "sum": 15.623433114999974, + "mean_ms": 10.251596532152215 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 3.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 30.0 + } + }, + "payload_sources": { + "prepared": 1543768.0, + "fallback": 0.0 + } + }, + "mcp-registry-57d9d88679-vfcxl": { + "first": "burst-000-mcp-registry-57d9d88679-vfcxl.prom", + "last": "burst-180-mcp-registry-57d9d88679-vfcxl.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 19930.0, + "sum": 1093.5004873270102, + "mean_ms": 54.86705907310638 + }, + "mcp_registry_db_pool_acquire_duration/list": { + "count": 23.0, + "sum": 0.372437761, + "mean_ms": 16.19294613043478 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 134.0, + "sum": 3.6421931099999982, + "mean_ms": 27.180545597014913 + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 1586.0, + "sum": 64.97912822000038, + "mean_ms": 40.97044654476694 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 19930.0, + "sum": 247.1292158339992, + "mean_ms": 12.39986030275962 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 23.0, + "sum": 0.12974925, + "mean_ms": 5.641271739130436 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 134.0, + "sum": 10.597778951, + "mean_ms": 79.08790261940298 + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 1586.0, + "sum": 15.962422908000002, + "mean_ms": 10.064579387137455 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 19930.0, + "sum": 0.7841181170000013, + "mean_ms": 0.039343608479678945 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 23.0, + "sum": 0.007150085, + "mean_ms": 0.3108732608695652 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 134.0, + "sum": 0.002950665, + "mean_ms": 0.02201988805970149 + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 1586.0, + "sum": 0.07006116999999991, + "mean_ms": 0.044174760403530834 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_page": { + "count": 19930.0, + "sum": 244.76379579400017, + "mean_ms": 12.281173898344214 + }, + "mcp_registry_db_query_duration/list": { + "count": 23.0, + "sum": 0.119770133, + "mean_ms": 5.207397086956521 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 134.0, + "sum": 10.571343098000002, + "mean_ms": 78.89062013432837 + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 1586.0, + "sum": 15.792451620000007, + "mean_ms": 9.957409596469109 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 2.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 30.0 + } + }, + "payload_sources": { + "prepared": 1550656.0, + "fallback": 0.0 + } + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/cleanup.json b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/cleanup.json new file mode 100644 index 000000000..1cc148024 --- /dev/null +++ b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/cleanup.json @@ -0,0 +1,22 @@ +{ + "at": 1793218621.450293, + "production_changed": false, + "instances_absent": true, + "disks_absent": true, + "app_spec_restored": true, + "ingress_hpa_restored": true, + "db_resources_restored": true, + "whitelist_restored": true, + "verified_at": "2026-10-28T20:22:38.369288+00:00", + "response_hashes_matched": 48, + "corpus": { + "names": 52366, + "versions": 187108, + "benchmark_runs": null + }, + "final_snapshot_at": 1793218899.924099, + "observed_nodes": 4, + "observed_app_pods": 2, + "observed_ingress_pods": 5, + "natural_two_node_recovery_observed": false +} diff --git a/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/coordinated-burst.json b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/coordinated-burst.json new file mode 100644 index 000000000..86467b464 --- /dev/null +++ b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/coordinated-burst.json @@ -0,0 +1,1443 @@ +{ + "Calibration": false, + "RequestPhases": { + "page:HIT": { + "Attempts": 713019, + "NewConnections": 1697, + "ReusedConnections": 711322, + "SchedulerLag": { + "Count": 713019, + "MeanMS": 0.08165591053394265, + "P95UpperMS": 0.5, + "MaximumMS": 38.428123 + }, + "QueueDelay": { + "Count": 713019, + "MeanMS": 14.179049499670523, + "P95UpperMS": 200, + "MaximumMS": 500.658352 + }, + "ConnectionWait": { + "Count": 713019, + "MeanMS": 2.0874282099493815, + "P95UpperMS": 0.1, + "MaximumMS": 363.722087 + }, + "TLS": { + "Count": 11757, + "MeanMS": 219.06712238292099, + "P95UpperMS": 500, + "MaximumMS": 1186.296307 + }, + "FirstByte": { + "Count": 713019, + "MeanMS": 22.03630743267993, + "P95UpperMS": 100, + "MaximumMS": 8620.139249 + }, + "Body": { + "Count": 713019, + "MeanMS": 5.835497210718153, + "P95UpperMS": 50, + "MaximumMS": 1285.787538 + }, + "HTTP": { + "Count": 713019, + "MeanMS": 30.002589903457505, + "P95UpperMS": 200, + "MaximumMS": 8646.353863 + } + }, + "page:MISS": { + "Attempts": 20940, + "NewConnections": 189, + "ReusedConnections": 20751, + "SchedulerLag": { + "Count": 20940, + "MeanMS": 0.16615290090735468, + "P95UpperMS": 0.5, + "MaximumMS": 33.460068 + }, + "QueueDelay": { + "Count": 20940, + "MeanMS": 45.81554350864362, + "P95UpperMS": 500, + "MaximumMS": 498.99328 + }, + "ConnectionWait": { + "Count": 20940, + "MeanMS": 3.65680584030566, + "P95UpperMS": 0.1, + "MaximumMS": 367.156201 + }, + "TLS": { + "Count": 465, + "MeanMS": 218.23922643225805, + "P95UpperMS": 500, + "MaximumMS": 1140.91082 + }, + "FirstByte": { + "Count": 20940, + "MeanMS": 320.5407171165719, + "P95UpperMS": 2000, + "MaximumMS": 9489.101932 + }, + "Body": { + "Count": 20940, + "MeanMS": 25.152635043361883, + "P95UpperMS": 200, + "MaximumMS": 2242.613668 + }, + "HTTP": { + "Count": 20940, + "MeanMS": 349.4498975372504, + "P95UpperMS": 2000, + "MaximumMS": 9490.291434 + } + }, + "page:STALE": { + "Attempts": 60063, + "NewConnections": 360, + "ReusedConnections": 59703, + "SchedulerLag": { + "Count": 60063, + "MeanMS": 0.1247562418627108, + "P95UpperMS": 0.5, + "MaximumMS": 38.214001 + }, + "QueueDelay": { + "Count": 60063, + "MeanMS": 11.90878991647098, + "P95UpperMS": 100, + "MaximumMS": 500.063221 + }, + "ConnectionWait": { + "Count": 60063, + "MeanMS": 3.2106080271381314, + "P95UpperMS": 0.1, + "MaximumMS": 375.070147 + }, + "TLS": { + "Count": 1287, + "MeanMS": 222.23646338616945, + "P95UpperMS": 500, + "MaximumMS": 1025.934819 + }, + "FirstByte": { + "Count": 60063, + "MeanMS": 24.712181939147296, + "P95UpperMS": 100, + "MaximumMS": 6518.357436 + }, + "Body": { + "Count": 60063, + "MeanMS": 9.542081970397795, + "P95UpperMS": 50, + "MaximumMS": 1144.227699 + }, + "HTTP": { + "Count": 60063, + "MeanMS": 37.53880525358358, + "P95UpperMS": 200, + "MaximumMS": 6522.756772 + } + }, + "page:UPDATING": { + "Attempts": 1608, + "NewConnections": 115, + "ReusedConnections": 1493, + "SchedulerLag": { + "Count": 1608, + "MeanMS": 1.1425484800995016, + "P95UpperMS": 10, + "MaximumMS": 37.551074 + }, + "QueueDelay": { + "Count": 1608, + "MeanMS": 157.02670031529837, + "P95UpperMS": 500, + "MaximumMS": 499.916481 + }, + "ConnectionWait": { + "Count": 1608, + "MeanMS": 24.31713172512432, + "P95UpperMS": 500, + "MaximumMS": 365.89721 + }, + "TLS": { + "Count": 173, + "MeanMS": 252.75200379190755, + "P95UpperMS": 1000, + "MaximumMS": 1144.799715 + }, + "FirstByte": { + "Count": 1608, + "MeanMS": 214.40058556343286, + "P95UpperMS": 2000, + "MaximumMS": 1761.247918 + }, + "Body": { + "Count": 1608, + "MeanMS": 32.612557202736305, + "P95UpperMS": 200, + "MaximumMS": 1148.997804 + }, + "HTTP": { + "Count": 1608, + "MeanMS": 272.06397558582097, + "P95UpperMS": 2000, + "MaximumMS": 2331.182621 + } + }, + "search:MISS": { + "Attempts": 440, + "NewConnections": 3, + "ReusedConnections": 437, + "SchedulerLag": { + "Count": 440, + "MeanMS": 0.22489134772727298, + "P95UpperMS": 0.5, + "MaximumMS": 30.681556 + }, + "QueueDelay": { + "Count": 440, + "MeanMS": 13.446583372727266, + "P95UpperMS": 200, + "MaximumMS": 287.786535 + }, + "ConnectionWait": { + "Count": 440, + "MeanMS": 2.726831254545454, + "P95UpperMS": 0.1, + "MaximumMS": 299.611331 + }, + "TLS": { + "Count": 11, + "MeanMS": 124.13990518181818, + "P95UpperMS": 500, + "MaximumMS": 225.469896 + }, + "FirstByte": { + "Count": 440, + "MeanMS": 158.41204148181816, + "P95UpperMS": 1000, + "MaximumMS": 3912.521785 + }, + "Body": { + "Count": 440, + "MeanMS": 21.26126894318182, + "P95UpperMS": 100, + "MaximumMS": 593.724093 + }, + "HTTP": { + "Count": 440, + "MeanMS": 182.43670651363655, + "P95UpperMS": 1000, + "MaximumMS": 3914.574248 + } + }, + "search:STALE": { + "Attempts": 94, + "NewConnections": 1, + "ReusedConnections": 93, + "SchedulerLag": { + "Count": 94, + "MeanMS": 0.06727985106382978, + "P95UpperMS": 0.5, + "MaximumMS": 0.515271 + }, + "QueueDelay": { + "Count": 94, + "MeanMS": 9.664432319148942, + "P95UpperMS": 100, + "MaximumMS": 263.12365 + }, + "ConnectionWait": { + "Count": 94, + "MeanMS": 2.936747265957447, + "P95UpperMS": 0.5, + "MaximumMS": 95.40282 + }, + "TLS": { + "Count": 4, + "MeanMS": 114.41284200000001, + "P95UpperMS": 200, + "MaximumMS": 179.006269 + }, + "FirstByte": { + "Count": 94, + "MeanMS": 31.677760308510646, + "P95UpperMS": 200, + "MaximumMS": 1022.797171 + }, + "Body": { + "Count": 94, + "MeanMS": 14.886364499999996, + "P95UpperMS": 50, + "MaximumMS": 474.202774 + }, + "HTTP": { + "Count": 94, + "MeanMS": 49.535109606382996, + "P95UpperMS": 500, + "MaximumMS": 1592.437335 + } + }, + "updated:MISS": { + "Attempts": 6418, + "NewConnections": 8, + "ReusedConnections": 6410, + "SchedulerLag": { + "Count": 6418, + "MeanMS": 0.08339560034278587, + "P95UpperMS": 0.5, + "MaximumMS": 27.487366 + }, + "QueueDelay": { + "Count": 6418, + "MeanMS": 15.209535078996531, + "P95UpperMS": 200, + "MaximumMS": 499.77501 + }, + "ConnectionWait": { + "Count": 6418, + "MeanMS": 2.2989985456528483, + "P95UpperMS": 0.1, + "MaximumMS": 338.469506 + }, + "TLS": { + "Count": 110, + "MeanMS": 239.6773747727274, + "P95UpperMS": 1000, + "MaximumMS": 605.159848 + }, + "FirstByte": { + "Count": 6418, + "MeanMS": 87.33164471751336, + "P95UpperMS": 500, + "MaximumMS": 8459.744335 + }, + "Body": { + "Count": 6418, + "MeanMS": 11.785305485197837, + "P95UpperMS": 100, + "MaximumMS": 1075.255672 + }, + "HTTP": { + "Count": 6418, + "MeanMS": 101.45203425568647, + "P95UpperMS": 1000, + "MaximumMS": 8798.395121 + } + } + }, + "StartedAt": "2026-10-28T20:07:36.167225061Z", + "FinishedAt": "2026-10-28T20:10:36.195036054Z", + "CatalogSHA256": "120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be", + "ReadRate": 4500, + "UpdatedRate": 36, + "SearchRate": 3, + "HotPercent": 80, + "Scheduled": 810000, + "ScheduledPublishes": 0, + "Completed": 802582, + "Dropped": 7418, + "DroppedPublishes": 0, + "Bytes": 46019037264, + "Status": { + "page:200": 795630, + "search:200": 534, + "updated:200": 6418 + }, + "Cache": { + "HIT": 713019, + "MISS": 27798, + "STALE": 60157, + "UPDATING": 1608 + }, + "Errors": {}, + "Protocols": { + "HTTP/1.1": 802582 + }, + "ErrorDetails": {}, + "LatencyMS": { + "page_p50": 5.297902, + "page_p95": 315.829021, + "page_p99": 805.774962, + "search_p50": 44.51659, + "search_p95": 835.071745, + "search_p99": 1902.086433, + "updated_p50": 17.625467, + "updated_p95": 664.151381, + "updated_p99": 1832.257212 + }, + "ElapsedSeconds": 180.027810793, + "Failures": null, + "OmittedFailures": 0, + "Windows": [ + { + "StartedAt": "2026-10-28T20:07:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 17248, + "Dropped": 2285, + "Dispatched": 13940, + "Completed": 12916, + "Failures": 0, + "SchedulerLag": { + "Count": 17248, + "MeanMS": 0.8442905687615965, + "P95UpperMS": 2, + "MaximumMS": 38.428123 + }, + "QueueDelay": { + "Count": 13940, + "MeanMS": 213.6809692517938, + "P95UpperMS": 500, + "MaximumMS": 500.658352 + }, + "HTTPDuration": { + "Count": 13940, + "MeanMS": 326.3681287040183, + "P95UpperMS": 2000, + "MaximumMS": 9490.291434 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:07:40Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 419, + "Dispatched": 45604, + "Completed": 46556, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.06605980668888842, + "P95UpperMS": 0.5, + "MaximumMS": 1.028886 + }, + "QueueDelay": { + "Count": 45604, + "MeanMS": 43.270668228247494, + "P95UpperMS": 500, + "MaximumMS": 276.572613 + }, + "HTTPDuration": { + "Count": 45604, + "MeanMS": 113.89658250609597, + "P95UpperMS": 1000, + "MaximumMS": 4094.587666 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:07:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45000, + "Completed": 44997, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.07658721413333357, + "P95UpperMS": 0.5, + "MaximumMS": 1.429974 + }, + "QueueDelay": { + "Count": 45000, + "MeanMS": 0.008872631644444268, + "P95UpperMS": 0.1, + "MaximumMS": 2.855264 + }, + "HTTPDuration": { + "Count": 45000, + "MeanMS": 13.439471541155665, + "P95UpperMS": 100, + "MaximumMS": 817.834476 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:08:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 941, + "Dispatched": 44059, + "Completed": 44107, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.08420232242222185, + "P95UpperMS": 0.5, + "MaximumMS": 3.968565 + }, + "QueueDelay": { + "Count": 44059, + "MeanMS": 28.79822917292725, + "P95UpperMS": 500, + "MaximumMS": 345.787359 + }, + "HTTPDuration": { + "Count": 44059, + "MeanMS": 64.3199040288928, + "P95UpperMS": 500, + "MaximumMS": 1581.724261 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:08:10Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45000, + "Completed": 44996, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.07061820126666661, + "P95UpperMS": 0.5, + "MaximumMS": 1.146597 + }, + "QueueDelay": { + "Count": 45000, + "MeanMS": 0.007772964688888865, + "P95UpperMS": 0.1, + "MaximumMS": 0.502872 + }, + "HTTPDuration": { + "Count": 45000, + "MeanMS": 7.378679990466675, + "P95UpperMS": 50, + "MaximumMS": 324.229158 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:08:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 243, + "Dispatched": 44756, + "Completed": 44734, + "Failures": 0, + "SchedulerLag": { + "Count": 44999, + "MeanMS": 0.07498152281161895, + "P95UpperMS": 0.5, + "MaximumMS": 3.848979 + }, + "QueueDelay": { + "Count": 44756, + "MeanMS": 13.869595809701352, + "P95UpperMS": 200, + "MaximumMS": 281.46234 + }, + "HTTPDuration": { + "Count": 44756, + "MeanMS": 39.23124791905033, + "P95UpperMS": 200, + "MaximumMS": 1524.455845 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:08:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 377, + "Dispatched": 44624, + "Completed": 44605, + "Failures": 0, + "SchedulerLag": { + "Count": 45001, + "MeanMS": 0.07314234172573907, + "P95UpperMS": 0.5, + "MaximumMS": 5.95595 + }, + "QueueDelay": { + "Count": 44624, + "MeanMS": 14.992814879593919, + "P95UpperMS": 200, + "MaximumMS": 311.183902 + }, + "HTTPDuration": { + "Count": 44624, + "MeanMS": 43.227432091408325, + "P95UpperMS": 200, + "MaximumMS": 1478.304219 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:08:40Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45000, + "Completed": 44991, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.0702780376888896, + "P95UpperMS": 0.5, + "MaximumMS": 1.228185 + }, + "QueueDelay": { + "Count": 45000, + "MeanMS": 0.007601221888888889, + "P95UpperMS": 0.1, + "MaximumMS": 0.386872 + }, + "HTTPDuration": { + "Count": 45000, + "MeanMS": 18.173573111555605, + "P95UpperMS": 100, + "MaximumMS": 275.704091 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:08:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 359, + "Dispatched": 44641, + "Completed": 44662, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.07413656068888907, + "P95UpperMS": 0.5, + "MaximumMS": 3.946289 + }, + "QueueDelay": { + "Count": 44641, + "MeanMS": 13.214562427364903, + "P95UpperMS": 200, + "MaximumMS": 307.350948 + }, + "HTTPDuration": { + "Count": 44641, + "MeanMS": 42.23357691380151, + "P95UpperMS": 200, + "MaximumMS": 1363.650928 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:09:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 881, + "Dispatched": 44119, + "Completed": 44132, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.07690622235555568, + "P95UpperMS": 0.5, + "MaximumMS": 4.579564 + }, + "QueueDelay": { + "Count": 44119, + "MeanMS": 17.203704341938828, + "P95UpperMS": 200, + "MaximumMS": 423.248184 + }, + "HTTPDuration": { + "Count": 44119, + "MeanMS": 45.98887099965999, + "P95UpperMS": 200, + "MaximumMS": 1669.281724 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:09:10Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45000, + "Completed": 45014, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.07272572515555537, + "P95UpperMS": 0.5, + "MaximumMS": 1.25905 + }, + "QueueDelay": { + "Count": 45000, + "MeanMS": 0.008139775044444422, + "P95UpperMS": 0.1, + "MaximumMS": 0.627202 + }, + "HTTPDuration": { + "Count": 45000, + "MeanMS": 9.735137489666675, + "P95UpperMS": 50, + "MaximumMS": 268.873626 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:09:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 281, + "Dispatched": 44719, + "Completed": 44707, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.0758642277777782, + "P95UpperMS": 0.5, + "MaximumMS": 2.431008 + }, + "QueueDelay": { + "Count": 44719, + "MeanMS": 15.92773623075211, + "P95UpperMS": 200, + "MaximumMS": 289.938135 + }, + "HTTPDuration": { + "Count": 44719, + "MeanMS": 39.33668516444928, + "P95UpperMS": 200, + "MaximumMS": 1428.411607 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:09:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 614, + "Dispatched": 44386, + "Completed": 44374, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.0737808785777776, + "P95UpperMS": 0.5, + "MaximumMS": 2.006806 + }, + "QueueDelay": { + "Count": 44386, + "MeanMS": 14.915392924525657, + "P95UpperMS": 200, + "MaximumMS": 363.807736 + }, + "HTTPDuration": { + "Count": 44386, + "MeanMS": 37.68804816273169, + "P95UpperMS": 200, + "MaximumMS": 1348.369669 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:09:40Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45000, + "Completed": 45022, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.0682038351555557, + "P95UpperMS": 0.5, + "MaximumMS": 1.392962 + }, + "QueueDelay": { + "Count": 45000, + "MeanMS": 0.007523938599999925, + "P95UpperMS": 0.1, + "MaximumMS": 1.192816 + }, + "HTTPDuration": { + "Count": 45000, + "MeanMS": 11.525656405044534, + "P95UpperMS": 50, + "MaximumMS": 272.211511 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:09:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 320, + "Dispatched": 44680, + "Completed": 44674, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.07506908902222227, + "P95UpperMS": 0.5, + "MaximumMS": 3.486372 + }, + "QueueDelay": { + "Count": 44680, + "MeanMS": 14.224477692905031, + "P95UpperMS": 200, + "MaximumMS": 298.652008 + }, + "HTTPDuration": { + "Count": 44680, + "MeanMS": 39.059328822426274, + "P95UpperMS": 200, + "MaximumMS": 1504.105656 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:10:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 478, + "Dispatched": 44522, + "Completed": 44498, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.07368747024444457, + "P95UpperMS": 0.5, + "MaximumMS": 2.741353 + }, + "QueueDelay": { + "Count": 44522, + "MeanMS": 16.104277310138766, + "P95UpperMS": 200, + "MaximumMS": 333.680535 + }, + "HTTPDuration": { + "Count": 44522, + "MeanMS": 40.50615853126551, + "P95UpperMS": 200, + "MaximumMS": 1758.621662 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:10:10Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45000, + "Completed": 45023, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.07080246935555491, + "P95UpperMS": 0.5, + "MaximumMS": 1.1586 + }, + "QueueDelay": { + "Count": 45000, + "MeanMS": 0.007914696644444375, + "P95UpperMS": 0.1, + "MaximumMS": 0.381272 + }, + "HTTPDuration": { + "Count": 45000, + "MeanMS": 10.656980107622111, + "P95UpperMS": 50, + "MaximumMS": 331.642233 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:10:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 220, + "Dispatched": 44780, + "Completed": 44773, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.07588294555555608, + "P95UpperMS": 0.5, + "MaximumMS": 8.613503 + }, + "QueueDelay": { + "Count": 44780, + "MeanMS": 12.343762833452386, + "P95UpperMS": 200, + "MaximumMS": 276.234499 + }, + "HTTPDuration": { + "Count": 44780, + "MeanMS": 34.82458980757042, + "P95UpperMS": 200, + "MaximumMS": 1386.266418 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:10:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 27752, + "Dropped": 0, + "Dispatched": 27752, + "Completed": 27801, + "Failures": 0, + "SchedulerLag": { + "Count": 27752, + "MeanMS": 0.06893410056932853, + "P95UpperMS": 0.5, + "MaximumMS": 1.874886 + }, + "QueueDelay": { + "Count": 27752, + "MeanMS": 0.007579642836552295, + "P95UpperMS": 0.1, + "MaximumMS": 0.395973 + }, + "HTTPDuration": { + "Count": 27752, + "MeanMS": 8.306099014197164, + "P95UpperMS": 50, + "MaximumMS": 154.455314 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + } + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/coordinated-preparation.json b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/coordinated-preparation.json new file mode 100644 index 000000000..791ae6a4e --- /dev/null +++ b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/coordinated-preparation.json @@ -0,0 +1,1012 @@ +{ + "Calibration": false, + "RequestPhases": { + "page:HIT": { + "Attempts": 7758, + "NewConnections": 5, + "ReusedConnections": 7753, + "SchedulerLag": { + "Count": 7758, + "MeanMS": 0.6102065532353663, + "P95UpperMS": 2, + "MaximumMS": 1.25466 + }, + "QueueDelay": { + "Count": 7758, + "MeanMS": 0.008080155194637768, + "P95UpperMS": 0.1, + "MaximumMS": 0.138628 + }, + "ConnectionWait": { + "Count": 7758, + "MeanMS": 0.06835049432843542, + "P95UpperMS": 0.1, + "MaximumMS": 82.465807 + }, + "TLS": { + "Count": 8, + "MeanMS": 10.725214, + "P95UpperMS": 20, + "MaximumMS": 16.890521 + }, + "FirstByte": { + "Count": 7758, + "MeanMS": 1.742621534287188, + "P95UpperMS": 5, + "MaximumMS": 7.110818 + }, + "Body": { + "Count": 7758, + "MeanMS": 0.3463701854859493, + "P95UpperMS": 1, + "MaximumMS": 5.34709 + }, + "HTTP": { + "Count": 7758, + "MeanMS": 2.2305540738592424, + "P95UpperMS": 5, + "MaximumMS": 84.378646 + } + }, + "page:MISS": { + "Attempts": 2972, + "NewConnections": 11, + "ReusedConnections": 2961, + "SchedulerLag": { + "Count": 2972, + "MeanMS": 0.5685634656796764, + "P95UpperMS": 2, + "MaximumMS": 1.195506 + }, + "QueueDelay": { + "Count": 2972, + "MeanMS": 0.008133191453566611, + "P95UpperMS": 0.1, + "MaximumMS": 0.238007 + }, + "ConnectionWait": { + "Count": 2972, + "MeanMS": 0.14213784185733555, + "P95UpperMS": 0.1, + "MaximumMS": 70.94173 + }, + "TLS": { + "Count": 16, + "MeanMS": 13.22768, + "P95UpperMS": 50, + "MaximumMS": 21.676959 + }, + "FirstByte": { + "Count": 2972, + "MeanMS": 10.365214488559879, + "P95UpperMS": 50, + "MaximumMS": 325.28092 + }, + "Body": { + "Count": 2972, + "MeanMS": 1.0294676053162892, + "P95UpperMS": 5, + "MaximumMS": 12.952607 + }, + "HTTP": { + "Count": 2972, + "MeanMS": 11.606814613728154, + "P95UpperMS": 50, + "MaximumMS": 327.278232 + } + }, + "page:STALE": { + "Attempts": 1030, + "NewConnections": 3, + "ReusedConnections": 1027, + "SchedulerLag": { + "Count": 1030, + "MeanMS": 0.6068653834951458, + "P95UpperMS": 2, + "MaximumMS": 1.19214 + }, + "QueueDelay": { + "Count": 1030, + "MeanMS": 0.007886353398058262, + "P95UpperMS": 0.1, + "MaximumMS": 0.08558 + }, + "ConnectionWait": { + "Count": 1030, + "MeanMS": 0.07612816990291257, + "P95UpperMS": 0.1, + "MaximumMS": 19.249949 + }, + "TLS": { + "Count": 3, + "MeanMS": 6.857199333333333, + "P95UpperMS": 10, + "MaximumMS": 8.183694 + }, + "FirstByte": { + "Count": 1030, + "MeanMS": 1.7890110378640791, + "P95UpperMS": 5, + "MaximumMS": 6.109994 + }, + "Body": { + "Count": 1030, + "MeanMS": 0.3673701592233011, + "P95UpperMS": 1, + "MaximumMS": 4.708376 + }, + "HTTP": { + "Count": 1030, + "MeanMS": 2.3045229737864075, + "P95UpperMS": 5, + "MaximumMS": 22.70378 + } + }, + "search:MISS": { + "Attempts": 120, + "NewConnections": 0, + "ReusedConnections": 120, + "SchedulerLag": { + "Count": 120, + "MeanMS": 0.6580516666666668, + "P95UpperMS": 2, + "MaximumMS": 1.1651 + }, + "QueueDelay": { + "Count": 120, + "MeanMS": 0.02290523333333334, + "P95UpperMS": 0.1, + "MaximumMS": 0.101029 + }, + "ConnectionWait": { + "Count": 120, + "MeanMS": 0.017395625000000005, + "P95UpperMS": 0.1, + "MaximumMS": 0.08882 + }, + "TLS": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FirstByte": { + "Count": 120, + "MeanMS": 13.330504341666664, + "P95UpperMS": 50, + "MaximumMS": 29.334801 + }, + "Body": { + "Count": 120, + "MeanMS": 0.7136782749999999, + "P95UpperMS": 2, + "MaximumMS": 2.090795 + }, + "HTTP": { + "Count": 120, + "MeanMS": 14.139466041666669, + "P95UpperMS": 50, + "MaximumMS": 31.826239 + } + }, + "updated:MISS": { + "Attempts": 120, + "NewConnections": 0, + "ReusedConnections": 120, + "SchedulerLag": { + "Count": 120, + "MeanMS": 0.5873757166666665, + "P95UpperMS": 2, + "MaximumMS": 1.124677 + }, + "QueueDelay": { + "Count": 120, + "MeanMS": 0.020266283333333326, + "P95UpperMS": 0.1, + "MaximumMS": 0.079851 + }, + "ConnectionWait": { + "Count": 120, + "MeanMS": 0.018414124999999996, + "P95UpperMS": 0.1, + "MaximumMS": 0.315129 + }, + "TLS": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FirstByte": { + "Count": 120, + "MeanMS": 5.36502699166667, + "P95UpperMS": 10, + "MaximumMS": 13.716849 + }, + "Body": { + "Count": 120, + "MeanMS": 0.539956625, + "P95UpperMS": 5, + "MaximumMS": 3.44704 + }, + "HTTP": { + "Count": 120, + "MeanMS": 5.987646666666666, + "P95UpperMS": 20, + "MaximumMS": 13.893377 + } + } + }, + "StartedAt": "2026-10-28T20:05:32.067527528Z", + "FinishedAt": "2026-10-28T20:07:32.062247042Z", + "CatalogSHA256": "120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be", + "ReadRate": 100, + "UpdatedRate": 1, + "SearchRate": 1, + "HotPercent": 80, + "Scheduled": 12000, + "ScheduledPublishes": 0, + "Completed": 12000, + "Dropped": 0, + "DroppedPublishes": 0, + "Bytes": 686648150, + "Status": { + "page:200": 11760, + "search:200": 120, + "updated:200": 120 + }, + "Cache": { + "HIT": 7758, + "MISS": 3212, + "STALE": 1030 + }, + "Errors": {}, + "Protocols": { + "HTTP/1.1": 12000 + }, + "ErrorDetails": {}, + "LatencyMS": { + "page_p50": 2.965104, + "page_p95": 15.546276, + "page_p99": 39.464061, + "search_p50": 11.929439, + "search_p95": 28.649121, + "search_p99": 30.941536, + "updated_p50": 5.024697, + "updated_p95": 12.162068, + "updated_p99": 13.494919 + }, + "ElapsedSeconds": 119.994719365, + "Failures": null, + "OmittedFailures": 0, + "Windows": [ + { + "StartedAt": "2026-10-28T20:05:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 794, + "Dropped": 0, + "Dispatched": 794, + "Completed": 789, + "Failures": 0, + "SchedulerLag": { + "Count": 794, + "MeanMS": 0.6006814521410576, + "P95UpperMS": 2, + "MaximumMS": 1.156068 + }, + "QueueDelay": { + "Count": 794, + "MeanMS": 0.009422819899244324, + "P95UpperMS": 0.1, + "MaximumMS": 0.238007 + }, + "HTTPDuration": { + "Count": 794, + "MeanMS": 9.980458386649874, + "P95UpperMS": 50, + "MaximumMS": 327.278232 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:05:40Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1001, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.580320864, + "P95UpperMS": 2, + "MaximumMS": 1.171312 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.008007918999999999, + "P95UpperMS": 0.1, + "MaximumMS": 0.138628 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 6.797210924999998, + "P95UpperMS": 50, + "MaximumMS": 72.701668 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:05:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1002, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.6205297109999994, + "P95UpperMS": 2, + "MaximumMS": 1.170396 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.007964283999999992, + "P95UpperMS": 0.1, + "MaximumMS": 0.064379 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 5.440396379999996, + "P95UpperMS": 50, + "MaximumMS": 71.201188 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:06:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.5983163430000007, + "P95UpperMS": 2, + "MaximumMS": 1.193079 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.008123834999999996, + "P95UpperMS": 0.1, + "MaximumMS": 0.092269 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 5.720885304999994, + "P95UpperMS": 50, + "MaximumMS": 84.378646 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:06:10Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1001, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.6307411869999994, + "P95UpperMS": 2, + "MaximumMS": 1.199928 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.008047018000000005, + "P95UpperMS": 0.1, + "MaximumMS": 0.101229 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 4.503612236, + "P95UpperMS": 20, + "MaximumMS": 66.588292 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:06:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.6052201290000006, + "P95UpperMS": 2, + "MaximumMS": 1.206219 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.008286784, + "P95UpperMS": 0.1, + "MaximumMS": 0.101029 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 4.420898788, + "P95UpperMS": 20, + "MaximumMS": 27.905212 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:06:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.6066023409999999, + "P95UpperMS": 2, + "MaximumMS": 1.25466 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.008030190000000006, + "P95UpperMS": 0.1, + "MaximumMS": 0.08558 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 3.721858054999997, + "P95UpperMS": 20, + "MaximumMS": 33.226266 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:06:40Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.6090240500000004, + "P95UpperMS": 2, + "MaximumMS": 1.21839 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.008089864999999991, + "P95UpperMS": 0.1, + "MaximumMS": 0.070229 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 3.3925331290000047, + "P95UpperMS": 20, + "MaximumMS": 35.30935 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:06:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.5947721530000001, + "P95UpperMS": 2, + "MaximumMS": 1.16818 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.008648585999999993, + "P95UpperMS": 0.1, + "MaximumMS": 0.06091 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 3.4053930690000027, + "P95UpperMS": 10, + "MaximumMS": 25.666418 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:07:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.5749014600000002, + "P95UpperMS": 2, + "MaximumMS": 1.184171 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.008327549000000007, + "P95UpperMS": 0.1, + "MaximumMS": 0.05931 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 4.138121346999999, + "P95UpperMS": 10, + "MaximumMS": 37.336487 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:07:10Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.5898364259999996, + "P95UpperMS": 2, + "MaximumMS": 1.242755 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.008803939000000005, + "P95UpperMS": 0.1, + "MaximumMS": 0.1175 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 3.2589335590000017, + "P95UpperMS": 10, + "MaximumMS": 27.109729 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:07:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.584435746, + "P95UpperMS": 2, + "MaximumMS": 1.19214 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.008528758999999999, + "P95UpperMS": 0.1, + "MaximumMS": 0.07201 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 3.242628703000001, + "P95UpperMS": 10, + "MaximumMS": 25.986904 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:07:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 206, + "Dropped": 0, + "Dispatched": 206, + "Completed": 207, + "Failures": 0, + "SchedulerLag": { + "Count": 206, + "MeanMS": 0.6147291650485437, + "P95UpperMS": 2, + "MaximumMS": 1.151846 + }, + "QueueDelay": { + "Count": 206, + "MeanMS": 0.008838679611650482, + "P95UpperMS": 0.1, + "MaximumMS": 0.079851 + }, + "HTTPDuration": { + "Count": 206, + "MeanMS": 3.0196517961165084, + "P95UpperMS": 10, + "MaximumMS": 13.907639 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + } + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/coordinated-recovery.json b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/coordinated-recovery.json new file mode 100644 index 000000000..bd4b01ca4 --- /dev/null +++ b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/coordinated-recovery.json @@ -0,0 +1,1059 @@ +{ + "Calibration": false, + "RequestPhases": { + "page:HIT": { + "Attempts": 8530, + "NewConnections": 15, + "ReusedConnections": 8515, + "SchedulerLag": { + "Count": 8530, + "MeanMS": 0.6054950301289568, + "P95UpperMS": 2, + "MaximumMS": 1.194649 + }, + "QueueDelay": { + "Count": 8530, + "MeanMS": 0.0073014385697538434, + "P95UpperMS": 0.1, + "MaximumMS": 0.108561 + }, + "ConnectionWait": { + "Count": 8530, + "MeanMS": 0.23251113036342272, + "P95UpperMS": 0.1, + "MaximumMS": 197.853448 + }, + "TLS": { + "Count": 22, + "MeanMS": 16.279535727272723, + "P95UpperMS": 50, + "MaximumMS": 32.096695 + }, + "FirstByte": { + "Count": 8530, + "MeanMS": 1.7654619949589665, + "P95UpperMS": 5, + "MaximumMS": 206.832112 + }, + "Body": { + "Count": 8530, + "MeanMS": 0.3302826284876914, + "P95UpperMS": 1, + "MaximumMS": 3.732654 + }, + "HTTP": { + "Count": 8530, + "MeanMS": 2.395159540445487, + "P95UpperMS": 5, + "MaximumMS": 207.149154 + } + }, + "page:MISS": { + "Attempts": 3, + "NewConnections": 0, + "ReusedConnections": 3, + "SchedulerLag": { + "Count": 3, + "MeanMS": 0.5114956666666667, + "P95UpperMS": 1, + "MaximumMS": 0.799467 + }, + "QueueDelay": { + "Count": 3, + "MeanMS": 0.007706666666666666, + "P95UpperMS": 0.1, + "MaximumMS": 0.00895 + }, + "ConnectionWait": { + "Count": 3, + "MeanMS": 0.016326666666666666, + "P95UpperMS": 0.1, + "MaximumMS": 0.03216 + }, + "TLS": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FirstByte": { + "Count": 3, + "MeanMS": 4.848178, + "P95UpperMS": 10, + "MaximumMS": 5.193514 + }, + "Body": { + "Count": 3, + "MeanMS": 0.7490513333333334, + "P95UpperMS": 2, + "MaximumMS": 1.170877 + }, + "HTTP": { + "Count": 3, + "MeanMS": 5.67576, + "P95UpperMS": 10, + "MaximumMS": 6.432202 + } + }, + "page:STALE": { + "Attempts": 3227, + "NewConnections": 16, + "ReusedConnections": 3211, + "SchedulerLag": { + "Count": 3227, + "MeanMS": 0.6083422317942356, + "P95UpperMS": 2, + "MaximumMS": 1.314108 + }, + "QueueDelay": { + "Count": 3227, + "MeanMS": 0.007375136659435988, + "P95UpperMS": 0.1, + "MaximumMS": 0.077781 + }, + "ConnectionWait": { + "Count": 3227, + "MeanMS": 0.6080066904245424, + "P95UpperMS": 0.1, + "MaximumMS": 159.857352 + }, + "TLS": { + "Count": 23, + "MeanMS": 15.255267782608698, + "P95UpperMS": 50, + "MaximumMS": 32.919329 + }, + "FirstByte": { + "Count": 3227, + "MeanMS": 1.7690714031608326, + "P95UpperMS": 5, + "MaximumMS": 31.950055 + }, + "Body": { + "Count": 3227, + "MeanMS": 0.4891915215370317, + "P95UpperMS": 2, + "MaximumMS": 5.999019 + }, + "HTTP": { + "Count": 3227, + "MeanMS": 2.9320535667802967, + "P95UpperMS": 5, + "MaximumMS": 166.346658 + } + }, + "search:MISS": { + "Attempts": 78, + "NewConnections": 0, + "ReusedConnections": 78, + "SchedulerLag": { + "Count": 78, + "MeanMS": 0.6258682435897439, + "P95UpperMS": 2, + "MaximumMS": 1.132731 + }, + "QueueDelay": { + "Count": 78, + "MeanMS": 0.021385102564102563, + "P95UpperMS": 0.1, + "MaximumMS": 0.160911 + }, + "ConnectionWait": { + "Count": 78, + "MeanMS": 0.015408012820512825, + "P95UpperMS": 0.1, + "MaximumMS": 0.07172 + }, + "TLS": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FirstByte": { + "Count": 78, + "MeanMS": 12.648594730769231, + "P95UpperMS": 50, + "MaximumMS": 28.283195 + }, + "Body": { + "Count": 78, + "MeanMS": 0.686208512820513, + "P95UpperMS": 2, + "MaximumMS": 2.333215 + }, + "HTTP": { + "Count": 78, + "MeanMS": 13.408796115384613, + "P95UpperMS": 50, + "MaximumMS": 29.354671 + } + }, + "search:STALE": { + "Attempts": 42, + "NewConnections": 0, + "ReusedConnections": 42, + "SchedulerLag": { + "Count": 42, + "MeanMS": 0.6428315238095239, + "P95UpperMS": 2, + "MaximumMS": 1.132143 + }, + "QueueDelay": { + "Count": 42, + "MeanMS": 0.0194622380952381, + "P95UpperMS": 0.1, + "MaximumMS": 0.04041 + }, + "ConnectionWait": { + "Count": 42, + "MeanMS": 0.013218952380952378, + "P95UpperMS": 0.1, + "MaximumMS": 0.0351 + }, + "TLS": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FirstByte": { + "Count": 42, + "MeanMS": 1.8138779999999994, + "P95UpperMS": 5, + "MaximumMS": 2.366856 + }, + "Body": { + "Count": 42, + "MeanMS": 0.41717059523809513, + "P95UpperMS": 1, + "MaximumMS": 1.753551 + }, + "HTTP": { + "Count": 42, + "MeanMS": 2.3121028333333333, + "P95UpperMS": 5, + "MaximumMS": 4.169557 + } + }, + "updated:MISS": { + "Attempts": 120, + "NewConnections": 0, + "ReusedConnections": 120, + "SchedulerLag": { + "Count": 120, + "MeanMS": 0.6236253416666666, + "P95UpperMS": 2, + "MaximumMS": 1.167873 + }, + "QueueDelay": { + "Count": 120, + "MeanMS": 0.020249833333333352, + "P95UpperMS": 0.1, + "MaximumMS": 0.095941 + }, + "ConnectionWait": { + "Count": 120, + "MeanMS": 0.016530225000000003, + "P95UpperMS": 0.1, + "MaximumMS": 0.078051 + }, + "TLS": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FirstByte": { + "Count": 120, + "MeanMS": 4.848505491666667, + "P95UpperMS": 10, + "MaximumMS": 9.369008 + }, + "Body": { + "Count": 120, + "MeanMS": 0.4865586583333333, + "P95UpperMS": 2, + "MaximumMS": 2.244344 + }, + "HTTP": { + "Count": 120, + "MeanMS": 5.415946758333333, + "P95UpperMS": 10, + "MaximumMS": 10.945238 + } + } + }, + "StartedAt": "2026-10-28T20:10:58.184279679Z", + "FinishedAt": "2026-10-28T20:12:58.179258109Z", + "CatalogSHA256": "120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be", + "ReadRate": 100, + "UpdatedRate": 1, + "SearchRate": 1, + "HotPercent": 80, + "Scheduled": 12000, + "ScheduledPublishes": 0, + "Completed": 12000, + "Dropped": 0, + "DroppedPublishes": 0, + "Bytes": 686648150, + "Status": { + "page:200": 11760, + "search:200": 120, + "updated:200": 120 + }, + "Cache": { + "HIT": 8530, + "MISS": 201, + "STALE": 3269 + }, + "Errors": {}, + "Protocols": { + "HTTP/1.1": 12000 + }, + "ErrorDetails": {}, + "LatencyMS": { + "page_p50": 2.758622, + "page_p95": 3.675634, + "page_p99": 5.42456, + "search_p50": 8.622515, + "search_p95": 26.852288, + "search_p99": 29.450764, + "updated_p50": 4.779784, + "updated_p95": 9.606068, + "updated_p99": 10.175718 + }, + "ElapsedSeconds": 119.99497825, + "Failures": null, + "OmittedFailures": 0, + "Windows": [ + { + "StartedAt": "2026-10-28T20:10:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 182, + "Dropped": 0, + "Dispatched": 182, + "Completed": 182, + "Failures": 0, + "SchedulerLag": { + "Count": 182, + "MeanMS": 0.5736446208791208, + "P95UpperMS": 2, + "MaximumMS": 1.163059 + }, + "QueueDelay": { + "Count": 182, + "MeanMS": 0.01362871978021978, + "P95UpperMS": 0.1, + "MaximumMS": 0.04454 + }, + "HTTPDuration": { + "Count": 182, + "MeanMS": 14.160724483516475, + "P95UpperMS": 200, + "MaximumMS": 202.950818 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:11:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.6348515770000005, + "P95UpperMS": 2, + "MaximumMS": 1.168131 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.0077896719999999975, + "P95UpperMS": 0.1, + "MaximumMS": 0.160911 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 2.194456371000001, + "P95UpperMS": 5, + "MaximumMS": 23.881158 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:11:10Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.6095377899999993, + "P95UpperMS": 2, + "MaximumMS": 1.237709 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.007498978000000009, + "P95UpperMS": 0.1, + "MaximumMS": 0.08375 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 2.190852900000001, + "P95UpperMS": 5, + "MaximumMS": 25.07925 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:11:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.635166216, + "P95UpperMS": 2, + "MaximumMS": 1.163471 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.007222482999999996, + "P95UpperMS": 0.1, + "MaximumMS": 0.05899 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 2.3674744779999997, + "P95UpperMS": 5, + "MaximumMS": 49.54641 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:11:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.6077039079999998, + "P95UpperMS": 2, + "MaximumMS": 1.181672 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.00719693, + "P95UpperMS": 0.1, + "MaximumMS": 0.042691 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 2.5245341170000035, + "P95UpperMS": 5, + "MaximumMS": 207.149154 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:11:40Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.609485044, + "P95UpperMS": 2, + "MaximumMS": 1.153784 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.007219903000000002, + "P95UpperMS": 0.1, + "MaximumMS": 0.06226 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 2.2734473520000065, + "P95UpperMS": 5, + "MaximumMS": 29.354671 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:11:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.6080443389999998, + "P95UpperMS": 2, + "MaximumMS": 1.154899 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.007424860000000005, + "P95UpperMS": 0.1, + "MaximumMS": 0.07208 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 2.742123821000002, + "P95UpperMS": 5, + "MaximumMS": 79.573947 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:12:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.6050329240000002, + "P95UpperMS": 2, + "MaximumMS": 1.171807 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.007637416999999991, + "P95UpperMS": 0.1, + "MaximumMS": 0.095941 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 2.2776574059999986, + "P95UpperMS": 5, + "MaximumMS": 26.481787 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:12:10Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.6020763519999999, + "P95UpperMS": 2, + "MaximumMS": 1.155203 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.007404020000000001, + "P95UpperMS": 0.1, + "MaximumMS": 0.02616 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 2.209794049000001, + "P95UpperMS": 5, + "MaximumMS": 26.068576 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:12:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.5908976329999995, + "P95UpperMS": 2, + "MaximumMS": 1.175619 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.007775054, + "P95UpperMS": 0.1, + "MaximumMS": 0.091911 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 3.672824033000002, + "P95UpperMS": 5, + "MaximumMS": 153.978378 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:12:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.5845371510000011, + "P95UpperMS": 2, + "MaximumMS": 1.162343 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.007590898999999994, + "P95UpperMS": 0.1, + "MaximumMS": 0.061051 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 2.3703842429999997, + "P95UpperMS": 5, + "MaximumMS": 20.376961 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:12:40Z", + "Seconds": 10, + "Reads": { + "Scheduled": 1000, + "Dropped": 0, + "Dispatched": 1000, + "Completed": 1000, + "Failures": 0, + "SchedulerLag": { + "Count": 1000, + "MeanMS": 0.5889164210000001, + "P95UpperMS": 2, + "MaximumMS": 1.314108 + }, + "QueueDelay": { + "Count": 1000, + "MeanMS": 0.007529098000000005, + "P95UpperMS": 0.1, + "MaximumMS": 0.06384 + }, + "HTTPDuration": { + "Count": 1000, + "MeanMS": 2.3697637510000016, + "P95UpperMS": 5, + "MaximumMS": 26.126769 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:12:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 818, + "Dropped": 0, + "Dispatched": 818, + "Completed": 818, + "Failures": 0, + "SchedulerLag": { + "Count": 818, + "MeanMS": 0.6106680782396088, + "P95UpperMS": 2, + "MaximumMS": 1.183337 + }, + "QueueDelay": { + "Count": 818, + "MeanMS": 0.0076401564792176, + "P95UpperMS": 0.1, + "MaximumMS": 0.059451 + }, + "HTTPDuration": { + "Count": 818, + "MeanMS": 2.36163651589242, + "P95UpperMS": 5, + "MaximumMS": 16.84122 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + } + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/cpu-budget.json b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/cpu-budget.json new file mode 100644 index 000000000..7822a04ce --- /dev/null +++ b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/cpu-budget.json @@ -0,0 +1,70 @@ +{ + "source": "Saved staging snapshot 2026-10-28T15:51Z; requires live refresh before apply", + "node_allocatable_mcpu": { + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-2846": 3920.0, + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k": 3920.0, + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-d57s": 3920.0, + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25": 3920.0 + }, + "candidate": { + "db_request_mcpu": 1500, + "db_limit_mcpu": 3000, + "app_request_mcpu": 800, + "ingress_request_mcpu": 800, + "app_min_max": [ + 2, + 4 + ], + "ingress_min_max": [ + 2, + 6 + ], + "node_min_max": [ + 2, + 4 + ], + "pool_max_min_per_app": [ + 30, + 5 + ] + }, + "idle_requested_mcpu": { + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25": 3726.0, + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k": 2683.0 + }, + "idle_spare_mcpu": { + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25": 194.0, + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k": 1237.0 + }, + "peak_requested_mcpu": { + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-2846": 2756.0, + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k": 2653.0, + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-d57s": 2756.0, + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25": 3726.0 + }, + "peak_spare_mcpu": { + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-2846": 1164.0, + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k": 1267.0, + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-d57s": 1164.0, + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25": 194.0 + }, + "idle_other_pod_relocations": [ + { + "name": "konnectivity-agent-6977df8778-4f62d", + "to": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "mcpu": 15.0 + }, + { + "name": "konnectivity-agent-6977df8778-mqm4k", + "to": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "mcpu": 15.0 + } + ], + "fits_cpu_requests": true, + "limitations": [ + "Constructive CPU request packing, not a Kubernetes scheduling simulation.", + "Does not prove runtime CPU headroom, memory fit, volume/affinity feasibility, autoscaler evacuation, or two-node recovery.", + "Keeps singleton DB nodes fixed; allows other movable workloads to relocate.", + "HPA reacts after demand, and node provisioning and load-balancer readiness add delay." + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/equivalence-restored.json b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/equivalence-restored.json new file mode 100644 index 000000000..0d2c88e41 --- /dev/null +++ b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/equivalence-restored.json @@ -0,0 +1,50 @@ +{ + "/v0/servers?limit=50": "efdc61f649d37dea71f2b9bc4cfad4556e851a7429c4bcfb40b8144de8d0c3cd", + "/v0/servers?cursor=ai.vidocu%2Fvidocu%3A1.0.2&limit=50": "9f64a5c4f98ae84f8372d6338ce09c80c5bd86964bb20ada48de127606acdb15", + "/v0/servers?cursor=com.aicontentdrop%2Fai-content-drop%3A1.3.5&limit=50": "54093908af35e141c91262039e8914e085bf3acbdc553e6707b0a51724db06cf", + "/v0/servers?cursor=com.getuserfeedback%2Fmcp%3Ae31d8eae5d26&limit=50": "17463327797187dc92ff1718c29a6162910cab7baff46342fb9e93844cee8fa1", + "/v0/servers?cursor=com.pdfhaul.www%2Fmcp%3A1.3.2&limit=50": "70a5e9187fbd1887d4c4e60506b0700e0da019a3adef9c56ec8cc9aa5c025911", + "/v0/servers?cursor=com.websterberrymarketing.synaptic%2Fdata-agents%3A1.0.0&limit=50": "ffc1084677b7a63c92a2d8042762c205a589a6f1d71003602a02a11ac6402321", + "/v0/servers?cursor=io.capawesome%2Fionic-framework-mcp%3A0.0.1&limit=50": "a97c58fca136484e45690a7047b1bb245c672268405511b92b816d387fa93aab", + "/v0/servers?cursor=io.github.BlockRunAI%2Fblockrun-mcp%3A0.54.4&limit=50": "69203c6d28475ddd6193205196e03c27a99a1f9c66a81973f400ca56cad3176b", + "/v0/servers?cursor=app.scfcontrolsplatform%2Fmcp-server-scf%3A1.6.1&limit=100": "08ea7c61e3201bac33066371bd21ba251d162a677d13da636d7d3ed1c3bcd322", + "/v0/servers?cursor=com.linkbreakers%2Fmcp%3A1.30.0&limit=100": "2b6f5aa4f413885eae41152246090ea5c7df82121ec92a61c57760e20d63226a", + "/v0/servers?cursor=dev.layrcake%2Flayrcake%3A1.0.0&limit=100": "a0257f052b910fa437497f9f6e59c238b0cda2ba5cdbe5da92b0c66570271608", + "/v0/servers?cursor=io.github.CloudCorpRecords%2Fhookcheck%3A0.1.11&limit=100": "738ebb44477af1faee7cb4b7a59f625fd3c722d6b9c551550fe523d2052a5a61", + "/v0/servers?cursor=io.github.OpenHCSDev%2Fopenhcs%3A0.7.26&limit=100": "b6c7aaf4437d336598881f359760f5373b7d33f200eb3b05e8a6982fceb9b5b8", + "/v0/servers?cursor=io.github.antonbabenko%2Fdeliberation%3A3.6.2&limit=100": "3cb9875c03790140ddc2cada26e980034ed5d5e7b6ddb0b4fe14f10539628c8f", + "/v0/servers?cursor=io.github.cyanheads%2Fevals-mcp-server%3A0.1.5&limit=100": "efb971b8964c50e63bcfffa8cd9dfb9345a29e6a97000b2ed326252d82c65cf2", + "/v0.1/servers?cursor=ai.getvda%2Foauth-multi-llm-orchestrator%3A1.0.0&limit=50": "b83a9c2a0a0fd56a91ea443d2f699bbd48a87cfcd829bf382f7d9e422ff2a7e4", + "/v0.1/servers?cursor=app.scfcontrolsplatform%2Fmcp-server-scf%3A1.6.1&limit=50": "5625429a197a6a525c027542e4646696521d3fca73c290218d1e5b03652bb958", + "/v0.1/servers?cursor=com.civic%2Fnexus%3A0.1.94&limit=50": "3f4800dca255267136c57f549dc8050244353f585b5c2495f820e7b65055a0b7", + "/v0.1/servers?cursor=com.linkbreakers%2Fmcp%3A1.30.0&limit=50": "95dc9bf8656448b9ab91de9252c630875a6f5b95556624b0279e0eb61fc682e2", + "/v0.1/servers?cursor=com.senzing%2Fmcp%3A1.30.0&limit=50": "8dc79692cad29b9c82fae8a872ea89384e8202b6d0fd22767d4df04949c0d366", + "/v0.1/servers?cursor=dev.layrcake%2Flayrcake%3A1.0.0&limit=50": "119811b407d30a721055a8d503ad8255eb6840b4639d92dcb90ac87b339acefa", + "/v0.1/servers?cursor=io.github.AceDataCloud%2Fmcp-flux-pro%3A2026.9.30.0&limit=50": "2634ed3d9f8be0507b80a318d6a9f697eb5773ad93ea46b3561001ca2c714717", + "/v0.1/servers?cursor=ai.bowmark%2Fbowmark%3A8.182.1&limit=100": "64d47be6dbd0eb47ae32ad6c847d65c0380de836b490846df2630cceef82a567", + "/v0.1/servers?cursor=com.babyblueviper%2Finvinoveritas%3A1.9.0&limit=100": "7084ecb8668727bfb24d800b0572ab85562695e05badb2f4f1d70f75da77dd6f", + "/v0.1/servers?cursor=com.reletter%2Freletter%3A0.1.0&limit=100": "cceeaa833c468fa9ebeb4daa2642b1597c841ad3febb9cbbd5e9bfc89d05aada", + "/v0.1/servers?cursor=io.github.AAAZZZR%2Flivermore%3A2.2.0&limit=100": "2324022e7225f9237457d241cdc65a38e30262075805de97f754d9f13a6965ca", + "/v0.1/servers?cursor=io.github.HelpCode-ai%2Fanythingmcp%3A0.44.1&limit=100": "f19ed76afbae83ec1ae8b6d68fa672a519c20544c3705c531cdd86486aa9e3b4", + "/v0.1/servers?cursor=io.github.Starksood%2Ffireweed-mcp%3A0.4.0&limit=100": "79c8513df6867f378906b699b7c5aca44fa7901e57944520a1691d6c10f4663a", + "/v0.1/servers?cursor=io.github.brilliantdirectories%2Fbrilliant-directories-mcp%3A6.41.92&limit=100": "5ec2cf3e2cf498b71f3e71dcec683a528d8ed455cbd71fca9c0be591f591ab21", + "/v0.1/servers?cursor=io.github.dinosaur24%2Fmockhero%3A0.2.0&limit=100": "8dd385d256228edb6c52ada232d919581ae8bd58aa1eee05c96340b3797366e0", + "/v0.1/servers?search=github&limit=1": "18a8e568347e939db6f10206febdbc9c4a5b578852ec65296b8d304bbd08142a", + "/v0.1/servers?search=github&limit=50": "1d9c9da028df4c910aba2da3cc8e38161eacb20039f84f89aafa24977ed3dcb1", + "/v0.1/servers?search=github&limit=100": "d834639dcb51b09162e540022f115ccfcc027c0b55e14702b7da15aa7fe7062f", + "/v0.1/servers?search=mcp&limit=1": "f09269dc25c8f53aec76cae410f238126c23e6f2afaf09d0b8696c5665372e32", + "/v0.1/servers?search=mcp&limit=50": "b9fb2063619613061f864a3f29c731582176be6da603cb35d2e24742809093e3", + "/v0.1/servers?search=mcp&limit=100": "4ec928a4d179ebdc9654df75835900f2e943275c5907bfcba56656f2c2ec2ea3", + "/v0.1/servers?search=postgres&limit=1": "d6d407d2b8d656bc6ad21794c4cc0e1db93a5be67b788a56e0399fa3808326af", + "/v0.1/servers?search=postgres&limit=50": "be6b7e31c4c283a0a0c49ae12544343f75c10691ebcf5f9474c8b4b7e3e2b30c", + "/v0.1/servers?search=postgres&limit=100": "88d6720b7cfb5c4bcd01ac77abf57a1093b41f444a8cf8259f856ea333981ad7", + "/v0.1/servers?search=server&limit=1": "8fc86d584920c961d25ee9fcac30231d1873fedd765a189caf4523a54a7fafde", + "/v0.1/servers?search=server&limit=50": "c8c8ad35bf4a9d97e6a427999fa6e0708889a82782526624080eff8b01e62909", + "/v0.1/servers?search=server&limit=100": "26bd91b45b671dea950952b6b2167ceaf251af6968c1dfdb0315f882621e8219", + "/v0.1/servers?search=cloud&limit=1": "4104bd00f5cb26ac14a9708b584afe171005f071ced079810765b826c51fd689", + "/v0.1/servers?search=cloud&limit=50": "a37e70a131c8221dec17a719418b5caa7695ad0061a8c1b2d1a68436611f0fa7", + "/v0.1/servers?search=cloud&limit=100": "f2915fa2a2c8168d87e832fbc4cd887e01548324db416c29b8a2b421c03ab1fc", + "/v0.1/servers?updated_since=2026-10-26T00%3A00%3A00Z&limit=1": "1e08fcd0370657e4ee613b8da1d0b53c3a12719d0c19509745b88d572f52f255", + "/v0.1/servers?updated_since=2026-10-26T00%3A00%3A00Z&limit=50": "fd493a039948f1d7f6463fec5de112e90e16c3d58025518ed703fb4b46d9ff9f", + "/v0.1/servers?updated_since=2026-10-26T00%3A00%3A00Z&limit=100": "039cef68cc78623911efe83e9cfd0183df8639f0d87de5f7f1a067da9c5a1d1f" +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/generator-summary.json b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/generator-summary.json new file mode 100644 index 000000000..a81d9a10c --- /dev/null +++ b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/generator-summary.json @@ -0,0 +1,219 @@ +{ + "sample_intervals": 17, + "scope": "Whole sixteen-vCPU generator from /proc/stat and non-loopback network devices; includes monitoring overhead. CPU busy includes steal, reported separately. Does not alone rule out single-thread or network constraints.", + "intervals": [ + { + "at": 1793218068.7579281, + "cpu_busy_percent": 16.666666666666668, + "cpu_steal_percent": 0.18349784864591243, + "cpu_iowait_percent": 0.006327512022272842, + "cpu_softirq_percent": 5.062009617818274, + "received_gbps": 2.168580031962932, + "transmitted_gbps": 0.026813923961286717, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218078.764958, + "cpu_busy_percent": 16.40546006066734, + "cpu_steal_percent": 0.2717391304347826, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 4.581648129423661, + "received_gbps": 2.1222977698975765, + "transmitted_gbps": 0.02526379612782981, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218088.7694807, + "cpu_busy_percent": 22.59479786900658, + "cpu_steal_percent": 0.38232528987778125, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 5.954246317768725, + "received_gbps": 2.0939565044588657, + "transmitted_gbps": 0.031550045743905214, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218098.7746837, + "cpu_busy_percent": 16.559546313799622, + "cpu_steal_percent": 0.20163831127914303, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 4.83301827347196, + "received_gbps": 2.119994765689249, + "transmitted_gbps": 0.025025906998936705, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218108.7818828, + "cpu_busy_percent": 19.98373271601076, + "cpu_steal_percent": 0.21898266908590377, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 5.662266157792654, + "received_gbps": 2.11563901950408, + "transmitted_gbps": 0.028499197687949854, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218118.78634, + "cpu_busy_percent": 18.869707661290324, + "cpu_steal_percent": 0.14490927419354838, + "cpu_iowait_percent": 0.006300403225806451, + "cpu_softirq_percent": 5.254536290322581, + "received_gbps": 2.1082934719837376, + "transmitted_gbps": 0.02902318048543684, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218128.7926638, + "cpu_busy_percent": 16.566340606972663, + "cpu_steal_percent": 0.13794833207925758, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 4.947328818660647, + "received_gbps": 2.12166049248425, + "transmitted_gbps": 0.025416375156099312, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218138.7968364, + "cpu_busy_percent": 18.241104216791186, + "cpu_steal_percent": 0.1709509940483728, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 4.830948461441054, + "received_gbps": 2.111374105336163, + "transmitted_gbps": 0.028392760140387323, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218148.8012009, + "cpu_busy_percent": 17.938785877571757, + "cpu_steal_percent": 0.158750317500635, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 4.470408940817881, + "received_gbps": 2.0890449119308934, + "transmitted_gbps": 0.028566430608473237, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218158.806124, + "cpu_busy_percent": 15.485098287888396, + "cpu_steal_percent": 0.1648700063411541, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 4.185161699429296, + "received_gbps": 2.1215279646560674, + "transmitted_gbps": 0.024787959026902902, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218168.8111727, + "cpu_busy_percent": 18.94624466214519, + "cpu_steal_percent": 0.16327555890479778, + "cpu_iowait_percent": 0.006279829188646069, + "cpu_softirq_percent": 5.344134639537804, + "received_gbps": 2.1147404216424026, + "transmitted_gbps": 0.028201761630432937, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218178.8163397, + "cpu_busy_percent": 17.726580350342726, + "cpu_steal_percent": 0.16501650165016502, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 4.747397816704748, + "received_gbps": 2.097181029000694, + "transmitted_gbps": 0.028315800404846055, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218188.8204024, + "cpu_busy_percent": 15.578813828100222, + "cpu_steal_percent": 0.1522359657469077, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 4.548049476688868, + "received_gbps": 2.1217985677557722, + "transmitted_gbps": 0.02554131145249294, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218198.8266678, + "cpu_busy_percent": 18.61156815592963, + "cpu_steal_percent": 0.2088343247690166, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 4.632325022149095, + "received_gbps": 2.1133074295741814, + "transmitted_gbps": 0.028737330107905922, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218208.8313894, + "cpu_busy_percent": 18.44236365936986, + "cpu_steal_percent": 0.1771479185119575, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 4.605845881310895, + "received_gbps": 2.105774347236711, + "transmitted_gbps": 0.028506323535862535, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218218.8356597, + "cpu_busy_percent": 15.372878123211901, + "cpu_steal_percent": 0.17165744802593935, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 4.132494119142985, + "received_gbps": 2.1215138739120256, + "transmitted_gbps": 0.02487169420269355, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793218228.8399763, + "cpu_busy_percent": 17.975180448271495, + "cpu_steal_percent": 0.15195643915410917, + "cpu_iowait_percent": 0.006331518298087881, + "cpu_softirq_percent": 4.7423072052678235, + "received_gbps": 2.1178395112943, + "transmitted_gbps": 0.028475102927124036, + "receive_drops": 0, + "transmit_drops": 0 + } + ], + "cpu_busy_percent": { + "mean": 17.762639382590372, + "max": 22.59479786900658 + }, + "cpu_steal_percent": { + "mean": 0.18974919589702258, + "max": 0.38232528987778125 + }, + "cpu_iowait_percent": { + "mean": 0.0014846625138125438, + "max": 0.006331518298087881 + }, + "cpu_softirq_percent": { + "mean": 4.85494863927935, + "max": 5.954246317768725 + }, + "received_gbps": { + "mean": 2.1155602481364646, + "max": 2.168580031962932 + }, + "transmitted_gbps": { + "mean": 0.027411111776386228, + "max": 0.031550045743905214 + }, + "receive_drops": 0, + "transmit_drops": 0 +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/preflight.json b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/preflight.json new file mode 100644 index 000000000..afae320a5 --- /dev/null +++ b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/preflight.json @@ -0,0 +1,238 @@ +{ + "at": 1793217505.732235, + "server_dry_run": "passed", + "production_changed": false, + "candidate_image": "us-central1-docker.pkg.dev/mcp-registry-staging/registry-staging-experiments/registry:staging-replicas-20261028-v1@sha256:60fb8b7d7931a257542848460f7d1c76a99f16c46ddad795249fb5c25b1c9b0e", + "driver_sha256": "93de7e015c0fe51df8ff472f36a306678e1e48fa2c7e81a33b061b44c870550c", + "catalog_sha256": "120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be", + "live_idle_budget": { + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k": { + "cpu_m": 2653.0, + "memory_mi": 2892.1278076171875, + "pods": [ + { + "name": "cert-manager-85586dbccb-t76gt", + "cpu_m": 0.0, + "memory_mi": 0.0 + }, + { + "name": "cert-manager-cainjector-55cd9f77b5-r2vk4", + "cpu_m": 0.0, + "memory_mi": 0.0 + }, + { + "name": "cert-manager-webhook-7987476d56-4pj8x", + "cpu_m": 0.0, + "memory_mi": 0.0 + }, + { + "name": "k8up-86967d97c5-x4vrt", + "cpu_m": 20.0, + "memory_mi": 128.0 + }, + { + "name": "mcp-registry-6b9df99d6c-2rjjc", + "cpu_m": 800, + "memory_mi": 512.0 + }, + { + "name": "kube-state-metrics-0", + "cpu_m": 105.0, + "memory_mi": 130.0 + }, + { + "name": "collector-tlb2v", + "cpu_m": 5.0, + "memory_mi": 34.332275390625 + }, + { + "name": "gmp-operator-84bcc79c46-d65wx", + "cpu_m": 1.0, + "memory_mi": 15.2587890625 + }, + { + "name": "ingress-nginx-controller-54dc7ccbb8-mzmdk", + "cpu_m": 800.0, + "memory_mi": 512.0 + }, + { + "name": "event-exporter-gke-6d4f698b66-gpnz2", + "cpu_m": 3.0, + "memory_mi": 100.0 + }, + { + "name": "fluentbit-gke-f7582", + "cpu_m": 105.0, + "memory_mi": 230.0 + }, + { + "name": "gke-metrics-agent-mpf8h", + "cpu_m": 21.0, + "memory_mi": 145.0 + }, + { + "name": "konnectivity-agent-6977df8778-ppts2", + "cpu_m": 15.0, + "memory_mi": 60.0 + }, + { + "name": "konnectivity-agent-autoscaler-76cd9ccc9c-ngst9", + "cpu_m": 10.0, + "memory_mi": 9.5367431640625 + }, + { + "name": "kube-dns-7d659bf477-d959c", + "cpu_m": 270.0, + "memory_mi": 155.0 + }, + { + "name": "kube-dns-autoscaler-5b85d8b66d-9dtsz", + "cpu_m": 20.0, + "memory_mi": 10.0 + }, + { + "name": "kube-proxy-gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "cpu_m": 100.0, + "memory_mi": 0.0 + }, + { + "name": "metrics-server-v1.35.1-5998cfcd8f-4bmr5", + "cpu_m": 43.0, + "memory_mi": 55.0 + }, + { + "name": "pdcsi-node-s64kk", + "cpu_m": 10.0, + "memory_mi": 20.0 + }, + { + "name": "grafana-7cc5d86f9-p2wmz", + "cpu_m": 50.0, + "memory_mi": 128.0 + }, + { + "name": "grafana-pg-1", + "cpu_m": 0.0, + "memory_mi": 0.0 + }, + { + "name": "opentelemetry-collector-agent-866hv", + "cpu_m": 100.0, + "memory_mi": 200.0 + }, + { + "name": "victoria-logs-victoria-logs-single-server-0", + "cpu_m": 100.0, + "memory_mi": 256.0 + }, + { + "name": "victoria-metrics-agent-dfc8bd7d7-d66bt", + "cpu_m": 25.0, + "memory_mi": 64.0 + }, + { + "name": "victoria-metrics-victoria-metrics-single-server-0", + "cpu_m": 50.0, + "memory_mi": 128.0 + } + ] + }, + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25": { + "cpu_m": 3726.0, + "memory_mi": 2380.332275390625, + "pods": [ + { + "name": "cloudnative-pg-5b6dd77f9d-slkqg", + "cpu_m": 0.0, + "memory_mi": 0.0 + }, + { + "name": "mcp-registry-6b9df99d6c-nws25", + "cpu_m": 800, + "memory_mi": 512.0 + }, + { + "name": "registry-pg-1", + "cpu_m": 1500, + "memory_mi": 512.0 + }, + { + "name": "collector-5t5hd", + "cpu_m": 5.0, + "memory_mi": 34.332275390625 + }, + { + "name": "ingress-nginx-controller-54dc7ccbb8-5zhms", + "cpu_m": 800.0, + "memory_mi": 512.0 + }, + { + "name": "fluentbit-gke-v8g89", + "cpu_m": 105.0, + "memory_mi": 230.0 + }, + { + "name": "gke-metrics-agent-jblcc", + "cpu_m": 21.0, + "memory_mi": 145.0 + }, + { + "name": "konnectivity-agent-6977df8778-9ncmm", + "cpu_m": 15.0, + "memory_mi": 60.0 + }, + { + "name": "kube-dns-7d659bf477-w5jfg", + "cpu_m": 270.0, + "memory_mi": 155.0 + }, + { + "name": "kube-proxy-gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "cpu_m": 100.0, + "memory_mi": 0.0 + }, + { + "name": "pdcsi-node-bhrkr", + "cpu_m": 10.0, + "memory_mi": 20.0 + }, + { + "name": "opentelemetry-collector-agent-8xc4k", + "cpu_m": 100.0, + "memory_mi": 200.0 + } + ] + } + }, + "live_pretest_nodes": 2, + "node_min_max": [ + 2, + 4 + ], + "cpu_request_milli": { + "app": 800, + "ingress": 800, + "postgres": 1500 + }, + "db_cpu_limit": 3, + "warm_gate_replicas": { + "app": 4, + "ingress": 6 + }, + "eventual_idle_replicas": { + "app": 2, + "ingress": 2 + }, + "pool_max_min_each": [ + 30, + 5 + ], + "app_hpa_cpu_target": 70, + "ingress_hpa_cpu_target": 70, + "baseline_equivalence_count": 48, + "limitations": [ + "Reserved CPU fits, runtime performance still unproven.", + "DB pod restart is expected before testing.", + "Warm short gate does not test autoscaling reaction time." + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/restored-corpus.json b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/restored-corpus.json new file mode 100644 index 000000000..2ae4026aa --- /dev/null +++ b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/restored-corpus.json @@ -0,0 +1,6 @@ +{ + "at": 1793218901.7175639, + "code": 0, + "stdout": "BEGIN\nSET\n{\"at\": \"2026-10-28T20:21:41.187327+00:00\", \"names\": 52366, \"versions\": 187108, \"benchmark_runs\": null, \"database_started\": \"2026-10-28T20:21:23.962946+00:00\"}\nCOMMIT\n", + "stderr": "" +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/runtime-integrity.json b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/runtime-integrity.json new file mode 100644 index 000000000..cfeef7637 --- /dev/null +++ b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/runtime-integrity.json @@ -0,0 +1,37 @@ +{ + "as_of": 1793218723.269331, + "samples": 17, + "commands_failed": [], + "app_restarts": { + "mcp-registry-57d9d88679-52rg4/mcp-registry": 0, + "mcp-registry-57d9d88679-bzq4l/mcp-registry": 0, + "mcp-registry-57d9d88679-sjl52/mcp-registry": 0, + "mcp-registry-57d9d88679-vfcxl/mcp-registry": 0, + "registry-pg-1/postgres": 0 + }, + "node_count_timeline": [ + { + "at": 1793217912.826272, + "nodes": 4 + } + ], + "db_throttled_usec_range": [ + 0, + 0 + ], + "db_endpoint_samples": 90, + "db_endpoint_errors_or_empty": [], + "db_endpoint_not_ready": [], + "metric_window": { + "start": 1793217928.0, + "end": 1793218622.6379101 + }, + "metric_errors": [], + "empty_metrics": [ + "alerts", + "raw_5xx", + "raw_cancellations" + ], + "cadvisor_failures_or_empty": [], + "lb_collection_failures": [] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/summary.json b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/summary.json new file mode 100644 index 000000000..c4abf20e3 --- /dev/null +++ b/tools/ingressbench/evidence/coordinated-capacity-2026-10-28/summary.json @@ -0,0 +1,440 @@ +{ + "as_of": "2026-10-28T20:19:25.005885+00:00", + "verdict": "FAIL_SHORT_DELIVERY_GATE", + "production_changed": false, + "long_burst_and_30min_recovery_run": false, + "autoscaling_from_idle_validated": false, + "short_gate_capacity": { + "apps": 4, + "app_cpu_request_m": 800, + "pool_per_app_max_min": [ + 30, + 5 + ], + "ingress_replicas": 6, + "ingress_cpu_request_m": 800, + "nodes": 4, + "db_cpu_request_m": 1500, + "db_cpu_limit": 3 + }, + "phases": { + "preparation": { + "StartedAt": "2026-10-28T20:05:32.067527528Z", + "FinishedAt": "2026-10-28T20:07:32.062247042Z", + "Scheduled": 12000, + "Completed": 12000, + "Dropped": 0, + "DroppedPublishes": 0, + "ScheduledPublishes": 0, + "Status": { + "page:200": 11760, + "search:200": 120, + "updated:200": 120 + }, + "Errors": {}, + "LatencyMS": { + "page_p50": 2.965104, + "page_p95": 15.546276, + "page_p99": 39.464061, + "search_p50": 11.929439, + "search_p95": 28.649121, + "search_p99": 30.941536, + "updated_p50": 5.024697, + "updated_p95": 12.162068, + "updated_p99": 13.494919 + } + }, + "burst": { + "StartedAt": "2026-10-28T20:07:36.167225061Z", + "FinishedAt": "2026-10-28T20:10:36.195036054Z", + "Scheduled": 810000, + "Completed": 802582, + "Dropped": 7418, + "DroppedPublishes": 0, + "ScheduledPublishes": 0, + "Status": { + "page:200": 795630, + "search:200": 534, + "updated:200": 6418 + }, + "Errors": {}, + "LatencyMS": { + "page_p50": 5.297902, + "page_p95": 315.829021, + "page_p99": 805.774962, + "search_p50": 44.51659, + "search_p95": 835.071745, + "search_p99": 1902.086433, + "updated_p50": 17.625467, + "updated_p95": 664.151381, + "updated_p99": 1832.257212 + } + }, + "recovery": { + "StartedAt": "2026-10-28T20:10:58.184279679Z", + "FinishedAt": "2026-10-28T20:12:58.179258109Z", + "Scheduled": 12000, + "Completed": 12000, + "Dropped": 0, + "DroppedPublishes": 0, + "ScheduledPublishes": 0, + "Status": { + "page:200": 11760, + "search:200": 120, + "updated:200": 120 + }, + "Errors": {}, + "LatencyMS": { + "page_p50": 2.758622, + "page_p95": 3.675634, + "page_p99": 5.42456, + "search_p50": 8.622515, + "search_p95": 26.852288, + "search_p99": 29.450764, + "updated_p50": 4.779784, + "updated_p95": 9.606068, + "updated_p99": 10.175718 + } + } + }, + "burst_drop_percent": 0.9158024691358025, + "app_acquisition_and_hold": [ + { + "pod": "mcp-registry-57d9d88679-52rg4", + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "listing_mean_ms": { + "mcp_registry_db_pool_acquire_duration": 134.69053044367593, + "mcp_registry_db_pool_checkout_duration": 16.002782964753937, + "mcp_registry_db_pool_after_query_duration": 0.0773305129588544, + "mcp_registry_db_query_duration": 15.27088754517947 + }, + "peak_sampled_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 2.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 30.0 + } + } + }, + { + "pod": "mcp-registry-57d9d88679-bzq4l", + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-zn6z", + "listing_mean_ms": { + "mcp_registry_db_pool_acquire_duration": 65.24364724882436, + "mcp_registry_db_pool_checkout_duration": 12.83627115214656, + "mcp_registry_db_pool_after_query_duration": 0.039367093832974105, + "mcp_registry_db_query_duration": 12.674925104854452 + }, + "peak_sampled_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 4.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 30.0 + } + } + }, + { + "pod": "mcp-registry-57d9d88679-sjl52", + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "listing_mean_ms": { + "mcp_registry_db_pool_acquire_duration": 28.887802061064285, + "mcp_registry_db_pool_checkout_duration": 12.766237444116337, + "mcp_registry_db_pool_after_query_duration": 0.04770569442915059, + "mcp_registry_db_query_duration": 12.599668013914567 + }, + "peak_sampled_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 3.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 30.0 + } + } + }, + { + "pod": "mcp-registry-57d9d88679-vfcxl", + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-zn6z", + "listing_mean_ms": { + "mcp_registry_db_pool_acquire_duration": 54.86705907310638, + "mcp_registry_db_pool_checkout_duration": 12.39986030275962, + "mcp_registry_db_pool_after_query_duration": 0.039343608479678945, + "mcp_registry_db_query_duration": 12.281173898344214 + }, + "peak_sampled_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 2.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 30.0 + } + } + } + ], + "app_cpu_pressure": [ + { + "pod": "mcp-registry-57d9d88679-52rg4", + "metric": "container_cpu_usage_seconds_total", + "seconds": 177.7170000076294, + "rate": 0.4621973980906368, + "start": 1793218057.983, + "end": 1793218235.7 + }, + { + "pod": "mcp-registry-57d9d88679-52rg4", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 177.7170000076294, + "rate": 0.15492261853856432, + "start": 1793218057.983, + "end": 1793218235.7 + }, + { + "pod": "mcp-registry-57d9d88679-52rg4", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 177.7170000076294, + "rate": 0.25454387029973485, + "start": 1793218057.983, + "end": 1793218235.7 + }, + { + "pod": "mcp-registry-57d9d88679-vfcxl", + "metric": "container_cpu_usage_seconds_total", + "seconds": 176.40199995040894, + "rate": 0.4234080170349387, + "start": 1793218056.57, + "end": 1793218232.972 + }, + { + "pod": "mcp-registry-57d9d88679-bzq4l", + "metric": "container_cpu_usage_seconds_total", + "seconds": 153.7590000629425, + "rate": 0.43019241782869694, + "start": 1793218057.166, + "end": 1793218210.925 + }, + { + "pod": "mcp-registry-57d9d88679-vfcxl", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 176.40199995040894, + "rate": 0.09445222279046714, + "start": 1793218056.57, + "end": 1793218232.972 + }, + { + "pod": "mcp-registry-57d9d88679-bzq4l", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 153.7590000629425, + "rate": 0.09256245159095655, + "start": 1793218057.166, + "end": 1793218210.925 + }, + { + "pod": "mcp-registry-57d9d88679-vfcxl", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 176.40199995040894, + "rate": 0.14315949369678024, + "start": 1793218056.57, + "end": 1793218232.972 + }, + { + "pod": "mcp-registry-57d9d88679-bzq4l", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 153.7590000629425, + "rate": 0.1405596744980965, + "start": 1793218057.166, + "end": 1793218210.925 + }, + { + "pod": "mcp-registry-57d9d88679-sjl52", + "metric": "container_cpu_usage_seconds_total", + "seconds": 125.23699998855591, + "rate": 0.45432400971916703, + "start": 1793218088.328, + "end": 1793218213.565 + }, + { + "pod": "mcp-registry-57d9d88679-sjl52", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 125.23699998855591, + "rate": 0.087224398548338, + "start": 1793218088.328, + "end": 1793218213.565 + }, + { + "pod": "mcp-registry-57d9d88679-sjl52", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 125.23699998855591, + "rate": 0.1408256266248123, + "start": 1793218088.328, + "end": 1793218213.565 + } + ], + "generator": { + "sample_intervals": 17, + "scope": "Whole sixteen-vCPU generator from /proc/stat and non-loopback network devices; includes monitoring overhead. CPU busy includes steal, reported separately. Does not alone rule out single-thread or network constraints.", + "cpu_busy_percent": { + "mean": 17.762639382590372, + "max": 22.59479786900658 + }, + "cpu_steal_percent": { + "mean": 0.18974919589702258, + "max": 0.38232528987778125 + }, + "cpu_iowait_percent": { + "mean": 0.0014846625138125438, + "max": 0.006331518298087881 + }, + "cpu_softirq_percent": { + "mean": 4.85494863927935, + "max": 5.954246317768725 + }, + "received_gbps": { + "mean": 2.1155602481364646, + "max": 2.168580031962932 + }, + "transmitted_gbps": { + "mean": 0.027411111776386228, + "max": 0.031550045743905214 + }, + "receive_drops": 0, + "transmit_drops": 0 + }, + "capture_count": 140, + "capture_failures": [], + "runtime_integrity": { + "as_of": 1793218723.269331, + "samples": 17, + "commands_failed": [], + "app_restarts": { + "mcp-registry-57d9d88679-52rg4/mcp-registry": 0, + "mcp-registry-57d9d88679-bzq4l/mcp-registry": 0, + "mcp-registry-57d9d88679-sjl52/mcp-registry": 0, + "mcp-registry-57d9d88679-vfcxl/mcp-registry": 0, + "registry-pg-1/postgres": 0 + }, + "node_count_timeline": [ + { + "at": 1793217912.826272, + "nodes": 4 + } + ], + "db_throttled_usec_range": [ + 0, + 0 + ], + "db_endpoint_samples": 90, + "db_endpoint_errors_or_empty": [], + "db_endpoint_not_ready": [], + "metric_window": { + "start": 1793217928.0, + "end": 1793218622.6379101 + }, + "metric_errors": [], + "empty_metrics": [ + "alerts", + "raw_5xx", + "raw_cancellations" + ], + "cadvisor_failures_or_empty": [], + "lb_collection_failures": [] + }, + "ingress_distribution": { + "scope": "Raw counter deltas at query timestamps 20:07:28\u201320:12:58 UTC. Scrape-aligned; includes surrounding traffic/recovery and any counted background work, not exact generator requests.", + "pods": [ + { + "pod": "ingress-nginx-controller-54dc7ccbb8-dfgrl", + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "counter_delta": 100064.0, + "share_percent": 11.414608883327192 + }, + { + "pod": "ingress-nginx-controller-54dc7ccbb8-mzmdk", + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "counter_delta": 118292.0, + "share_percent": 13.493933023130598 + }, + { + "pod": "ingress-nginx-controller-54dc7ccbb8-dlgmt", + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-zn6z", + "counter_delta": 110849.0, + "share_percent": 12.644887073352415 + }, + { + "pod": "ingress-nginx-controller-54dc7ccbb8-mkxsp", + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-zn6z", + "counter_delta": 106013.0, + "share_percent": 12.093229648506613 + }, + { + "pod": "ingress-nginx-controller-54dc7ccbb8-5zhms", + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "counter_delta": 265381.0, + "share_percent": 30.272828590364703 + }, + { + "pod": "ingress-nginx-controller-54dc7ccbb8-j9z7d", + "node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-ntsl", + "counter_delta": 176032.0, + "share_percent": 20.08051278131848 + } + ] + }, + "limitations": [ + "Warm 4-app/6-ingress capacity gate only: CPU HPA scale-up and natural 2-node recovery not tested.", + "Actual scheduler placed apps 1/1/2/0 across four nodes; previous four-app comparison used hard one-app-per-node placement.", + "DB resource change restarted PostgreSQL before preparation; cache state differs from previous tests.", + "15-second zero pending gauges do not imply zero waiting: acquire histogram means 29\u2013135 ms across pods, versus 12\u201316 ms checkout. Acquire time includes pool and scheduler delays.", + "App CPU full-stall rates cover unequal 125\u2013178-second intervals inside the burst.", + "Generator calibration excludes the external network; whole-VM spare CPU and zero NIC drops do not exclude a client or network limit.", + "Periodic drops overlap two 30-second mechanisms: generator idle-connection rotation and ingress cache TTL. Causality is unproven.", + "No publishes were generated; publish latency gate was not tested." + ], + "next_step": "Keep production unchanged. Use the captured evidence to isolate periodic cache-refill and connection-churn effects and uneven ingress placement before another capacity proposal; do not increase pools based on sampled zero queues.", + "restoration": { + "at": 1793218621.450293, + "production_changed": false, + "instances_absent": true, + "disks_absent": true, + "app_spec_restored": true, + "ingress_hpa_restored": true, + "db_resources_restored": true, + "whitelist_restored": true, + "verified_at": "2026-10-28T20:22:38.369288+00:00", + "response_hashes_matched": 48, + "corpus": { + "names": 52366, + "versions": 187108, + "benchmark_runs": null + }, + "final_snapshot_at": 1793218899.924099, + "observed_nodes": 4, + "observed_app_pods": 2, + "observed_ingress_pods": 5, + "natural_two_node_recovery_observed": false + } +} diff --git a/tools/ingressbench/evidence/delivery-validation-2026-10-28/cleanup.json b/tools/ingressbench/evidence/delivery-validation-2026-10-28/cleanup.json new file mode 100644 index 000000000..453c21649 --- /dev/null +++ b/tools/ingressbench/evidence/delivery-validation-2026-10-28/cleanup.json @@ -0,0 +1,13 @@ +{ + "at": 1793202439.1455748, + "production_changed": false, + "generator_instances_absent": true, + "generator_disks_absent": true, + "ingress_placement_restored": true, + "app_restored": true, + "replicas": 2, + "ready_replicas": 2, + "profiling_disabled": true, + "hpa_spec_restored": true, + "whitelist_restored": true +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/delivery-validation-2026-10-28/experiment.json b/tools/ingressbench/evidence/delivery-validation-2026-10-28/experiment.json new file mode 100644 index 000000000..92d676faf --- /dev/null +++ b/tools/ingressbench/evidence/delivery-validation-2026-10-28/experiment.json @@ -0,0 +1,42 @@ +{ + "recorded_at": 1793200558.415754, + "production_changed": false, + "app_image": "us-central1-docker.pkg.dev/mcp-registry-staging/registry-staging-experiments/registry:staging-replicas-20261028-v1@sha256:60fb8b7d7931a257542848460f7d1c76a99f16c46ddad795249fb5c25b1c9b0e", + "app_source_commit": "43e17e0993e1afed08950da75b461a1e4966fab3", + "calibration": { + "loopback_https_body_bytes": 65536, + "read_rate": 4500, + "seconds": 90, + "workers": 1024, + "connection_rotation_seconds": 30, + "scope": "Generator and fixture share CPU. Does not certify external network throughput.", + "first_generator_vcpus": 4, + "first_driver_sha256": "e930edbf92b5822499312b680bad20cc645b2ead753619faa546ae1bfb3c3e8c", + "second_generator_vcpus": 16, + "second_driver_sha256": "93de7e015c0fe51df8ff472f36a306678e1e48fa2c7e81a33b061b44c870550c", + "note": "Only helper extraction for lint separates instrumented binaries; traffic generation and connection rotation are unchanged." + }, + "staging_short": { + "app_replicas": 4, + "pool_max_each": 30, + "pool_min_each": 5, + "cpu_request_m": 400, + "db_cpu_quota": 3, + "nodes": 4, + "ingress_replicas": 6, + "burst_read_rate": 4500, + "updated_rate": 36, + "search_rate": 3, + "burst_seconds": 180, + "recovery_seconds": 120, + "driver_sha256": "93de7e015c0fe51df8ff472f36a306678e1e48fa2c7e81a33b061b44c870550c" + }, + "gates": { + "short_drops": 0, + "attempt_failures": 0, + "page_p95_ms_max": 1000, + "recovery_drops": 0, + "recovery_failures": 0, + "full_if_short_passes": "600-second burst with 1 publish/s; 1800-second recovery with publishes; publish p95 <=2000 ms; no dropped work/errors. Fixed app capacity only, not app-HPA behavior." + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/delivery-validation-2026-10-28/four-windows.json b/tools/ingressbench/evidence/delivery-validation-2026-10-28/four-windows.json new file mode 100644 index 000000000..25ffba044 --- /dev/null +++ b/tools/ingressbench/evidence/delivery-validation-2026-10-28/four-windows.json @@ -0,0 +1,1074 @@ +{ + "StartedAt": "2026-10-28T15:23:28.609381908Z", + "FinishedAt": "2026-10-28T15:26:28.647697129Z", + "Windows": [ + { + "StartedAt": "2026-10-28T15:23:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 6258, + "Dropped": 2467, + "Dispatched": 2778, + "Completed": 1754, + "Failures": 0, + "SchedulerLag": { + "Count": 6258, + "MeanMS": 1.1962897828379675, + "P95UpperMS": 10, + "MaximumMS": 27.8808 + }, + "QueueDelay": { + "Count": 2778, + "MeanMS": 344.68894627969706, + "P95UpperMS": 1000, + "MaximumMS": 754.080957 + }, + "HTTPDuration": { + "Count": 2778, + "MeanMS": 792.1700802325411, + "P95UpperMS": 5000, + "MaximumMS": 6553.068045 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:23:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 2445, + "Dispatched": 43568, + "Completed": 43778, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.0999388625555565, + "P95UpperMS": 0.5, + "MaximumMS": 2.485503 + }, + "QueueDelay": { + "Count": 43568, + "MeanMS": 112.57574358423568, + "P95UpperMS": 500, + "MaximumMS": 456.990125 + }, + "HTTPDuration": { + "Count": 43568, + "MeanMS": 245.29920472922782, + "P95UpperMS": 5000, + "MaximumMS": 6206.963706 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:23:40Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 44999, + "Completed": 45590, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.11911566206666718, + "P95UpperMS": 0.5, + "MaximumMS": 2.878237 + }, + "QueueDelay": { + "Count": 44999, + "MeanMS": 0.016883423409409015, + "P95UpperMS": 0.1, + "MaximumMS": 1.427286 + }, + "HTTPDuration": { + "Count": 44999, + "MeanMS": 88.00946620851676, + "P95UpperMS": 50, + "MaximumMS": 5375.274706 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:23:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 793, + "Dispatched": 43419, + "Completed": 42618, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.10942956086666808, + "P95UpperMS": 0.5, + "MaximumMS": 3.525654 + }, + "QueueDelay": { + "Count": 43419, + "MeanMS": 20.576596623851188, + "P95UpperMS": 500, + "MaximumMS": 402.274435 + }, + "HTTPDuration": { + "Count": 43419, + "MeanMS": 46.798856598263555, + "P95UpperMS": 500, + "MaximumMS": 2030.765135 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:24:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45789, + "Completed": 46775, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.1044585954666676, + "P95UpperMS": 0.5, + "MaximumMS": 3.376212 + }, + "QueueDelay": { + "Count": 45789, + "MeanMS": 4.1880877453536876, + "P95UpperMS": 0.5, + "MaximumMS": 190.890151 + }, + "HTTPDuration": { + "Count": 45789, + "MeanMS": 25.527673308523884, + "P95UpperMS": 200, + "MaximumMS": 1756.518144 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:24:10Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 804, + "Dispatched": 44196, + "Completed": 43921, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.08861496448888924, + "P95UpperMS": 0.5, + "MaximumMS": 2.195086 + }, + "QueueDelay": { + "Count": 44196, + "MeanMS": 22.173861193252744, + "P95UpperMS": 200, + "MaximumMS": 406.131607 + }, + "HTTPDuration": { + "Count": 44196, + "MeanMS": 75.07237891770711, + "P95UpperMS": 500, + "MaximumMS": 4752.274229 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:24:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 584, + "Dispatched": 44101, + "Completed": 43390, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.09587286648888935, + "P95UpperMS": 0.5, + "MaximumMS": 2.879563 + }, + "QueueDelay": { + "Count": 44101, + "MeanMS": 20.088790343665654, + "P95UpperMS": 200, + "MaximumMS": 357.255395 + }, + "HTTPDuration": { + "Count": 44101, + "MeanMS": 85.99740665717373, + "P95UpperMS": 500, + "MaximumMS": 6266.298104 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:24:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45312, + "Completed": 45979, + "Failures": 0, + "SchedulerLag": { + "Count": 44997, + "MeanMS": 0.09036718270106933, + "P95UpperMS": 0.5, + "MaximumMS": 2.509684 + }, + "QueueDelay": { + "Count": 45312, + "MeanMS": 0.7154708922139861, + "P95UpperMS": 0.1, + "MaximumMS": 69.845612 + }, + "HTTPDuration": { + "Count": 45312, + "MeanMS": 96.97038195063114, + "P95UpperMS": 200, + "MaximumMS": 5797.482847 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:24:40Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45003, + "Completed": 45066, + "Failures": 0, + "SchedulerLag": { + "Count": 45003, + "MeanMS": 0.09446618272115191, + "P95UpperMS": 0.5, + "MaximumMS": 1.952753 + }, + "QueueDelay": { + "Count": 45003, + "MeanMS": 0.011639698997844542, + "P95UpperMS": 0.1, + "MaximumMS": 0.532681 + }, + "HTTPDuration": { + "Count": 45003, + "MeanMS": 58.635125494677894, + "P95UpperMS": 200, + "MaximumMS": 3939.321287 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:24:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 293, + "Dispatched": 44359, + "Completed": 43629, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.10277615415555638, + "P95UpperMS": 0.5, + "MaximumMS": 3.766363 + }, + "QueueDelay": { + "Count": 44359, + "MeanMS": 19.078144722536543, + "P95UpperMS": 200, + "MaximumMS": 291.077221 + }, + "HTTPDuration": { + "Count": 44359, + "MeanMS": 83.53344506075418, + "P95UpperMS": 500, + "MaximumMS": 6208.242948 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:25:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45348, + "Completed": 45652, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.11334768446666668, + "P95UpperMS": 0.5, + "MaximumMS": 2.466323 + }, + "QueueDelay": { + "Count": 45348, + "MeanMS": 1.1931045344667783, + "P95UpperMS": 0.5, + "MaximumMS": 79.575803 + }, + "HTTPDuration": { + "Count": 45348, + "MeanMS": 189.64683763817598, + "P95UpperMS": 500, + "MaximumMS": 10000.660131 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:25:10Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 44999, + "Completed": 44895, + "Failures": 5, + "SchedulerLag": { + "Count": 44999, + "MeanMS": 0.08969003068957117, + "P95UpperMS": 0.5, + "MaximumMS": 2.925956 + }, + "QueueDelay": { + "Count": 44999, + "MeanMS": 1.4990034844552123, + "P95UpperMS": 0.1, + "MaximumMS": 87.691783 + }, + "HTTPDuration": { + "Count": 44999, + "MeanMS": 190.1044539136428, + "P95UpperMS": 100, + "MaximumMS": 10000.358585 + }, + "FailureClasses": { + "request:timeout": 5 + } + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:25:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45001, + "Completed": 45173, + "Failures": 4, + "SchedulerLag": { + "Count": 45001, + "MeanMS": 0.08046396246750034, + "P95UpperMS": 0.5, + "MaximumMS": 2.390058 + }, + "QueueDelay": { + "Count": 45001, + "MeanMS": 4.1512339282682555, + "P95UpperMS": 10, + "MaximumMS": 139.279955 + }, + "HTTPDuration": { + "Count": 45001, + "MeanMS": 137.00727651378847, + "P95UpperMS": 100, + "MaximumMS": 7780.191738 + }, + "FailureClasses": { + "request:timeout": 4 + } + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:25:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45000, + "Completed": 45636, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.07774425268889006, + "P95UpperMS": 0.5, + "MaximumMS": 1.836921 + }, + "QueueDelay": { + "Count": 45000, + "MeanMS": 0.008740614800000076, + "P95UpperMS": 0.1, + "MaximumMS": 1.041133 + }, + "HTTPDuration": { + "Count": 45000, + "MeanMS": 31.313626654533614, + "P95UpperMS": 50, + "MaximumMS": 3820.209697 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:25:40Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45000, + "Completed": 44998, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.07789617991111099, + "P95UpperMS": 0.5, + "MaximumMS": 1.311196 + }, + "QueueDelay": { + "Count": 45000, + "MeanMS": 0.008957294911111177, + "P95UpperMS": 0.1, + "MaximumMS": 0.859328 + }, + "HTTPDuration": { + "Count": 45000, + "MeanMS": 4.9160890228444, + "P95UpperMS": 20, + "MaximumMS": 376.451061 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:25:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 752, + "Dispatched": 44244, + "Completed": 43482, + "Failures": 0, + "SchedulerLag": { + "Count": 44997, + "MeanMS": 0.08843068295664175, + "P95UpperMS": 0.5, + "MaximumMS": 4.755934 + }, + "QueueDelay": { + "Count": 44244, + "MeanMS": 30.836867512928478, + "P95UpperMS": 500, + "MaximumMS": 327.975361 + }, + "HTTPDuration": { + "Count": 44244, + "MeanMS": 65.50647520321364, + "P95UpperMS": 500, + "MaximumMS": 1676.019962 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:26:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45004, + "Completed": 45726, + "Failures": 0, + "SchedulerLag": { + "Count": 45003, + "MeanMS": 0.08925479874675026, + "P95UpperMS": 0.5, + "MaximumMS": 3.009371 + }, + "QueueDelay": { + "Count": 45004, + "MeanMS": 0.011300005088436628, + "P95UpperMS": 0.1, + "MaximumMS": 2.75889 + }, + "HTTPDuration": { + "Count": 45004, + "MeanMS": 14.82189149191178, + "P95UpperMS": 100, + "MaximumMS": 386.019857 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:26:10Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 44123, + "Completed": 43157, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.08358569806666735, + "P95UpperMS": 0.5, + "MaximumMS": 3.014807 + }, + "QueueDelay": { + "Count": 44123, + "MeanMS": 0.37814765151961577, + "P95UpperMS": 0.1, + "MaximumMS": 194.877006 + }, + "HTTPDuration": { + "Count": 44123, + "MeanMS": 29.29785400802284, + "P95UpperMS": 100, + "MaximumMS": 1347.789712 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:26:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 38742, + "Dropped": 503, + "Dispatched": 39116, + "Completed": 40140, + "Failures": 0, + "SchedulerLag": { + "Count": 38742, + "MeanMS": 0.08238797413659651, + "P95UpperMS": 0.5, + "MaximumMS": 1.785179 + }, + "QueueDelay": { + "Count": 39116, + "MeanMS": 16.403023870206624, + "P95UpperMS": 200, + "MaximumMS": 339.111737 + }, + "HTTPDuration": { + "Count": 39116, + "MeanMS": 24.084918347479455, + "P95UpperMS": 200, + "MaximumMS": 947.811766 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + } + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/delivery-validation-2026-10-28/isolated-windows.json b/tools/ingressbench/evidence/delivery-validation-2026-10-28/isolated-windows.json new file mode 100644 index 000000000..8260dbcc8 --- /dev/null +++ b/tools/ingressbench/evidence/delivery-validation-2026-10-28/isolated-windows.json @@ -0,0 +1,1070 @@ +{ + "StartedAt": "2026-10-28T15:40:51.278207988Z", + "FinishedAt": "2026-10-28T15:43:51.346585427Z", + "Windows": [ + { + "StartedAt": "2026-10-28T15:40:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 39249, + "Dropped": 7196, + "Dispatched": 31047, + "Completed": 30023, + "Failures": 0, + "SchedulerLag": { + "Count": 39249, + "MeanMS": 0.23694685194527232, + "P95UpperMS": 0.5, + "MaximumMS": 28.571824 + }, + "QueueDelay": { + "Count": 31047, + "MeanMS": 267.9016955153477, + "P95UpperMS": 500, + "MaximumMS": 843.726757 + }, + "HTTPDuration": { + "Count": 31047, + "MeanMS": 323.11490179050924, + "P95UpperMS": 5000, + "MaximumMS": 5741.803288 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:41:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 46005, + "Completed": 46548, + "Failures": 0, + "SchedulerLag": { + "Count": 44999, + "MeanMS": 0.09242296011022533, + "P95UpperMS": 0.5, + "MaximumMS": 3.068366 + }, + "QueueDelay": { + "Count": 46005, + "MeanMS": 27.525343581676065, + "P95UpperMS": 200, + "MaximumMS": 239.519212 + }, + "HTTPDuration": { + "Count": 46005, + "MeanMS": 154.69569979867393, + "P95UpperMS": 2000, + "MaximumMS": 5102.070426 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:41:10Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 44999, + "Completed": 45456, + "Failures": 0, + "SchedulerLag": { + "Count": 44999, + "MeanMS": 0.10204691604257707, + "P95UpperMS": 0.5, + "MaximumMS": 2.68454 + }, + "QueueDelay": { + "Count": 44999, + "MeanMS": 0.013361254827885059, + "P95UpperMS": 0.1, + "MaximumMS": 1.20332 + }, + "HTTPDuration": { + "Count": 44999, + "MeanMS": 18.31512340596446, + "P95UpperMS": 100, + "MaximumMS": 1021.694849 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:41:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 1748, + "Dispatched": 43253, + "Completed": 43213, + "Failures": 0, + "SchedulerLag": { + "Count": 45001, + "MeanMS": 0.12890728130486107, + "P95UpperMS": 0.5, + "MaximumMS": 4.344692 + }, + "QueueDelay": { + "Count": 43253, + "MeanMS": 33.09220697718073, + "P95UpperMS": 500, + "MaximumMS": 576.621221 + }, + "HTTPDuration": { + "Count": 43253, + "MeanMS": 63.028649035281404, + "P95UpperMS": 500, + "MaximumMS": 2484.104716 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:41:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45000, + "Completed": 45039, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.10491457717777834, + "P95UpperMS": 0.5, + "MaximumMS": 2.620427 + }, + "QueueDelay": { + "Count": 45000, + "MeanMS": 0.014162152155555668, + "P95UpperMS": 0.1, + "MaximumMS": 2.298878 + }, + "HTTPDuration": { + "Count": 45000, + "MeanMS": 15.147153296200003, + "P95UpperMS": 100, + "MaximumMS": 460.411642 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:41:40Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 1670, + "Dispatched": 43329, + "Completed": 43279, + "Failures": 0, + "SchedulerLag": { + "Count": 44999, + "MeanMS": 0.11290053356741217, + "P95UpperMS": 0.5, + "MaximumMS": 2.906229 + }, + "QueueDelay": { + "Count": 43329, + "MeanMS": 33.774979440420594, + "P95UpperMS": 500, + "MaximumMS": 585.345505 + }, + "HTTPDuration": { + "Count": 43329, + "MeanMS": 76.393681196727, + "P95UpperMS": 500, + "MaximumMS": 2144.785434 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:41:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 2108, + "Dispatched": 42893, + "Completed": 42924, + "Failures": 0, + "SchedulerLag": { + "Count": 45002, + "MeanMS": 0.10478721539042646, + "P95UpperMS": 0.5, + "MaximumMS": 3.038751 + }, + "QueueDelay": { + "Count": 42893, + "MeanMS": 33.373711003286985, + "P95UpperMS": 500, + "MaximumMS": 679.586754 + }, + "HTTPDuration": { + "Count": 42893, + "MeanMS": 76.16084315851117, + "P95UpperMS": 500, + "MaximumMS": 1877.883515 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:42:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45000, + "Completed": 45003, + "Failures": 0, + "SchedulerLag": { + "Count": 44999, + "MeanMS": 0.08202442723171623, + "P95UpperMS": 0.5, + "MaximumMS": 1.582499 + }, + "QueueDelay": { + "Count": 45000, + "MeanMS": 0.009604190533333404, + "P95UpperMS": 0.1, + "MaximumMS": 0.47286 + }, + "HTTPDuration": { + "Count": 45000, + "MeanMS": 9.80160499617776, + "P95UpperMS": 50, + "MaximumMS": 304.536527 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:42:10Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 1987, + "Dispatched": 43014, + "Completed": 42986, + "Failures": 0, + "SchedulerLag": { + "Count": 45001, + "MeanMS": 0.1630445780760422, + "P95UpperMS": 1, + "MaximumMS": 12.70322 + }, + "QueueDelay": { + "Count": 43014, + "MeanMS": 25.406581713813992, + "P95UpperMS": 500, + "MaximumMS": 563.098717 + }, + "HTTPDuration": { + "Count": 43014, + "MeanMS": 60.832514259636305, + "P95UpperMS": 500, + "MaximumMS": 2912.710538 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:42:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 1614, + "Dispatched": 43385, + "Completed": 43365, + "Failures": 0, + "SchedulerLag": { + "Count": 44999, + "MeanMS": 0.14075949796662163, + "P95UpperMS": 1, + "MaximumMS": 13.84874 + }, + "QueueDelay": { + "Count": 43385, + "MeanMS": 26.565664624248125, + "P95UpperMS": 200, + "MaximumMS": 585.999484 + }, + "HTTPDuration": { + "Count": 43385, + "MeanMS": 69.55050351923455, + "P95UpperMS": 500, + "MaximumMS": 1910.644942 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:42:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 44998, + "Completed": 45021, + "Failures": 0, + "SchedulerLag": { + "Count": 44998, + "MeanMS": 0.1156336339837319, + "P95UpperMS": 0.5, + "MaximumMS": 3.857241 + }, + "QueueDelay": { + "Count": 44998, + "MeanMS": 0.017027878639050816, + "P95UpperMS": 0.1, + "MaximumMS": 1.402031 + }, + "HTTPDuration": { + "Count": 44998, + "MeanMS": 10.88512733988178, + "P95UpperMS": 50, + "MaximumMS": 360.06126 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:42:40Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 2306, + "Dispatched": 42696, + "Completed": 42584, + "Failures": 0, + "SchedulerLag": { + "Count": 45002, + "MeanMS": 0.10578774201146632, + "P95UpperMS": 0.5, + "MaximumMS": 2.247626 + }, + "QueueDelay": { + "Count": 42696, + "MeanMS": 43.525726370269, + "P95UpperMS": 500, + "MaximumMS": 681.743083 + }, + "HTTPDuration": { + "Count": 42696, + "MeanMS": 76.39810315905466, + "P95UpperMS": 500, + "MaximumMS": 2315.79876 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:42:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 1733, + "Dispatched": 43267, + "Completed": 43384, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.11967690206666753, + "P95UpperMS": 0.5, + "MaximumMS": 14.718237 + }, + "QueueDelay": { + "Count": 43267, + "MeanMS": 31.55386523877318, + "P95UpperMS": 500, + "MaximumMS": 595.017865 + }, + "HTTPDuration": { + "Count": 43267, + "MeanMS": 69.19720636709208, + "P95UpperMS": 500, + "MaximumMS": 2085.598226 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:43:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45000, + "Completed": 45005, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.12652255675555674, + "P95UpperMS": 0.5, + "MaximumMS": 3.001717 + }, + "QueueDelay": { + "Count": 45000, + "MeanMS": 0.01918081244444448, + "P95UpperMS": 0.1, + "MaximumMS": 1.00523 + }, + "HTTPDuration": { + "Count": 45000, + "MeanMS": 12.584605865355545, + "P95UpperMS": 50, + "MaximumMS": 436.46336 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:43:10Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45000, + "Completed": 44987, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.11444188684444381, + "P95UpperMS": 0.5, + "MaximumMS": 4.045038 + }, + "QueueDelay": { + "Count": 45000, + "MeanMS": 0.01707995795555549, + "P95UpperMS": 0.1, + "MaximumMS": 1.683731 + }, + "HTTPDuration": { + "Count": 45000, + "MeanMS": 15.282199215577796, + "P95UpperMS": 100, + "MaximumMS": 343.194404 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:43:20Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 1472, + "Dispatched": 43528, + "Completed": 43530, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.1519738513555558, + "P95UpperMS": 1, + "MaximumMS": 9.846824 + }, + "QueueDelay": { + "Count": 43528, + "MeanMS": 35.55944625767326, + "P95UpperMS": 500, + "MaximumMS": 518.016373 + }, + "HTTPDuration": { + "Count": 43528, + "MeanMS": 61.45078859564044, + "P95UpperMS": 500, + "MaximumMS": 2072.29029 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:43:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 0, + "Dispatched": 45000, + "Completed": 45016, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.11032862704444484, + "P95UpperMS": 0.5, + "MaximumMS": 2.436817 + }, + "QueueDelay": { + "Count": 45000, + "MeanMS": 0.01598874695555542, + "P95UpperMS": 0.1, + "MaximumMS": 0.896042 + }, + "HTTPDuration": { + "Count": 45000, + "MeanMS": 12.480536168399976, + "P95UpperMS": 50, + "MaximumMS": 353.623227 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:43:40Z", + "Seconds": 10, + "Reads": { + "Scheduled": 45000, + "Dropped": 1592, + "Dispatched": 43408, + "Completed": 43395, + "Failures": 0, + "SchedulerLag": { + "Count": 45000, + "MeanMS": 0.11044280104444588, + "P95UpperMS": 0.5, + "MaximumMS": 6.690093 + }, + "QueueDelay": { + "Count": 43408, + "MeanMS": 27.14520158192075, + "P95UpperMS": 200, + "MaximumMS": 576.174975 + }, + "HTTPDuration": { + "Count": 43408, + "MeanMS": 58.570190370116336, + "P95UpperMS": 500, + "MaximumMS": 2263.424763 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T15:43:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 5751, + "Dropped": 0, + "Dispatched": 5752, + "Completed": 5816, + "Failures": 0, + "SchedulerLag": { + "Count": 5752, + "MeanMS": 0.09818765281641167, + "P95UpperMS": 0.5, + "MaximumMS": 1.699756 + }, + "QueueDelay": { + "Count": 5752, + "MeanMS": 0.012531547809457548, + "P95UpperMS": 0.1, + "MaximumMS": 0.481061 + }, + "HTTPDuration": { + "Count": 5752, + "MeanMS": 9.017332272774716, + "P95UpperMS": 50, + "MaximumMS": 171.262717 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + } + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/delivery-validation-2026-10-28/isolation-layout.json b/tools/ingressbench/evidence/delivery-validation-2026-10-28/isolation-layout.json new file mode 100644 index 000000000..a052ef2cc --- /dev/null +++ b/tools/ingressbench/evidence/delivery-validation-2026-10-28/isolation-layout.json @@ -0,0 +1,17 @@ +{ + "at": 1793201571.074346, + "db_node": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "workload_nodes": [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-2846", + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-d57s" + ], + "postgres_pod_uid": "c3c2cffa-78bc-45cd-9337-5ebaef84c83a", + "counts": { + "nodes": 4, + "apps": 4, + "ingress": 6 + }, + "postgres_restarted": false, + "app_and_ingress_resources_unchanged": true +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/delivery-validation-2026-10-28/restoration-verification.json b/tools/ingressbench/evidence/delivery-validation-2026-10-28/restoration-verification.json new file mode 100644 index 000000000..ba33a7015 --- /dev/null +++ b/tools/ingressbench/evidence/delivery-validation-2026-10-28/restoration-verification.json @@ -0,0 +1,13 @@ +{ + "at": 1793202671.212588, + "nodes": 4, + "ingress_hpa": [ + { + "minimum": 2, + "maximum": 6, + "current": 6, + "desired": 6 + } + ], + "candidate_scale_down_tested": false +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/delivery-validation-2026-10-28/summary.json b/tools/ingressbench/evidence/delivery-validation-2026-10-28/summary.json new file mode 100644 index 000000000..ffc8ae927 --- /dev/null +++ b/tools/ingressbench/evidence/delivery-validation-2026-10-28/summary.json @@ -0,0 +1,1025 @@ +{ + "production_changed": false, + "generator_calibration": { + "calibration": { + "vcpus": 4, + "scheduled": 405000, + "dropped": 25918, + "failures": 0, + "page_p95_ms": 63.858946, + "scope": "Loopback HTTPS fixture and client share host; not an external-network calibration." + }, + "calibration16": { + "vcpus": 16, + "scheduled": 405000, + "dropped": 0, + "failures": 0, + "page_p95_ms": 1.551173, + "scope": "Loopback HTTPS fixture and client share host; not an external-network calibration." + } + }, + "staging": { + "four": { + "scheduled": 810000, + "dropped": 8641, + "drop_percent": 1.0667901234567903, + "attempt_failures": 9, + "latency_ms": { + "page_p50": 5.093946, + "page_p95": 328.161218, + "page_p99": 2185.814363, + "search_p50": 46.794415, + "search_p95": 3369.474513, + "search_p99": 7276.583135, + "updated_p50": 20.981621, + "updated_p95": 3935.097604, + "updated_p99": 7347.887684 + }, + "max_scheduler_lag_ms": 27.8808, + "request_phases": { + "page:HIT": { + "Attempts": 703415, + "NewConnections": 1857, + "ReusedConnections": 701558, + "SchedulerLag": { + "Count": 703415, + "MeanMS": 0.09559757971041007, + "P95UpperMS": 0.5, + "MaximumMS": 26.556883 + }, + "QueueDelay": { + "Count": 703415, + "MeanMS": 13.16012722082664, + "P95UpperMS": 200, + "MaximumMS": 754.029742 + }, + "ConnectionWait": { + "Count": 703415, + "MeanMS": 1.4793695605865693, + "P95UpperMS": 0.1, + "MaximumMS": 488.751885 + }, + "TLS": { + "Count": 7450, + "MeanMS": 201.9859112740944, + "P95UpperMS": 500, + "MaximumMS": 1480.879771 + }, + "FirstByte": { + "Count": 703415, + "MeanMS": 51.68760104336966, + "P95UpperMS": 100, + "MaximumMS": 9262.328305 + }, + "Body": { + "Count": 703415, + "MeanMS": 5.529591125163679, + "P95UpperMS": 50, + "MaximumMS": 2144.003423 + }, + "HTTP": { + "Count": 703415, + "MeanMS": 58.75527569210267, + "P95UpperMS": 200, + "MaximumMS": 9281.393604 + } + }, + "page:MISS": { + "Attempts": 21057, + "NewConnections": 219, + "ReusedConnections": 20838, + "SchedulerLag": { + "Count": 21057, + "MeanMS": 0.14837896894144526, + "P95UpperMS": 0.5, + "MaximumMS": 23.239782 + }, + "QueueDelay": { + "Count": 21057, + "MeanMS": 50.730909462554145, + "P95UpperMS": 500, + "MaximumMS": 753.709931 + }, + "ConnectionWait": { + "Count": 21057, + "MeanMS": 4.764172574535824, + "P95UpperMS": 0.1, + "MaximumMS": 489.25387 + }, + "TLS": { + "Count": 421, + "MeanMS": 269.420130608076, + "P95UpperMS": 1000, + "MaximumMS": 1790.087696 + }, + "FirstByte": { + "Count": 21057, + "MeanMS": 758.7289245796601, + "P95UpperMS": 5000, + "MaximumMS": 8564.778974 + }, + "Body": { + "Count": 21057, + "MeanMS": 24.31324414579464, + "P95UpperMS": 200, + "MaximumMS": 3660.267921 + }, + "HTTP": { + "Count": 21057, + "MeanMS": 788.2197410107823, + "P95UpperMS": 5000, + "MaximumMS": 8698.875364 + } + }, + "page:STALE": { + "Attempts": 58567, + "NewConnections": 444, + "ReusedConnections": 58123, + "SchedulerLag": { + "Count": 58567, + "MeanMS": 0.11176806363651737, + "P95UpperMS": 0.5, + "MaximumMS": 27.441633 + }, + "QueueDelay": { + "Count": 58567, + "MeanMS": 11.98553520499605, + "P95UpperMS": 100, + "MaximumMS": 753.553777 + }, + "ConnectionWait": { + "Count": 58567, + "MeanMS": 3.3991704405894447, + "P95UpperMS": 0.1, + "MaximumMS": 490.049992 + }, + "TLS": { + "Count": 979, + "MeanMS": 208.58922298569925, + "P95UpperMS": 500, + "MaximumMS": 1169.37919 + }, + "FirstByte": { + "Count": 58567, + "MeanMS": 76.873753997268, + "P95UpperMS": 200, + "MaximumMS": 8420.714425 + }, + "Body": { + "Count": 58567, + "MeanMS": 10.082141246691819, + "P95UpperMS": 50, + "MaximumMS": 1466.628809 + }, + "HTTP": { + "Count": 58567, + "MeanMS": 90.5382506042303, + "P95UpperMS": 200, + "MaximumMS": 8421.607791 + } + }, + "page:UNKNOWN": { + "Attempts": 4, + "NewConnections": 0, + "ReusedConnections": 4, + "SchedulerLag": { + "Count": 4, + "MeanMS": 0.0921355, + "P95UpperMS": 0.5, + "MaximumMS": 0.30834 + }, + "QueueDelay": { + "Count": 4, + "MeanMS": 7.77792925, + "P95UpperMS": 50, + "MaximumMS": 31.086732 + }, + "ConnectionWait": { + "Count": 4, + "MeanMS": 0.00470875, + "P95UpperMS": 0.1, + "MaximumMS": 0.006859 + }, + "TLS": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FirstByte": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "Body": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTP": { + "Count": 4, + "MeanMS": 10000.1831235, + "P95UpperMS": 20000, + "MaximumMS": 10000.224879 + } + }, + "page:UPDATING": { + "Attempts": 11383, + "NewConnections": 150, + "ReusedConnections": 11233, + "SchedulerLag": { + "Count": 11383, + "MeanMS": 0.2320675629447411, + "P95UpperMS": 0.5, + "MaximumMS": 27.8808 + }, + "QueueDelay": { + "Count": 11383, + "MeanMS": 78.84827231872121, + "P95UpperMS": 500, + "MaximumMS": 754.080957 + }, + "ConnectionWait": { + "Count": 11383, + "MeanMS": 7.4187985848194415, + "P95UpperMS": 0.1, + "MaximumMS": 488.585684 + }, + "TLS": { + "Count": 261, + "MeanMS": 270.0277648888892, + "P95UpperMS": 1000, + "MaximumMS": 1056.667149 + }, + "FirstByte": { + "Count": 11383, + "MeanMS": 94.2967268617237, + "P95UpperMS": 500, + "MaximumMS": 8060.916263 + }, + "Body": { + "Count": 11383, + "MeanMS": 9.930635858912394, + "P95UpperMS": 50, + "MaximumMS": 1470.614972 + }, + "HTTP": { + "Count": 11383, + "MeanMS": 112.62244453043985, + "P95UpperMS": 500, + "MaximumMS": 8065.360931 + } + }, + "search:MISS": { + "Attempts": 433, + "NewConnections": 8, + "ReusedConnections": 425, + "SchedulerLag": { + "Count": 433, + "MeanMS": 0.10812153117782916, + "P95UpperMS": 0.5, + "MaximumMS": 1.002513 + }, + "QueueDelay": { + "Count": 433, + "MeanMS": 11.746329974595838, + "P95UpperMS": 200, + "MaximumMS": 376.515776 + }, + "ConnectionWait": { + "Count": 433, + "MeanMS": 5.1888199676674365, + "P95UpperMS": 2, + "MaximumMS": 373.737209 + }, + "TLS": { + "Count": 18, + "MeanMS": 158.4044611666667, + "P95UpperMS": 500, + "MaximumMS": 391.042075 + }, + "FirstByte": { + "Count": 433, + "MeanMS": 614.3257287113161, + "P95UpperMS": 5000, + "MaximumMS": 7957.628255 + }, + "Body": { + "Count": 433, + "MeanMS": 26.31506489838339, + "P95UpperMS": 200, + "MaximumMS": 650.060939 + }, + "HTTP": { + "Count": 433, + "MeanMS": 645.8886304064661, + "P95UpperMS": 5000, + "MaximumMS": 7958.687092 + } + }, + "search:STALE": { + "Attempts": 103, + "NewConnections": 0, + "ReusedConnections": 103, + "SchedulerLag": { + "Count": 103, + "MeanMS": 0.1208896699029126, + "P95UpperMS": 0.5, + "MaximumMS": 0.592884 + }, + "QueueDelay": { + "Count": 103, + "MeanMS": 10.15727988349515, + "P95UpperMS": 100, + "MaximumMS": 283.901355 + }, + "ConnectionWait": { + "Count": 103, + "MeanMS": 3.461204796116505, + "P95UpperMS": 0.5, + "MaximumMS": 164.079407 + }, + "TLS": { + "Count": 5, + "MeanMS": 160.8513514, + "P95UpperMS": 500, + "MaximumMS": 446.563731 + }, + "FirstByte": { + "Count": 103, + "MeanMS": 34.433826485436896, + "P95UpperMS": 200, + "MaximumMS": 733.750659 + }, + "Body": { + "Count": 103, + "MeanMS": 12.139617970873784, + "P95UpperMS": 50, + "MaximumMS": 406.742953 + }, + "HTTP": { + "Count": 103, + "MeanMS": 50.074065262135925, + "P95UpperMS": 500, + "MaximumMS": 939.534804 + } + }, + "updated:MISS": { + "Attempts": 6392, + "NewConnections": 16, + "ReusedConnections": 6376, + "SchedulerLag": { + "Count": 6392, + "MeanMS": 0.09532567787859836, + "P95UpperMS": 0.5, + "MaximumMS": 10.261938 + }, + "QueueDelay": { + "Count": 6392, + "MeanMS": 14.823988297715916, + "P95UpperMS": 200, + "MaximumMS": 748.340805 + }, + "ConnectionWait": { + "Count": 6392, + "MeanMS": 1.463632779411763, + "P95UpperMS": 0.1, + "MaximumMS": 356.682789 + }, + "TLS": { + "Count": 62, + "MeanMS": 248.8275058387097, + "P95UpperMS": 1000, + "MaximumMS": 1021.45693 + }, + "FirstByte": { + "Count": 6392, + "MeanMS": 575.4583857393623, + "P95UpperMS": 5000, + "MaximumMS": 8139.201591 + }, + "Body": { + "Count": 6392, + "MeanMS": 11.57594532086981, + "P95UpperMS": 100, + "MaximumMS": 1913.841676 + }, + "HTTP": { + "Count": 6392, + "MeanMS": 588.5375524105149, + "P95UpperMS": 5000, + "MaximumMS": 8141.13393 + } + }, + "updated:UNKNOWN": { + "Attempts": 5, + "NewConnections": 0, + "ReusedConnections": 5, + "SchedulerLag": { + "Count": 5, + "MeanMS": 0.0508626, + "P95UpperMS": 0.1, + "MaximumMS": 0.065922 + }, + "QueueDelay": { + "Count": 5, + "MeanMS": 0.013286799999999998, + "P95UpperMS": 0.1, + "MaximumMS": 0.015264 + }, + "ConnectionWait": { + "Count": 5, + "MeanMS": 0.004341599999999999, + "P95UpperMS": 0.1, + "MaximumMS": 0.005487 + }, + "TLS": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FirstByte": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "Body": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTP": { + "Count": 5, + "MeanMS": 10000.298772400001, + "P95UpperMS": 20000, + "MaximumMS": 10000.660131 + } + } + }, + "recovery": { + "scheduled": 12000, + "dropped": 0, + "attempt_failures": 0 + }, + "replicas": { + "mcp-registry-7c9c464c69-hv928": { + "peak_sampled_pool_pending": 0.0, + "final_recovery_pool_pending": 0.0, + "page_acquire_mean_ms": 4.111069506252242, + "page_checkout_mean_ms": 6.786923867044345, + "cpu_stalled_fraction": 0.08639270684330586 + }, + "mcp-registry-7c9c464c69-nv4g6": { + "peak_sampled_pool_pending": 0.0, + "final_recovery_pool_pending": 0.0, + "page_acquire_mean_ms": 203.43746100745457, + "page_checkout_mean_ms": 17.86458496839724, + "cpu_stalled_fraction": 0.18650880774561354 + }, + "mcp-registry-7c9c464c69-t5l8x": { + "peak_sampled_pool_pending": 758.0, + "final_recovery_pool_pending": 0.0, + "page_acquire_mean_ms": 1632.1689058727768, + "page_checkout_mean_ms": 128.27550334109935, + "cpu_stalled_fraction": 0.39102449658149263 + }, + "mcp-registry-7c9c464c69-tqztc": { + "peak_sampled_pool_pending": 0.0, + "final_recovery_pool_pending": 0.0, + "page_acquire_mean_ms": 0.9654956471038418, + "page_checkout_mean_ms": 4.983655437203411, + "cpu_stalled_fraction": 0.04159489470392535 + } + }, + "dominant_db_statement": { + "queryid": 308564448692527448, + "calls": 82727, + "mean_ms": 1.718417499244493, + "total_exec_ms": 142159.52445999917, + "block_reads": 0, + "temp_blocks_written": 0 + }, + "runtime_integrity": { + "as_of": 1793201429.364608, + "samples": 17, + "commands_failed": [], + "app_restarts": { + "mcp-registry-7c9c464c69-hv928/mcp-registry": 0, + "mcp-registry-7c9c464c69-nv4g6/mcp-registry": 0, + "mcp-registry-7c9c464c69-t5l8x/mcp-registry": 0, + "mcp-registry-7c9c464c69-tqztc/mcp-registry": 0, + "registry-pg-1/postgres": 0 + }, + "node_count_timeline": [ + { + "at": 1793200865.394613, + "nodes": 4 + } + ], + "db_throttled_usec_range": [ + 1017850, + 1017850 + ], + "db_endpoint_samples": 90, + "db_endpoint_errors_or_empty": [], + "db_endpoint_not_ready": [], + "cadvisor_failures_or_empty": [], + "lb_collection_failures": [] + }, + "captures": { + "total": 140, + "failed": 0 + }, + "generator": { + "sample_intervals": 17, + "scope": "Whole sixteen-vCPU generator from /proc/stat and non-loopback network devices; includes monitoring overhead. CPU busy includes steal, reported separately. Does not alone rule out single-thread or network constraints.", + "cpu_busy_percent": { + "mean": 18.37092867352658, + "max": 23.098931841302136 + }, + "cpu_steal_percent": { + "mean": 1.1643889680676884, + "max": 2.537975870475714 + }, + "cpu_iowait_percent": { + "mean": 0.0014868889822261993, + "max": 0.006358087487283825 + }, + "cpu_softirq_percent": { + "mean": 5.094784765916156, + "max": 5.979420253482243 + }, + "received_gbps": { + "mean": 2.118015801288082, + "max": 2.1575673933278035 + }, + "transmitted_gbps": { + "mean": 0.02563179294636198, + "max": 0.03150955174321181 + }, + "receive_drops": 0, + "transmit_drops": 0 + } + }, + "isolated": { + "scheduled": 810000, + "dropped": 23426, + "drop_percent": 2.892098765432099, + "attempt_failures": 0, + "latency_ms": { + "page_p50": 8.973222, + "page_p95": 414.47619, + "page_p99": 1303.712822, + "search_p50": 55.074819, + "search_p95": 1635.270011, + "search_p99": 2883.021975, + "updated_p50": 31.432483, + "updated_p95": 1291.326802, + "updated_p99": 2943.624531 + }, + "max_scheduler_lag_ms": 28.571824, + "request_phases": { + "page:HIT": { + "Attempts": 693198, + "NewConnections": 2202, + "ReusedConnections": 690996, + "SchedulerLag": { + "Count": 693198, + "MeanMS": 0.11591246628668249, + "P95UpperMS": 0.5, + "MaximumMS": 28.353993 + }, + "QueueDelay": { + "Count": 693198, + "MeanMS": 26.21284182912156, + "P95UpperMS": 500, + "MaximumMS": 843.705121 + }, + "ConnectionWait": { + "Count": 693198, + "MeanMS": 2.912663961969009, + "P95UpperMS": 0.1, + "MaximumMS": 497.063069 + }, + "TLS": { + "Count": 9390, + "MeanMS": 294.66106904472787, + "P95UpperMS": 1000, + "MaximumMS": 1409.398886 + }, + "FirstByte": { + "Count": 693198, + "MeanMS": 29.958684159785363, + "P95UpperMS": 200, + "MaximumMS": 3881.420112 + }, + "Body": { + "Count": 693198, + "MeanMS": 7.716041042605739, + "P95UpperMS": 50, + "MaximumMS": 1653.30331 + }, + "HTTP": { + "Count": 693198, + "MeanMS": 40.65063486957314, + "P95UpperMS": 200, + "MaximumMS": 3882.216973 + } + }, + "page:MISS": { + "Attempts": 21024, + "NewConnections": 170, + "ReusedConnections": 20854, + "SchedulerLag": { + "Count": 21024, + "MeanMS": 0.13983545381468807, + "P95UpperMS": 0.5, + "MaximumMS": 23.767363 + }, + "QueueDelay": { + "Count": 21024, + "MeanMS": 80.59473598544541, + "P95UpperMS": 500, + "MaximumMS": 839.824585 + }, + "ConnectionWait": { + "Count": 21024, + "MeanMS": 4.4655820159817585, + "P95UpperMS": 0.1, + "MaximumMS": 497.040012 + }, + "TLS": { + "Count": 406, + "MeanMS": 333.574115684729, + "P95UpperMS": 1000, + "MaximumMS": 1022.92375 + }, + "FirstByte": { + "Count": 21024, + "MeanMS": 648.6164506669505, + "P95UpperMS": 5000, + "MaximumMS": 5734.192107 + }, + "Body": { + "Count": 21024, + "MeanMS": 23.664179374714603, + "P95UpperMS": 100, + "MaximumMS": 1242.409115 + }, + "HTTP": { + "Count": 21024, + "MeanMS": 676.8824378730507, + "P95UpperMS": 5000, + "MaximumMS": 5741.803288 + } + }, + "page:STALE": { + "Attempts": 61633, + "NewConnections": 516, + "ReusedConnections": 61117, + "SchedulerLag": { + "Count": 61633, + "MeanMS": 0.14472261588759258, + "P95UpperMS": 0.5, + "MaximumMS": 27.468609 + }, + "QueueDelay": { + "Count": 61633, + "MeanMS": 22.58462861473583, + "P95UpperMS": 200, + "MaximumMS": 843.452402 + }, + "ConnectionWait": { + "Count": 61633, + "MeanMS": 5.0853288288092475, + "P95UpperMS": 0.1, + "MaximumMS": 497.198184 + }, + "TLS": { + "Count": 1344, + "MeanMS": 291.3601996130947, + "P95UpperMS": 1000, + "MaximumMS": 1233.250228 + }, + "FirstByte": { + "Count": 61633, + "MeanMS": 37.06454117161279, + "P95UpperMS": 200, + "MaximumMS": 3344.516554 + }, + "Body": { + "Count": 61633, + "MeanMS": 13.597187478055853, + "P95UpperMS": 100, + "MaximumMS": 1647.027012 + }, + "HTTP": { + "Count": 61633, + "MeanMS": 55.854689980302744, + "P95UpperMS": 500, + "MaximumMS": 3348.377954 + } + }, + "page:UPDATING": { + "Attempts": 3889, + "NewConnections": 123, + "ReusedConnections": 3766, + "SchedulerLag": { + "Count": 3889, + "MeanMS": 0.30171256981229083, + "P95UpperMS": 1, + "MaximumMS": 28.571824 + }, + "QueueDelay": { + "Count": 3889, + "MeanMS": 168.84364120699476, + "P95UpperMS": 1000, + "MaximumMS": 843.726757 + }, + "ConnectionWait": { + "Count": 3889, + "MeanMS": 18.76399739496012, + "P95UpperMS": 200, + "MaximumMS": 494.77459 + }, + "TLS": { + "Count": 268, + "MeanMS": 354.0121233097016, + "P95UpperMS": 1000, + "MaximumMS": 1219.717566 + }, + "FirstByte": { + "Count": 3889, + "MeanMS": 124.37065554872763, + "P95UpperMS": 1000, + "MaximumMS": 2477.274246 + }, + "Body": { + "Count": 3889, + "MeanMS": 18.44644455232704, + "P95UpperMS": 100, + "MaximumMS": 655.571455 + }, + "HTTP": { + "Count": 3889, + "MeanMS": 161.9637800563131, + "P95UpperMS": 2000, + "MaximumMS": 2478.357431 + } + }, + "search:MISS": { + "Attempts": 449, + "NewConnections": 5, + "ReusedConnections": 444, + "SchedulerLag": { + "Count": 449, + "MeanMS": 0.13799816926503355, + "P95UpperMS": 0.5, + "MaximumMS": 22.226363 + }, + "QueueDelay": { + "Count": 449, + "MeanMS": 19.84802315144767, + "P95UpperMS": 500, + "MaximumMS": 434.350254 + }, + "ConnectionWait": { + "Count": 449, + "MeanMS": 5.045762153674832, + "P95UpperMS": 0.1, + "MaximumMS": 383.967091 + }, + "TLS": { + "Count": 16, + "MeanMS": 195.67339887499998, + "P95UpperMS": 1000, + "MaximumMS": 700.82349 + }, + "FirstByte": { + "Count": 449, + "MeanMS": 235.29386529621385, + "P95UpperMS": 2000, + "MaximumMS": 3219.84158 + }, + "Body": { + "Count": 449, + "MeanMS": 29.617935839643675, + "P95UpperMS": 200, + "MaximumMS": 1011.20417 + }, + "HTTP": { + "Count": 449, + "MeanMS": 269.9995350222715, + "P95UpperMS": 2000, + "MaximumMS": 3282.146917 + } + }, + "search:STALE": { + "Attempts": 69, + "NewConnections": 1, + "ReusedConnections": 68, + "SchedulerLag": { + "Count": 69, + "MeanMS": 0.13031698550724638, + "P95UpperMS": 0.5, + "MaximumMS": 0.673475 + }, + "QueueDelay": { + "Count": 69, + "MeanMS": 20.14004785507247, + "P95UpperMS": 200, + "MaximumMS": 531.23088 + }, + "ConnectionWait": { + "Count": 69, + "MeanMS": 3.962877565217393, + "P95UpperMS": 0.1, + "MaximumMS": 159.202573 + }, + "TLS": { + "Count": 2, + "MeanMS": 165.900665, + "P95UpperMS": 200, + "MaximumMS": 174.478227 + }, + "FirstByte": { + "Count": 69, + "MeanMS": 50.53309997101449, + "P95UpperMS": 200, + "MaximumMS": 699.646992 + }, + "Body": { + "Count": 69, + "MeanMS": 20.87254398550725, + "P95UpperMS": 100, + "MaximumMS": 357.89058 + }, + "HTTP": { + "Count": 69, + "MeanMS": 75.42026292753623, + "P95UpperMS": 500, + "MaximumMS": 1134.858894 + } + }, + "updated:MISS": { + "Attempts": 6312, + "NewConnections": 19, + "ReusedConnections": 6293, + "SchedulerLag": { + "Count": 6312, + "MeanMS": 0.11460095611533541, + "P95UpperMS": 0.5, + "MaximumMS": 13.734759 + }, + "QueueDelay": { + "Count": 6312, + "MeanMS": 29.48820675776302, + "P95UpperMS": 500, + "MaximumMS": 834.684376 + }, + "ConnectionWait": { + "Count": 6312, + "MeanMS": 3.041838082858048, + "P95UpperMS": 0.1, + "MaximumMS": 472.347651 + }, + "TLS": { + "Count": 84, + "MeanMS": 336.579743607143, + "P95UpperMS": 1000, + "MaximumMS": 1057.198819 + }, + "FirstByte": { + "Count": 6312, + "MeanMS": 181.80102889670454, + "P95UpperMS": 2000, + "MaximumMS": 5101.99475 + }, + "Body": { + "Count": 6312, + "MeanMS": 13.823816214828888, + "P95UpperMS": 100, + "MaximumMS": 868.393431 + }, + "HTTP": { + "Count": 6312, + "MeanMS": 198.7275450857109, + "P95UpperMS": 2000, + "MaximumMS": 5102.070426 + } + } + }, + "recovery": { + "scheduled": 12000, + "dropped": 0, + "attempt_failures": 0 + }, + "replicas": { + "mcp-registry-558c7cb4d6-5xkrj": { + "peak_sampled_pool_pending": 357.0, + "final_recovery_pool_pending": 0.0, + "page_acquire_mean_ms": 237.11473657594314, + "page_checkout_mean_ms": 44.480979661182154, + "cpu_stalled_fraction": 0.3073916413911048 + }, + "mcp-registry-558c7cb4d6-hj5kk": { + "peak_sampled_pool_pending": 0.0, + "final_recovery_pool_pending": 0.0, + "page_acquire_mean_ms": 29.14358570522299, + "page_checkout_mean_ms": 14.180265811582807, + "cpu_stalled_fraction": 0.1392282529336618 + }, + "mcp-registry-558c7cb4d6-p4ghv": { + "peak_sampled_pool_pending": 460.0, + "final_recovery_pool_pending": 0.0, + "page_acquire_mean_ms": 311.2957382305604, + "page_checkout_mean_ms": 46.70886719516627, + "cpu_stalled_fraction": 0.30570831741404403 + }, + "mcp-registry-558c7cb4d6-znp4f": { + "peak_sampled_pool_pending": 0.0, + "final_recovery_pool_pending": 0.0, + "page_acquire_mean_ms": 5.652464262813558, + "page_checkout_mean_ms": 11.51279432375799, + "cpu_stalled_fraction": 0.16143010717801645 + } + }, + "dominant_db_statement": { + "queryid": 308564448692527448, + "calls": 86114, + "mean_ms": 0.6274773268922902, + "total_exec_ms": 54034.58252800268, + "block_reads": 0, + "temp_blocks_written": 0 + }, + "runtime_integrity": { + "as_of": 1793202440.3141491, + "samples": 17, + "commands_failed": [], + "app_restarts": { + "mcp-registry-558c7cb4d6-5xkrj/mcp-registry": 0, + "mcp-registry-558c7cb4d6-hj5kk/mcp-registry": 0, + "mcp-registry-558c7cb4d6-p4ghv/mcp-registry": 0, + "mcp-registry-558c7cb4d6-znp4f/mcp-registry": 0, + "registry-pg-1/postgres": 0 + }, + "node_count_timeline": [ + { + "at": 1793201907.760947, + "nodes": 4 + } + ], + "db_throttled_usec_range": [ + 1017850, + 1017850 + ], + "db_endpoint_samples": 90, + "db_endpoint_errors_or_empty": [], + "db_endpoint_not_ready": [], + "cadvisor_failures_or_empty": [], + "lb_collection_failures": [] + }, + "captures": { + "total": 140, + "failed": 0 + }, + "generator": { + "sample_intervals": 17, + "scope": "Whole sixteen-vCPU generator from /proc/stat and non-loopback network devices; includes monitoring overhead. CPU busy includes steal, reported separately. Does not alone rule out single-thread or network constraints.", + "cpu_busy_percent": { + "mean": 21.73170009661974, + "max": 28.390014247661526 + }, + "cpu_steal_percent": { + "mean": 2.1881960590938485, + "max": 3.8406739763364928 + }, + "cpu_iowait_percent": { + "mean": 0.0018128296201882859, + "max": 0.0062000124000248 + }, + "cpu_softirq_percent": { + "mean": 5.6992087472738, + "max": 6.460845207648276 + }, + "received_gbps": { + "mean": 2.07460722321219, + "max": 2.171329519777456 + }, + "transmitted_gbps": { + "mean": 0.025005732777509497, + "max": 0.029727798782741847 + }, + "receive_drops": 0, + "transmit_drops": 0 + } + } + }, + "full_test_run": false, + "decision": "Neither staging short run passed; no promotion. Dedicated DB placement alone did not resolve app CPU contention at the same four-node budget." +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/cleanup-progress.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/cleanup-progress.json new file mode 100644 index 000000000..b5a26c532 --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/cleanup-progress.json @@ -0,0 +1,10 @@ +{ + "at": 1793199017.9505851, + "nodes": 4, + "ingress_current_replicas": 4, + "ingress_desired_replicas": 4, + "node_pool_min": 2, + "node_pool_max": 4, + "node_scale_down_candidate_observed": true, + "candidate_full_recovery_tested": false +} diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/cleanup.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/cleanup.json new file mode 100644 index 000000000..86ccfc182 --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/cleanup.json @@ -0,0 +1,12 @@ +{ + "at": 1793198610.657897, + "production_changed": false, + "generator_instances_absent": true, + "generator_disks_absent": true, + "app_restored": true, + "replicas": 2, + "ready_replicas": 2, + "profiling_disabled": true, + "hpa_spec_restored": true, + "whitelist_restored": true +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/comparison.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/comparison.json new file mode 100644 index 000000000..bfa6ad063 --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/comparison.json @@ -0,0 +1,140 @@ +{ + "production_changed": false, + "short_diagnostic_only": true, + "arms": { + "two": { + "scheduled_reads": 810000, + "attempts_finished": 759347, + "dropped": 50653, + "drop_percent": 6.253456790123457, + "attempt_failures": 0, + "errors": {}, + "latency_ms": { + "page_p50": 5.811258, + "page_p95": 688.39129, + "page_p99": 1715.119676, + "search_p50": 109.895904, + "search_p95": 2702.476342, + "search_p99": 3933.929401, + "updated_p50": 77.67783, + "updated_p95": 3360.372651, + "updated_p99": 4091.710536 + }, + "start": "2026-10-28T14:23:19.022947812Z", + "end": "2026-10-28T14:26:19.089668473Z", + "replicas": { + "mcp-registry-c46895d95-4bcgm": { + "observed_origin_completions": 41547.0, + "process_cpu_seconds": 132.89000000000001, + "cpu_ms_per_origin_completion": 3.198546224757504, + "peak_sampled_pool_waiters": 0.0, + "peak_heap_inuse_bytes": 55443456.0, + "payload_sources": { + "prepared": 2962360.0, + "fallback": 0.0 + }, + "first_sample": "burst-000-mcp-registry-c46895d95-4bcgm.prom", + "last_sample": "burst-180-mcp-registry-c46895d95-4bcgm.prom" + }, + "mcp-registry-c46895d95-fv2tn": { + "observed_origin_completions": 41562.0, + "process_cpu_seconds": 126.09, + "cpu_ms_per_origin_completion": 3.0337808575140754, + "peak_sampled_pool_waiters": 905.0, + "peak_heap_inuse_bytes": 105037824.0, + "payload_sources": { + "prepared": 2967097.0, + "fallback": 0.0 + }, + "first_sample": "burst-000-mcp-registry-c46895d95-fv2tn.prom", + "last_sample": "burst-180-mcp-registry-c46895d95-fv2tn.prom" + } + }, + "short_recovery": { + "reads": 12000, + "dropped": 0, + "failures": 0 + } + }, + "four": { + "scheduled_reads": 810000, + "attempts_finished": 761101, + "dropped": 48899, + "drop_percent": 6.036913580246914, + "attempt_failures": 0, + "errors": {}, + "latency_ms": { + "page_p50": 7.029942, + "page_p95": 620.769074, + "page_p99": 1251.604969, + "search_p50": 58.809584, + "search_p95": 803.619519, + "search_p99": 1358.229498, + "updated_p50": 22.128803, + "updated_p95": 829.322605, + "updated_p99": 1437.237721 + }, + "start": "2026-10-28T14:32:47.208281842Z", + "end": "2026-10-28T14:35:47.230801941Z", + "replicas": { + "mcp-registry-6bf4df77c5-b5zgz": { + "observed_origin_completions": 20395.0, + "process_cpu_seconds": 71.4, + "cpu_ms_per_origin_completion": 3.5008580534444715, + "peak_sampled_pool_waiters": 0.0, + "peak_heap_inuse_bytes": 32735232.0, + "payload_sources": { + "prepared": 1450258.0, + "fallback": 0.0 + }, + "first_sample": "burst-000-mcp-registry-6bf4df77c5-b5zgz.prom", + "last_sample": "burst-180-mcp-registry-6bf4df77c5-b5zgz.prom" + }, + "mcp-registry-6bf4df77c5-gtf7x": { + "observed_origin_completions": 20393.0, + "process_cpu_seconds": 67.17, + "cpu_ms_per_origin_completion": 3.293777276516452, + "peak_sampled_pool_waiters": 0.0, + "peak_heap_inuse_bytes": 20987904.0, + "payload_sources": { + "prepared": 1449134.0, + "fallback": 0.0 + }, + "first_sample": "burst-000-mcp-registry-6bf4df77c5-gtf7x.prom", + "last_sample": "burst-180-mcp-registry-6bf4df77c5-gtf7x.prom" + }, + "mcp-registry-6bf4df77c5-pkxsj": { + "observed_origin_completions": 20390.0, + "process_cpu_seconds": 80.17, + "cpu_ms_per_origin_completion": 3.931829328102011, + "peak_sampled_pool_waiters": 3.0, + "peak_heap_inuse_bytes": 22585344.0, + "payload_sources": { + "prepared": 1455983.0, + "fallback": 0.0 + }, + "first_sample": "burst-000-mcp-registry-6bf4df77c5-pkxsj.prom", + "last_sample": "burst-180-mcp-registry-6bf4df77c5-pkxsj.prom" + }, + "mcp-registry-6bf4df77c5-xd7ms": { + "observed_origin_completions": 20389.0, + "process_cpu_seconds": 70.99, + "cpu_ms_per_origin_completion": 3.4817793908480064, + "peak_sampled_pool_waiters": 1.0, + "peak_heap_inuse_bytes": 19079168.0, + "payload_sources": { + "prepared": 1458004.0, + "fallback": 0.0 + }, + "first_sample": "burst-000-mcp-registry-6bf4df77c5-xd7ms.prom", + "last_sample": "burst-180-mcp-registry-6bf4df77c5-xd7ms.prom" + } + }, + "short_recovery": { + "reads": 12000, + "dropped": 0, + "failures": 0 + } + } + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/experiment.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/experiment.json new file mode 100644 index 000000000..a27126c51 --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/experiment.json @@ -0,0 +1,53 @@ +{ + "at": 1793197315.025013, + "commit": "43e17e0993e1afed08950da75b461a1e4966fab3", + "image": "us-central1-docker.pkg.dev/mcp-registry-staging/registry-staging-experiments/registry:staging-replicas-20261028-v1@sha256:60fb8b7d7931a257542848460f7d1c76a99f16c46ddad795249fb5c25b1c9b0e", + "driver_sha256": "dad841c9c6ef8c73ada3681dee2ed7b900d65db8e75574b38d4c779097771229", + "hypothesis": "Distributing origin work across four app replicas on four nodes removes per-app CPU starvation without increasing the aggregate pool budget.", + "control": { + "replicas": 2, + "pool_max": 60, + "pool_min": 10, + "cpu_request_m": 400 + }, + "candidate": { + "replicas": 4, + "pool_max": 30, + "pool_min": 5, + "cpu_request_m": 400 + }, + "constants": { + "nodes": 4, + "ingress_replicas": 6, + "total_pool_max": 120, + "total_pool_min": 20, + "db_cpu_quota": 3, + "app_cpu_limit": null, + "burst_read_rate": 4500, + "burst_seconds": 180, + "updated_rate": 36, + "search_rate": 3, + "publish_rate": 0 + }, + "short_gate": { + "dropped_reads": 0, + "attempt_failures": 0, + "page_p95_ms_max": 1000, + "recovery_dropped_reads": 0, + "recovery_attempt_failures": 0, + "pool_queues_drain": true + }, + "full_gate_if_short_passes": { + "burst_minutes": 10, + "burst_read_rate": 4500, + "publish_rate": 1, + "recovery_minutes": 30, + "recovery_read_rate": 100, + "dropped_work": 0, + "attempt_failures": 0, + "page_p95_ms_max": 1000, + "publish_p95_ms_max": 2000 + }, + "production_changed": false, + "scope_note": "This is a fixed-capacity control. It cannot prove autoscaling behavior; a passing full test is required before implementing an app autoscaler." +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-app-pressure-summary.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-app-pressure-summary.json new file mode 100644 index 000000000..e3eac52a6 --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-app-pressure-summary.json @@ -0,0 +1,98 @@ +[ + { + "pod": "mcp-registry-6bf4df77c5-pkxsj", + "metric": "container_cpu_usage_seconds_total", + "seconds": 151.57299995422363, + "rate": 0.43716863834595865, + "start": 1793197991.969, + "end": 1793198143.542 + }, + { + "pod": "mcp-registry-6bf4df77c5-pkxsj", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 151.57299995422363, + "rate": 0.11679259502250626, + "start": 1793197991.969, + "end": 1793198143.542 + }, + { + "pod": "mcp-registry-6bf4df77c5-pkxsj", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 151.57299995422363, + "rate": 0.1719682265830463, + "start": 1793197991.969, + "end": 1793198143.542 + }, + { + "pod": "mcp-registry-6bf4df77c5-xd7ms", + "metric": "container_cpu_usage_seconds_total", + "seconds": 123.20300006866455, + "rate": 0.3848657660411951, + "start": 1793197995.25, + "end": 1793198118.453 + }, + { + "pod": "mcp-registry-6bf4df77c5-xd7ms", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 123.20300006866455, + "rate": 0.032506026620845174, + "start": 1793197995.25, + "end": 1793198118.453 + }, + { + "pod": "mcp-registry-6bf4df77c5-xd7ms", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 123.20300006866455, + "rate": 0.05558061083077184, + "start": 1793197995.25, + "end": 1793198118.453 + }, + { + "pod": "mcp-registry-6bf4df77c5-gtf7x", + "metric": "container_cpu_usage_seconds_total", + "seconds": 124.15499997138977, + "rate": 0.3626971689450835, + "start": 1793197995.381, + "end": 1793198119.536 + }, + { + "pod": "mcp-registry-6bf4df77c5-gtf7x", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 124.15499997138977, + "rate": 0.04721316097902445, + "start": 1793197995.381, + "end": 1793198119.536 + }, + { + "pod": "mcp-registry-6bf4df77c5-gtf7x", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 124.15499997138977, + "rate": 0.07228980711262717, + "start": 1793197995.381, + "end": 1793198119.536 + }, + { + "pod": "mcp-registry-6bf4df77c5-b5zgz", + "metric": "container_cpu_usage_seconds_total", + "seconds": 120.21399998664856, + "rate": 0.39486292782264953, + "start": 1793197996.797, + "end": 1793198117.011 + }, + { + "pod": "mcp-registry-6bf4df77c5-b5zgz", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 120.21399998664856, + "rate": 0.2278128171680639, + "start": 1793197996.797, + "end": 1793198117.011 + }, + { + "pod": "mcp-registry-6bf4df77c5-b5zgz", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 120.21399998664856, + "rate": 0.31036977393767684, + "start": 1793197996.797, + "end": 1793198117.011 + } +] \ No newline at end of file diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-checkout-summary.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-checkout-summary.json new file mode 100644 index 000000000..c2cc16fc6 --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-checkout-summary.json @@ -0,0 +1,498 @@ +{ + "mcp-registry-6bf4df77c5-b5zgz": { + "first": "burst-000-mcp-registry-6bf4df77c5-b5zgz.prom", + "last": "burst-180-mcp-registry-6bf4df77c5-b5zgz.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 18716.0, + "sum": 1489.053227332001, + "mean_ms": 79.56044172536872 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 114.0, + "sum": 5.4931599329999985, + "mean_ms": 48.185613447368404 + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 1540.0, + "sum": 69.24464279399983, + "mean_ms": 44.96405376233755 + }, + "mcp_registry_db_pool_acquire_duration/list": { + "count": 26.0, + "sum": 0.9977762969999999, + "mean_ms": 38.37601142307692 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 18716.0, + "sum": 456.5092840599992, + "mean_ms": 24.3913915398589 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 114.0, + "sum": 5.212296416000001, + "mean_ms": 45.72189838596492 + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 1540.0, + "sum": 24.32988893699996, + "mean_ms": 15.798629179870103 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 26.0, + "sum": 0.5888649250000001, + "mean_ms": 22.648650961538465 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 18716.0, + "sum": 4.250123910000004, + "mean_ms": 0.22708505610173135 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 114.0, + "sum": 0.045882541000000006, + "mean_ms": 0.40247842982456145 + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 1540.0, + "sum": 0.8191861509999985, + "mean_ms": 0.5319390590909081 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 26.0, + "sum": 0.0014326779999999995, + "mean_ms": 0.05510299999999998 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_page": { + "count": 18716.0, + "sum": 409.4759543299991, + "mean_ms": 21.878390378820214 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 114.0, + "sum": 4.921385816000001, + "mean_ms": 43.17005101754387 + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 1540.0, + "sum": 20.779866715000015, + "mean_ms": 13.493419944805204 + }, + "mcp_registry_db_query_duration/list": { + "count": 26.0, + "sum": 0.5861715479999998, + "mean_ms": 22.54505953846153 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 4.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 30.0 + } + }, + "payload_sources": { + "prepared": 1450258.0, + "fallback": 0.0 + } + }, + "mcp-registry-6bf4df77c5-gtf7x": { + "first": "burst-000-mcp-registry-6bf4df77c5-gtf7x.prom", + "last": "burst-180-mcp-registry-6bf4df77c5-gtf7x.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 18722.0, + "sum": 20.473407345999956, + "mean_ms": 1.0935480902681314 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 131.0, + "sum": 0.09128265700000007, + "mean_ms": 0.6968141755725197 + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 1523.0, + "sum": 1.0313890779999997, + "mean_ms": 0.6772088496388704 + }, + "mcp_registry_db_pool_acquire_duration/list": { + "count": 17.0, + "sum": 0.00018325299999999997, + "mean_ms": 0.010779588235294115 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 18722.0, + "sum": 100.85223054199939, + "mean_ms": 5.386829961649364 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 131.0, + "sum": 4.778361607000002, + "mean_ms": 36.47604280152673 + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 1523.0, + "sum": 7.205158776999994, + "mean_ms": 4.7308987373604685 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 17.0, + "sum": 0.08707231600000002, + "mean_ms": 5.121900941176472 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 18722.0, + "sum": 0.5840743050000066, + "mean_ms": 0.031197217444717797 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 131.0, + "sum": 0.0030275150000000015, + "mean_ms": 0.02311080152671757 + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 1523.0, + "sum": 0.05436281600000002, + "mean_ms": 0.03569456073539069 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 17.0, + "sum": 0.001110131, + "mean_ms": 0.06530182352941177 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_page": { + "count": 18722.0, + "sum": 99.83098312000068, + "mean_ms": 5.332281974148098 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 131.0, + "sum": 4.7736094540000025, + "mean_ms": 36.4397668244275 + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 1523.0, + "sum": 7.1281766189999995, + "mean_ms": 4.680352343401181 + }, + "mcp_registry_db_query_duration/list": { + "count": 17.0, + "sum": 0.08574400700000001, + "mean_ms": 5.04376511764706 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 2.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 30.0 + } + }, + "payload_sources": { + "prepared": 1449134.0, + "fallback": 0.0 + } + }, + "mcp-registry-6bf4df77c5-pkxsj": { + "first": "burst-000-mcp-registry-6bf4df77c5-pkxsj.prom", + "last": "burst-180-mcp-registry-6bf4df77c5-pkxsj.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 18736.0, + "sum": 91.61865717799863, + "mean_ms": 4.889979567570379 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 139.0, + "sum": 0.5502352839999999, + "mean_ms": 3.9585272230215818 + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 1491.0, + "sum": 2.844591826999995, + "mean_ms": 1.9078416009389638 + }, + "mcp_registry_db_pool_acquire_duration/list": { + "count": 24.0, + "sum": 0.00036587699999999997, + "mean_ms": 0.015244874999999998 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 18736.0, + "sum": 174.41006534000005, + "mean_ms": 9.308820737617424 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 139.0, + "sum": 5.920989958, + "mean_ms": 42.59705005755396 + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 1491.0, + "sum": 9.981696600999992, + "mean_ms": 6.69463219382964 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 24.0, + "sum": 0.09905449800000002, + "mean_ms": 4.127270750000001 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 18736.0, + "sum": 1.2650001459999995, + "mean_ms": 0.06751708721178477 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 139.0, + "sum": 0.010358268999999996, + "mean_ms": 0.07451992086330933 + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 1491.0, + "sum": 0.12660151899999986, + "mean_ms": 0.08491047551978528 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 24.0, + "sum": 0.0010413360000000001, + "mean_ms": 0.043389000000000004 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_page": { + "count": 18736.0, + "sum": 170.63018922600006, + "mean_ms": 9.107076709329636 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 139.0, + "sum": 5.907096860999998, + "mean_ms": 42.49709971942444 + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 1491.0, + "sum": 9.793633111000014, + "mean_ms": 6.568499739101283 + }, + "mcp_registry_db_query_duration/list": { + "count": 24.0, + "sum": 0.09742932099999999, + "mean_ms": 4.059555041666666 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 2.0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 1.0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 8.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 30.0 + } + }, + "payload_sources": { + "prepared": 1455983.0, + "fallback": 0.0 + } + }, + "mcp-registry-6bf4df77c5-xd7ms": { + "first": "burst-000-mcp-registry-6bf4df77c5-xd7ms.prom", + "last": "burst-180-mcp-registry-6bf4df77c5-xd7ms.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 18846.0, + "sum": 17.50576042300009, + "mean_ms": 0.9288846664013631 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 121.0, + "sum": 0.0019340709999999999, + "mean_ms": 0.01598405785123967 + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 1400.0, + "sum": 0.36957581200000017, + "mean_ms": 0.26398272285714297 + }, + "mcp_registry_db_pool_acquire_duration/list": { + "count": 22.0, + "sum": 0.000371228, + "mean_ms": 0.016874 + }, + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 18846.0, + "sum": 94.7892322890001, + "mean_ms": 5.029673792263615 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 121.0, + "sum": 3.719488645, + "mean_ms": 30.7395755785124 + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 1400.0, + "sum": 6.296949610999998, + "mean_ms": 4.497821150714285 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 22.0, + "sum": 0.077583495, + "mean_ms": 3.5265225 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 18846.0, + "sum": 0.4296833440000033, + "mean_ms": 0.022799710495596055 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 121.0, + "sum": 0.0027311320000000007, + "mean_ms": 0.02257133884297521 + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 1400.0, + "sum": 0.031579207, + "mean_ms": 0.022556576428571427 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 22.0, + "sum": 0.00035150900000000007, + "mean_ms": 0.01597768181818182 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_page": { + "count": 18846.0, + "sum": 93.85226569099962, + "mean_ms": 4.979956791414604 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 121.0, + "sum": 3.714890642999998, + "mean_ms": 30.701575561983457 + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 1400.0, + "sum": 6.246057030999998, + "mean_ms": 4.461469307857141 + }, + "mcp_registry_db_query_duration/list": { + "count": 22.0, + "sum": 0.076974601, + "mean_ms": 3.4988455000000003 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 1.0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 5.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 30.0 + } + }, + "payload_sources": { + "prepared": 1458004.0, + "fallback": 0.0 + } + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-db-burst-statement-deltas.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-db-burst-statement-deltas.json new file mode 100644 index 000000000..645a7e0e1 --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-db-burst-statement-deltas.json @@ -0,0 +1,33 @@ +{ + "before": "diagnostics-1793197823.json", + "after": "diagnostics-1793198164.json", + "window_seconds": 342.70637607574463, + "reset_unchanged": true, + "scope": "Only statements present in both bounded top-25 snapshots; interval brackets burst with collection padding", + "rows": [ + { + "queryid": 308564448692527448, + "calls": 80475, + "mean_ms": 1.8195282152345273, + "total_exec_ms": 146426.5331209986, + "block_reads": 0, + "temp_blocks_written": 0 + }, + { + "queryid": 9162679476881629384, + "calls": 6202, + "mean_ms": 2.3376570059658413, + "total_exec_ms": 14498.148751000146, + "block_reads": 0, + "temp_blocks_written": 0 + }, + { + "queryid": 8432540089838864685, + "calls": 634, + "mean_ms": 22.104106659306115, + "total_exec_ms": 14014.003622000077, + "block_reads": 0, + "temp_blocks_written": 0 + } + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-generator-summary.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-generator-summary.json new file mode 100644 index 000000000..a53a35e60 --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-generator-summary.json @@ -0,0 +1,219 @@ +{ + "sample_intervals": 17, + "scope": "Whole four-vCPU generator from /proc/stat and non-loopback network devices; includes monitoring overhead. CPU busy includes steal, reported separately. Does not alone rule out single-thread or network constraints.", + "intervals": [ + { + "at": 1793197979.236559, + "cpu_busy_percent": 60.41178003648684, + "cpu_steal_percent": 0.7036747458952306, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 18.582225697159238, + "received_gbps": 2.1735660545384574, + "transmitted_gbps": 0.026803592737355896, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197989.2446823, + "cpu_busy_percent": 68.43879668049793, + "cpu_steal_percent": 0.38900414937759337, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 17.557053941908713, + "received_gbps": 1.9756497730926053, + "transmitted_gbps": 0.029031728372082385, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197999.2527142, + "cpu_busy_percent": 67.94640556557589, + "cpu_steal_percent": 0.2061324400927596, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 16.722494202525123, + "received_gbps": 1.9219533416484342, + "transmitted_gbps": 0.02696785743466008, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793198009.2601385, + "cpu_busy_percent": 59.97915581031788, + "cpu_steal_percent": 0.26055237102657636, + "cpu_iowait_percent": 0.026055237102657634, + "cpu_softirq_percent": 20.036477331943722, + "received_gbps": 2.1737321194041814, + "transmitted_gbps": 0.026299709363304013, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793198019.268129, + "cpu_busy_percent": 64.51363870303655, + "cpu_steal_percent": 0.2573340195573855, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 18.502316006176017, + "received_gbps": 1.9560804154417157, + "transmitted_gbps": 0.025256479360939266, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793198029.2750576, + "cpu_busy_percent": 71.99792960662526, + "cpu_steal_percent": 0.2070393374741201, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 18.115942028985508, + "received_gbps": 1.8578045041699545, + "transmitted_gbps": 0.028713406077656347, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793198039.283154, + "cpu_busy_percent": 60.96918619963129, + "cpu_steal_percent": 1.0797998419805108, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 20.779562812746907, + "received_gbps": 2.1569346624272545, + "transmitted_gbps": 0.02604378736078165, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793198049.2900076, + "cpu_busy_percent": 58.00371648526679, + "cpu_steal_percent": 0.2123705866737457, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 19.72391823732413, + "received_gbps": 2.1239797773668854, + "transmitted_gbps": 0.02413286514666286, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793198059.2972236, + "cpu_busy_percent": 76.81794739556473, + "cpu_steal_percent": 0.15471892728210418, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 16.787003610108304, + "received_gbps": 1.690590264401732, + "transmitted_gbps": 0.02993889216512609, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793198069.3096907, + "cpu_busy_percent": 60.01583531274743, + "cpu_steal_percent": 0.2903140670361573, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 20.84982845077857, + "received_gbps": 2.1571676731844938, + "transmitted_gbps": 0.025576888120362903, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793198079.3170233, + "cpu_busy_percent": 68.80733944954129, + "cpu_steal_percent": 0.34076015727391873, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 17.929226736566186, + "received_gbps": 1.91677353525288, + "transmitted_gbps": 0.02714785526301027, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793198089.3240435, + "cpu_busy_percent": 67.15867158671587, + "cpu_steal_percent": 0.21085925144965736, + "cpu_iowait_percent": 0.02635740643120717, + "cpu_softirq_percent": 19.240906694781234, + "received_gbps": 1.9805508619397638, + "transmitted_gbps": 0.027906081274980028, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793198099.3362043, + "cpu_busy_percent": 58.85014639339899, + "cpu_steal_percent": 0.18631887143997872, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 19.244077721586372, + "received_gbps": 2.1492706476693653, + "transmitted_gbps": 0.02563749161548168, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793198109.3435338, + "cpu_busy_percent": 66.62313920083572, + "cpu_steal_percent": 0.18281535648994515, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 16.662313920083573, + "received_gbps": 1.9250599430528978, + "transmitted_gbps": 0.026314434530051733, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793198119.3491871, + "cpu_busy_percent": 69.06867845993756, + "cpu_steal_percent": 0.1040582726326743, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 18.10613943808533, + "received_gbps": 1.9269700495464401, + "transmitted_gbps": 0.028743669307607314, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793198129.354504, + "cpu_busy_percent": 59.909790395330326, + "cpu_steal_percent": 0.2122578933404086, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 19.23587158397453, + "received_gbps": 2.166475840303139, + "transmitted_gbps": 0.026297536871352195, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793198139.3599524, + "cpu_busy_percent": 64.68285043069694, + "cpu_steal_percent": 0.18271991647089533, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 16.992952231793264, + "received_gbps": 1.93273781765914, + "transmitted_gbps": 0.025598016526757684, + "receive_drops": 0, + "transmit_drops": 0 + } + ], + "cpu_busy_percent": { + "mean": 64.95264751248278, + "max": 76.81794739556473 + }, + "cpu_steal_percent": { + "mean": 0.3047488356172742, + "max": 1.0797998419805108 + }, + "cpu_iowait_percent": { + "mean": 0.0030830966784626355, + "max": 0.02635740643120717 + }, + "cpu_softirq_percent": { + "mean": 18.533430038030982, + "max": 20.84982845077857 + }, + "received_gbps": { + "mean": 2.010899840064667, + "max": 2.1737321194041814 + }, + "transmitted_gbps": { + "mean": 0.026847664207539552, + "max": 0.02993889216512609 + }, + "receive_drops": 0, + "transmit_drops": 0 +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-request-path-summary.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-request-path-summary.json new file mode 100644 index 000000000..5c5b1a338 --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-request-path-summary.json @@ -0,0 +1,255 @@ +{ + "scope": "VictoriaMetrics 180-second increase ending at burst finish; scrape-based estimate, not exact request matching. Histogram bounds, not interpolated quantiles. Bulk-read ingress only. Client latency starts at scheduled arrival and includes scheduler and worker queue delay.", + "ingress": { + "nginx_ingress_controller_connect_duration_seconds_bucket": { + "all": { + "count": 78251.0, + "cumulative_buckets": { + "0.005": 77845.0, + "0.01": 77944.0, + "0.025": 78083.0, + "0.05": 78152.0, + "0.1": 78198.0, + "0.25": 78233.0, + "0.5": 78250.0, + "1.0": 78251.0, + "2.5": 78251.0, + "5.0": 78251.0, + "10.0": 78251.0, + "inf": 78251.0 + }, + "p95_upper_ms": 5.0, + "p99_upper_ms": 5.0 + } + }, + "nginx_ingress_controller_header_duration_seconds_bucket": { + "all": { + "count": 78251.0, + "cumulative_buckets": { + "0.005": 18891.0, + "0.01": 39736.0, + "0.025": 60952.0, + "0.05": 69606.0, + "0.1": 73567.0, + "0.25": 75716.0, + "0.5": 76519.0, + "1.0": 76995.0, + "2.5": 78250.0, + "5.0": 78251.0, + "10.0": 78251.0, + "inf": 78251.0 + }, + "p95_upper_ms": 250.0, + "p99_upper_ms": 2500.0 + } + }, + "nginx_ingress_controller_request_duration_seconds_bucket": { + "all": { + "count": 764542.0, + "cumulative_buckets": { + "0.005": 378440.0, + "0.01": 454280.0, + "0.025": 573920.0, + "0.05": 661047.0, + "0.1": 723438.0, + "0.25": 755528.0, + "0.5": 761524.0, + "1.0": 762989.0, + "2.5": 764541.0, + "5.0": 764542.0, + "10.0": 764542.0, + "inf": 764542.0 + }, + "p95_upper_ms": 250.0, + "p99_upper_ms": 500.0 + } + }, + "nginx_ingress_controller_response_duration_seconds_bucket": { + "all": { + "count": 78251.0, + "cumulative_buckets": { + "0.005": 10971.0, + "0.01": 28436.0, + "0.025": 48029.0, + "0.05": 58977.0, + "0.1": 67856.0, + "0.25": 74462.0, + "0.5": 76219.0, + "1.0": 76928.0, + "2.5": 78250.0, + "5.0": 78251.0, + "10.0": 78251.0, + "inf": 78251.0 + }, + "p95_upper_ms": 250.0, + "p99_upper_ms": 2500.0 + } + } + }, + "client": { + "max_scheduler_lag_ms": 490.28643, + "received_response_bytes": 43647992717, + "cache": { + "HIT": 676598, + "MISS": 26679, + "STALE": 56852, + "UPDATING": 972 + }, + "dropped": 48899, + "scheduled": 810000, + "drop_windows": [ + { + "start": "2026-10-28T14:32:40Z", + "scheduled": 12563, + "dropped": 4302, + "scheduler_lag_max_ms": 307.342966, + "http_mean_ms": 315.8769224773606, + "queue_mean_ms": 207.15429189455168 + }, + { + "start": "2026-10-28T14:32:50Z", + "scheduled": 45000, + "dropped": 0, + "scheduler_lag_max_ms": 6.027901, + "http_mean_ms": 56.57579246983395, + "queue_mean_ms": 0.03561035145885668 + }, + { + "start": "2026-10-28T14:33:00Z", + "scheduled": 45000, + "dropped": 3267, + "scheduler_lag_max_ms": 319.173394, + "http_mean_ms": 66.69689974792162, + "queue_mean_ms": 30.78189123072896 + }, + { + "start": "2026-10-28T14:33:10Z", + "scheduled": 45000, + "dropped": 3501, + "scheduler_lag_max_ms": 214.585882, + "http_mean_ms": 71.49178826299386, + "queue_mean_ms": 40.10222373054776 + }, + { + "start": "2026-10-28T14:33:20Z", + "scheduled": 45000, + "dropped": 0, + "scheduler_lag_max_ms": 2.146983, + "http_mean_ms": 15.097912840688972, + "queue_mean_ms": 0.035238837688888865 + }, + { + "start": "2026-10-28T14:33:30Z", + "scheduled": 45000, + "dropped": 4177, + "scheduler_lag_max_ms": 462.127782, + "http_mean_ms": 61.828214698276, + "queue_mean_ms": 33.25300850563851 + }, + { + "start": "2026-10-28T14:33:40Z", + "scheduled": 45000, + "dropped": 4789, + "scheduler_lag_max_ms": 490.28643, + "http_mean_ms": 86.36733540009705, + "queue_mean_ms": 44.607110885096034 + }, + { + "start": "2026-10-28T14:33:50Z", + "scheduled": 45000, + "dropped": 0, + "scheduler_lag_max_ms": 9.706917, + "http_mean_ms": 19.103304622222346, + "queue_mean_ms": 0.04957219948888926 + }, + { + "start": "2026-10-28T14:34:00Z", + "scheduled": 45000, + "dropped": 0, + "scheduler_lag_max_ms": 2.487848, + "http_mean_ms": 11.997588064311039, + "queue_mean_ms": 0.02211763448888871 + }, + { + "start": "2026-10-28T14:34:10Z", + "scheduled": 45000, + "dropped": 8782, + "scheduler_lag_max_ms": 282.541236, + "http_mean_ms": 130.00378477698305, + "queue_mean_ms": 89.80548412151782 + }, + { + "start": "2026-10-28T14:34:20Z", + "scheduled": 45000, + "dropped": 0, + "scheduler_lag_max_ms": 4.129266, + "http_mean_ms": 40.30300710097569, + "queue_mean_ms": 0.03247769320681753 + }, + { + "start": "2026-10-28T14:34:30Z", + "scheduled": 45000, + "dropped": 4668, + "scheduler_lag_max_ms": 239.440387, + "http_mean_ms": 87.66371979068676, + "queue_mean_ms": 40.17027649045469 + }, + { + "start": "2026-10-28T14:34:40Z", + "scheduled": 45000, + "dropped": 2735, + "scheduler_lag_max_ms": 199.730233, + "http_mean_ms": 70.35024924086105, + "queue_mean_ms": 35.906052531763876 + }, + { + "start": "2026-10-28T14:34:50Z", + "scheduled": 45000, + "dropped": 0, + "scheduler_lag_max_ms": 1.843467, + "http_mean_ms": 23.05708995133339, + "queue_mean_ms": 0.026086371933333245 + }, + { + "start": "2026-10-28T14:35:00Z", + "scheduled": 45000, + "dropped": 3489, + "scheduler_lag_max_ms": 167.02381, + "http_mean_ms": 70.8865089599378, + "queue_mean_ms": 39.728461297391284 + }, + { + "start": "2026-10-28T14:35:10Z", + "scheduled": 45000, + "dropped": 4391, + "scheduler_lag_max_ms": 147.109401, + "http_mean_ms": 68.68577627951936, + "queue_mean_ms": 46.294208309365025 + }, + { + "start": "2026-10-28T14:35:20Z", + "scheduled": 45000, + "dropped": 0, + "scheduler_lag_max_ms": 3.481933, + "http_mean_ms": 20.931129828129468, + "queue_mean_ms": 0.030496771972710262 + }, + { + "start": "2026-10-28T14:35:30Z", + "scheduled": 45000, + "dropped": 4798, + "scheduler_lag_max_ms": 476.031706, + "http_mean_ms": 69.32207989729606, + "queue_mean_ms": 37.166894863094875 + }, + { + "start": "2026-10-28T14:35:40Z", + "scheduled": 32437, + "dropped": 0, + "scheduler_lag_max_ms": 2.917472, + "http_mean_ms": 9.41660763054529, + "queue_mean_ms": 0.023161071739063257 + } + ] + } +} diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-runtime-integrity.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-runtime-integrity.json new file mode 100644 index 000000000..6585a3a7c --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/four-runtime-integrity.json @@ -0,0 +1,27 @@ +{ + "as_of": 1793198600.268733, + "samples": 17, + "commands_failed": [], + "app_restarts": { + "mcp-registry-6bf4df77c5-b5zgz/mcp-registry": 0, + "mcp-registry-6bf4df77c5-gtf7x/mcp-registry": 0, + "mcp-registry-6bf4df77c5-pkxsj/mcp-registry": 0, + "mcp-registry-6bf4df77c5-xd7ms/mcp-registry": 0, + "registry-pg-1/postgres": 0 + }, + "node_count_timeline": [ + { + "at": 1793197823.106338, + "nodes": 4 + } + ], + "db_throttled_usec_range": [ + 976975, + 976975 + ], + "db_endpoint_samples": 90, + "db_endpoint_errors_or_empty": [], + "db_endpoint_not_ready": [], + "cadvisor_failures_or_empty": [], + "lb_collection_failures": [] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-app-pressure-summary.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-app-pressure-summary.json new file mode 100644 index 000000000..b75ab6422 --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-app-pressure-summary.json @@ -0,0 +1,50 @@ +[ + { + "pod": "mcp-registry-c46895d95-4bcgm", + "metric": "container_cpu_usage_seconds_total", + "seconds": 156.51500010490417, + "rate": 0.7532370502570507, + "start": 1793197401.978, + "end": 1793197558.493 + }, + { + "pod": "mcp-registry-c46895d95-4bcgm", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 156.51500010490417, + "rate": 0.28372311899968855, + "start": 1793197401.978, + "end": 1793197558.493 + }, + { + "pod": "mcp-registry-c46895d95-4bcgm", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 156.51500010490417, + "rate": 0.40651145869313005, + "start": 1793197401.978, + "end": 1793197558.493 + }, + { + "pod": "mcp-registry-c46895d95-fv2tn", + "metric": "container_cpu_usage_seconds_total", + "seconds": 155.8050000667572, + "rate": 0.7138816722977007, + "start": 1793197407.756, + "end": 1793197563.561 + }, + { + "pod": "mcp-registry-c46895d95-fv2tn", + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 155.8050000667572, + "rate": 0.28177460274823984, + "start": 1793197407.756, + "end": 1793197563.561 + }, + { + "pod": "mcp-registry-c46895d95-fv2tn", + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 155.8050000667572, + "rate": 0.38957453210097936, + "start": 1793197407.756, + "end": 1793197563.561 + } +] \ No newline at end of file diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-checkout-summary.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-checkout-summary.json new file mode 100644 index 000000000..3c97da975 --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-checkout-summary.json @@ -0,0 +1,250 @@ +{ + "mcp-registry-c46895d95-4bcgm": { + "first": "burst-000-mcp-registry-c46895d95-4bcgm.prom", + "last": "burst-180-mcp-registry-c46895d95-4bcgm.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 38301.0, + "sum": 1672.3159243200132, + "mean_ms": 43.662461145140156 + }, + "mcp_registry_db_pool_acquire_duration/list": { + "count": 49.0, + "sum": 2.7600764079999998, + "mean_ms": 56.32808995918367 + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 2957.0, + "sum": 103.2179278249998, + "mean_ms": 34.90629956881968 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 240.0, + "sum": 8.474273186000003, + "mean_ms": 35.30947160833335 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 38301.0, + "sum": 2361.9877845979986, + "mean_ms": 61.669089177775994 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 49.0, + "sum": 3.066831767, + "mean_ms": 62.588403408163266 + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 2957.0, + "sum": 135.45342907100007, + "mean_ms": 45.80772034866421 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 240.0, + "sum": 23.00128470800001, + "mean_ms": 95.83868628333337 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 38301.0, + "sum": 26.786343145999986, + "mean_ms": 0.6993640674133831 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 49.0, + "sum": 0.010417385000000001, + "mean_ms": 0.21259969387755104 + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 2957.0, + "sum": 3.3778861800000013, + "mean_ms": 1.142335536016233 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 240.0, + "sum": 0.015024344, + "mean_ms": 0.06260143333333333 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_page": { + "count": 38301.0, + "sum": 2192.732194635977, + "mean_ms": 57.24999855450189 + }, + "mcp_registry_db_query_duration/list": { + "count": 49.0, + "sum": 3.0550493950000006, + "mean_ms": 62.34794683673471 + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 2957.0, + "sum": 123.08825585100004, + "mean_ms": 41.62605879303349 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 240.0, + "sum": 22.24132436499998, + "mean_ms": 92.67218485416657 + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 0, + "{'operation': 'list_page'}": 0, + "{'operation': 'list_search'}": 0, + "{'operation': 'list_updated'}": 0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 29.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 60.0 + } + }, + "payload_sources": { + "prepared": 2962360.0, + "fallback": 0.0 + } + }, + "mcp-registry-c46895d95-fv2tn": { + "first": "burst-000-mcp-registry-c46895d95-fv2tn.prom", + "last": "burst-180-mcp-registry-c46895d95-fv2tn.prom", + "histograms": { + "mcp_registry_db_pool_acquire_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_acquire_duration/list_page": { + "count": 38253.0, + "sum": 20489.601879806596, + "mean_ms": 535.6338556402529 + }, + "mcp_registry_db_pool_acquire_duration/list": { + "count": 44.0, + "sum": 26.172650316000002, + "mean_ms": 594.8329617272728 + }, + "mcp_registry_db_pool_acquire_duration/list_updated": { + "count": 3017.0, + "sum": 1390.215166786002, + "mean_ms": 460.7938902174352 + }, + "mcp_registry_db_pool_acquire_duration/list_search": { + "count": 259.0, + "sum": 108.11066098199997, + "mean_ms": 417.41567946718135 + }, + "mcp_registry_db_pool_checkout_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_checkout_duration/list_page": { + "count": 38245.0, + "sum": 2177.4429771350083, + "mean_ms": 56.93405614158735 + }, + "mcp_registry_db_pool_checkout_duration/list": { + "count": 44.0, + "sum": 4.490169375000001, + "mean_ms": 102.04930397727274 + }, + "mcp_registry_db_pool_checkout_duration/list_updated": { + "count": 3017.0, + "sum": 137.94867682199998, + "mean_ms": 45.72379079284056 + }, + "mcp_registry_db_pool_checkout_duration/list_search": { + "count": 259.0, + "sum": 20.213473169999972, + "mean_ms": 78.04429795366785 + }, + "mcp_registry_db_pool_after_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_pool_after_query_duration/list_page": { + "count": 38245.0, + "sum": 16.343105980999834, + "mean_ms": 0.4273266042881379 + }, + "mcp_registry_db_pool_after_query_duration/list": { + "count": 44.0, + "sum": 0.000965804, + "mean_ms": 0.02195009090909091 + }, + "mcp_registry_db_pool_after_query_duration/list_updated": { + "count": 3017.0, + "sum": 3.312045036999995, + "mean_ms": 1.0977941786542906 + }, + "mcp_registry_db_pool_after_query_duration/list_search": { + "count": 259.0, + "sum": 0.025251504999999987, + "mean_ms": 0.09749615830115825 + }, + "mcp_registry_db_query_duration/other": { + "count": 0.0, + "sum": 0.0, + "mean_ms": null + }, + "mcp_registry_db_query_duration/list_page": { + "count": 38245.0, + "sum": 1922.2399931709904, + "mean_ms": 50.261210437207225 + }, + "mcp_registry_db_query_duration/list": { + "count": 44.0, + "sum": 4.2871981609999965, + "mean_ms": 97.43632184090902 + }, + "mcp_registry_db_query_duration/list_updated": { + "count": 3017.0, + "sum": 119.3813164599998, + "mean_ms": 39.56954473317859 + }, + "mcp_registry_db_query_duration/list_search": { + "count": 259.0, + "sum": 19.40796044599999, + "mean_ms": 74.93421021621617 + } + }, + "peak_gauges": { + "mcp_registry_db_pool_acquire_pending": { + "{'operation': 'list'}": 1.0, + "{'operation': 'list_page'}": 855.0, + "{'operation': 'list_search'}": 6.0, + "{'operation': 'list_updated'}": 47.0, + "{'operation': 'other'}": 0 + }, + "mcp_registry_db_pool_connections": { + "{'state': 'acquired'}": 47.0, + "{'state': 'constructing'}": 0, + "{'state': 'idle'}": 60.0 + } + }, + "payload_sources": { + "prepared": 2967097.0, + "fallback": 0.0 + } + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-db-burst-statement-deltas.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-db-burst-statement-deltas.json new file mode 100644 index 000000000..2c0b0d45b --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-db-burst-statement-deltas.json @@ -0,0 +1,33 @@ +{ + "before": "diagnostics-1793197262.json", + "after": "diagnostics-1793197595.json", + "window_seconds": 328.06239914894104, + "reset_unchanged": true, + "scope": "Only statements present in both bounded top-25 snapshots; interval brackets burst with collection padding", + "rows": [ + { + "queryid": 308564448692527448, + "calls": 81657, + "mean_ms": 1.5921378627797258, + "total_exec_ms": 130009.20146100406, + "block_reads": 0, + "temp_blocks_written": 0 + }, + { + "queryid": 8432540089838864685, + "calls": 628, + "mean_ms": 20.013119004777092, + "total_exec_ms": 12568.238735000014, + "block_reads": 0, + "temp_blocks_written": 0 + }, + { + "queryid": 9162679476881629384, + "calls": 6201, + "mean_ms": 2.0144477650378976, + "total_exec_ms": 12491.590591000004, + "block_reads": 0, + "temp_blocks_written": 0 + } + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-generator-summary.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-generator-summary.json new file mode 100644 index 000000000..5c3cfbb5e --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-generator-summary.json @@ -0,0 +1,219 @@ +{ + "sample_intervals": 17, + "scope": "Whole four-vCPU generator from /proc/stat and non-loopback network devices; includes monitoring overhead. CPU busy includes steal, reported separately. Does not alone rule out single-thread or network constraints.", + "intervals": [ + { + "at": 1793197411.922388, + "cpu_busy_percent": 48.110964332893, + "cpu_steal_percent": 0.23778071334214002, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 16.010568031704096, + "received_gbps": 1.6779014574233817, + "transmitted_gbps": 0.020842293230937666, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197421.929646, + "cpu_busy_percent": 55.06230529595015, + "cpu_steal_percent": 0.3374870197300104, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 17.834890965732086, + "received_gbps": 2.0290239804967847, + "transmitted_gbps": 0.023199153397405727, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197431.9391837, + "cpu_busy_percent": 67.85619878403384, + "cpu_steal_percent": 0.18503832936822628, + "cpu_iowait_percent": 0.026434047052603753, + "cpu_softirq_percent": 18.133756278086175, + "received_gbps": 2.002555735049714, + "transmitted_gbps": 0.02762191905099973, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197441.9484472, + "cpu_busy_percent": 59.15822291504287, + "cpu_steal_percent": 0.2338269680436477, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 20.706677058976357, + "received_gbps": 2.1273479815058116, + "transmitted_gbps": 0.024290997197162375, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197451.9572453, + "cpu_busy_percent": 68.37233221907945, + "cpu_steal_percent": 0.17999485728979173, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 18.616610953972742, + "received_gbps": 1.9640132243245378, + "transmitted_gbps": 0.028595266334655534, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197461.9639556, + "cpu_busy_percent": 66.36221294363257, + "cpu_steal_percent": 0.1304801670146138, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 17.301670146137788, + "received_gbps": 1.9664502242937287, + "transmitted_gbps": 0.02798923680787035, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197471.9712303, + "cpu_busy_percent": 56.05060506050605, + "cpu_steal_percent": 0.24752475247524752, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 16.061606160616062, + "received_gbps": 2.135041434447118, + "transmitted_gbps": 0.024184384360797104, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197481.978397, + "cpu_busy_percent": 54.85396383866481, + "cpu_steal_percent": 0.16689847009735745, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 18.331015299026426, + "received_gbps": 2.1226144748015225, + "transmitted_gbps": 0.02371661839117974, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197491.985852, + "cpu_busy_percent": 78.78550951264009, + "cpu_steal_percent": 0.13031013812874642, + "cpu_iowait_percent": 0.026062027625749284, + "cpu_softirq_percent": 17.539744592129267, + "received_gbps": 1.7981344532867114, + "transmitted_gbps": 0.03313804642005299, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197501.9929202, + "cpu_busy_percent": 57.853403141361255, + "cpu_steal_percent": 0.15706806282722513, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 18.7434554973822, + "received_gbps": 2.1311013680529896, + "transmitted_gbps": 0.024300253798624754, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197512.0001972, + "cpu_busy_percent": 57.6953642384106, + "cpu_steal_percent": 0.18543046357615894, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 19.072847682119207, + "received_gbps": 2.1212658331350016, + "transmitted_gbps": 0.023806053106893144, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197522.0074637, + "cpu_busy_percent": 77.04960835509138, + "cpu_steal_percent": 0.13054830287206268, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 16.762402088772845, + "received_gbps": 1.8228145103254318, + "transmitted_gbps": 0.032701554944941806, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197532.0133402, + "cpu_busy_percent": 58.188976377952756, + "cpu_steal_percent": 0.1837270341207349, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 19.16010498687664, + "received_gbps": 2.1252714346985226, + "transmitted_gbps": 0.024319434184456954, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197542.0206366, + "cpu_busy_percent": 56.37602179836512, + "cpu_steal_percent": 0.1362397820163488, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 16.920980926430516, + "received_gbps": 2.1205294082860093, + "transmitted_gbps": 0.02394637355053084, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197552.0287693, + "cpu_busy_percent": 67.05032119914347, + "cpu_steal_percent": 0.13383297644539616, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 17.558886509635975, + "received_gbps": 1.9328184510820885, + "transmitted_gbps": 0.027710675749397094, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197562.03594, + "cpu_busy_percent": 59.20983778126635, + "cpu_steal_percent": 0.15698587127158556, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 19.02145473574045, + "received_gbps": 2.130256245214411, + "transmitted_gbps": 0.024541440525233722, + "receive_drops": 0, + "transmit_drops": 0 + }, + { + "at": 1793197572.0429928, + "cpu_busy_percent": 57.154319336721045, + "cpu_steal_percent": 0.18721583311045734, + "cpu_iowait_percent": 0.0, + "cpu_softirq_percent": 17.999465097619684, + "received_gbps": 2.122259697396103, + "transmitted_gbps": 0.02413968412602825, + "receive_drops": 0, + "transmit_drops": 0 + } + ], + "cpu_busy_percent": { + "mean": 61.48177453710323, + "max": 78.78550951264009 + }, + "cpu_steal_percent": { + "mean": 0.18355233774880886, + "max": 0.3374870197300104 + }, + "cpu_iowait_percent": { + "mean": 0.0030880043928442963, + "max": 0.026434047052603753 + }, + "cpu_softirq_percent": { + "mean": 17.986831588879912, + "max": 20.706677058976357 + }, + "received_gbps": { + "mean": 2.0193764655188158, + "max": 2.135041434447118 + }, + "transmitted_gbps": { + "mean": 0.025826081481009868, + "max": 0.03313804642005299 + }, + "receive_drops": 0, + "transmit_drops": 0 +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-request-path-summary.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-request-path-summary.json new file mode 100644 index 000000000..c3b6c8f22 --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-request-path-summary.json @@ -0,0 +1,255 @@ +{ + "scope": "VictoriaMetrics 180-second increase ending at burst finish; scrape-based estimate, not exact request matching. Histogram bounds, not interpolated quantiles. Bulk-read ingress only. Client latency starts at scheduled arrival and includes scheduler and worker queue delay.", + "ingress": { + "nginx_ingress_controller_connect_duration_seconds_bucket": { + "all": { + "count": 80896.0, + "cumulative_buckets": { + "0.005": 80311.0, + "0.01": 80431.0, + "0.025": 80589.0, + "0.05": 80688.0, + "0.1": 80756.0, + "0.25": 80820.0, + "0.5": 80857.0, + "1.0": 80891.0, + "2.5": 80896.0, + "5.0": 80896.0, + "10.0": 80896.0, + "inf": 80896.0 + }, + "p95_upper_ms": 5.0, + "p99_upper_ms": 5.0 + } + }, + "nginx_ingress_controller_header_duration_seconds_bucket": { + "all": { + "count": 80896.0, + "cumulative_buckets": { + "0.005": 8475.0, + "0.01": 20261.0, + "0.025": 34841.0, + "0.05": 43276.0, + "0.1": 50624.0, + "0.25": 59368.0, + "0.5": 65791.0, + "1.0": 71540.0, + "2.5": 74370.0, + "5.0": 80893.0, + "10.0": 80896.0, + "inf": 80896.0 + }, + "p95_upper_ms": 5000.0, + "p99_upper_ms": 5000.0 + } + }, + "nginx_ingress_controller_request_duration_seconds_bucket": { + "all": { + "count": 775310.0, + "cumulative_buckets": { + "0.005": 470600.0, + "0.01": 550079.0, + "0.025": 637310.0, + "0.05": 683524.0, + "0.1": 713566.0, + "0.25": 737069.0, + "0.5": 751676.0, + "1.0": 762996.0, + "2.5": 767836.0, + "5.0": 775302.0, + "10.0": 775310.0, + "inf": 775310.0 + }, + "p95_upper_ms": 250.0, + "p99_upper_ms": 2500.0 + } + }, + "nginx_ingress_controller_response_duration_seconds_bucket": { + "all": { + "count": 80896.0, + "cumulative_buckets": { + "0.005": 4849.0, + "0.01": 14396.0, + "0.025": 29821.0, + "0.05": 39804.0, + "0.1": 48335.0, + "0.25": 58213.0, + "0.5": 65220.0, + "1.0": 71291.0, + "2.5": 74307.0, + "5.0": 80892.0, + "10.0": 80896.0, + "inf": 80896.0 + }, + "p95_upper_ms": 5000.0, + "p99_upper_ms": 5000.0 + } + } + }, + "client": { + "max_scheduler_lag_ms": 827.520144, + "received_response_bytes": 43546555833, + "cache": { + "HIT": 667937, + "MISS": 27433, + "STALE": 57124, + "UPDATING": 6853 + }, + "dropped": 50653, + "scheduled": 810000, + "drop_windows": [ + { + "start": "2026-10-28T14:23:10Z", + "scheduled": 4397, + "dropped": 1717, + "scheduler_lag_max_ms": 88.034641, + "http_mean_ms": 1102.2991854304792, + "queue_mean_ms": 159.00425740983596 + }, + { + "start": "2026-10-28T14:23:20Z", + "scheduled": 45000, + "dropped": 11627, + "scheduler_lag_max_ms": 140.695028, + "http_mean_ms": 334.2811504304106, + "queue_mean_ms": 304.090551134368 + }, + { + "start": "2026-10-28T14:23:30Z", + "scheduled": 45000, + "dropped": 4349, + "scheduler_lag_max_ms": 3.119615, + "http_mean_ms": 247.66244255459094, + "queue_mean_ms": 224.93835000141218 + }, + { + "start": "2026-10-28T14:23:40Z", + "scheduled": 45000, + "dropped": 0, + "scheduler_lag_max_ms": 39.625242, + "http_mean_ms": 154.36699239030665, + "queue_mean_ms": 28.859376850863537 + }, + { + "start": "2026-10-28T14:23:50Z", + "scheduled": 45000, + "dropped": 4433, + "scheduler_lag_max_ms": 376.655972, + "http_mean_ms": 27.24682788909963, + "queue_mean_ms": 31.67171863252932 + }, + { + "start": "2026-10-28T14:24:00Z", + "scheduled": 45000, + "dropped": 3617, + "scheduler_lag_max_ms": 615.034488, + "http_mean_ms": 78.97965105775444, + "queue_mean_ms": 36.78302557684513 + }, + { + "start": "2026-10-28T14:24:10Z", + "scheduled": 45000, + "dropped": 1533, + "scheduler_lag_max_ms": 140.635951, + "http_mean_ms": 45.54227179021937, + "queue_mean_ms": 2.4342923394649247 + }, + { + "start": "2026-10-28T14:24:20Z", + "scheduled": 45000, + "dropped": 1704, + "scheduler_lag_max_ms": 78.187373, + "http_mean_ms": 72.94898158398206, + "queue_mean_ms": 27.739321403206475 + }, + { + "start": "2026-10-28T14:24:30Z", + "scheduled": 45000, + "dropped": 0, + "scheduler_lag_max_ms": 4.086458, + "http_mean_ms": 11.350802682111128, + "queue_mean_ms": 0.024352406355555547 + }, + { + "start": "2026-10-28T14:24:40Z", + "scheduled": 45000, + "dropped": 4442, + "scheduler_lag_max_ms": 355.142772, + "http_mean_ms": 111.45503324148214, + "queue_mean_ms": 44.83820282410754 + }, + { + "start": "2026-10-28T14:24:50Z", + "scheduled": 45000, + "dropped": 2770, + "scheduler_lag_max_ms": 445.283328, + "http_mean_ms": 52.409492721606924, + "queue_mean_ms": 31.669945515949273 + }, + { + "start": "2026-10-28T14:25:00Z", + "scheduled": 45000, + "dropped": 0, + "scheduler_lag_max_ms": 1.340872, + "http_mean_ms": 11.329246451656736, + "queue_mean_ms": 0.018636747583386997 + }, + { + "start": "2026-10-28T14:25:10Z", + "scheduled": 45000, + "dropped": 4392, + "scheduler_lag_max_ms": 197.247595, + "http_mean_ms": 93.30501195771218, + "queue_mean_ms": 45.59207380189952 + }, + { + "start": "2026-10-28T14:25:20Z", + "scheduled": 45000, + "dropped": 2315, + "scheduler_lag_max_ms": 301.350987, + "http_mean_ms": 65.92700033842092, + "queue_mean_ms": 29.279705332905998 + }, + { + "start": "2026-10-28T14:25:30Z", + "scheduled": 45000, + "dropped": 0, + "scheduler_lag_max_ms": 1.342512, + "http_mean_ms": 23.069702272355695, + "queue_mean_ms": 0.01642779395555571 + }, + { + "start": "2026-10-28T14:25:40Z", + "scheduled": 45000, + "dropped": 935, + "scheduler_lag_max_ms": 27.215479, + "http_mean_ms": 58.26604477829205, + "queue_mean_ms": 14.398817717414207 + }, + { + "start": "2026-10-28T14:25:50Z", + "scheduled": 45000, + "dropped": 3236, + "scheduler_lag_max_ms": 827.520144, + "http_mean_ms": 45.823832441118924, + "queue_mean_ms": 19.048188803420985 + }, + { + "start": "2026-10-28T14:26:00Z", + "scheduled": 45000, + "dropped": 0, + "scheduler_lag_max_ms": 2.627444, + "http_mean_ms": 11.622971195311075, + "queue_mean_ms": 0.02179664611111091 + }, + { + "start": "2026-10-28T14:26:10Z", + "scheduled": 40603, + "dropped": 3583, + "scheduler_lag_max_ms": 497.002706, + "http_mean_ms": 74.49609462863292, + "queue_mean_ms": 35.24152276439747 + } + ] + } +} diff --git a/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-runtime-integrity.json b/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-runtime-integrity.json new file mode 100644 index 000000000..d96704ea2 --- /dev/null +++ b/tools/ingressbench/evidence/replica-capacity-2026-10-28/two-runtime-integrity.json @@ -0,0 +1,25 @@ +{ + "as_of": 1793197879.011792, + "samples": 17, + "commands_failed": [], + "app_restarts": { + "mcp-registry-c46895d95-4bcgm/mcp-registry": 0, + "mcp-registry-c46895d95-fv2tn/mcp-registry": 0, + "registry-pg-1/postgres": 0 + }, + "node_count_timeline": [ + { + "at": 1793197262.265404, + "nodes": 4 + } + ], + "db_throttled_usec_range": [ + 893774, + 893774 + ], + "db_endpoint_samples": 89, + "db_endpoint_errors_or_empty": [], + "db_endpoint_not_ready": [], + "cadvisor_failures_or_empty": [], + "lb_collection_failures": [] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/app-trace/app-pressure-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/app-trace/app-pressure-summary.json new file mode 100644 index 000000000..ad289cd9c --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/app-trace/app-pressure-summary.json @@ -0,0 +1,26 @@ +[ + { + "pod": "mcp-registry-76d45b4b4-bg8m4", + "metric": "container_cpu_usage_seconds_total", + "sampled_seconds": 213.2369999885559, + "rate": 0.7390077660464989 + }, + { + "pod": "mcp-registry-76d45b4b4-bg8m4", + "metric": "container_pressure_cpu_stalled_seconds_total", + "sampled_seconds": 213.2369999885559, + "rate": 0.10342767437725926 + }, + { + "pod": "mcp-registry-76d45b4b4-54cwg", + "metric": "container_cpu_usage_seconds_total", + "sampled_seconds": 206.74000000953674, + "rate": 0.7316454144965778 + }, + { + "pod": "mcp-registry-76d45b4b4-54cwg", + "metric": "container_pressure_cpu_stalled_seconds_total", + "sampled_seconds": 206.74000000953674, + "rate": 0.17915423236089512 + } +] \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/app-trace/capacity-timeline.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/app-trace/capacity-timeline.json new file mode 100644 index 000000000..17e97f8f0 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/app-trace/capacity-timeline.json @@ -0,0 +1,173 @@ +{ + "command_failures": [], + "points": [ + { + "at": "2026-10-28T22:46:09.601731+00:00", + "relative_seconds": -6.5, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 3 + }, + { + "at": "2026-10-28T22:46:39.602345+00:00", + "relative_seconds": 23.5, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 3 + }, + { + "at": "2026-10-28T22:47:09.612536+00:00", + "relative_seconds": 53.5, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 3 + }, + { + "at": "2026-10-28T22:47:39.614648+00:00", + "relative_seconds": 83.5, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 3 + }, + { + "at": "2026-10-28T22:48:09.624825+00:00", + "relative_seconds": 113.5, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 3 + }, + { + "at": "2026-10-28T22:48:39.630776+00:00", + "relative_seconds": 143.5, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 3 + }, + { + "at": "2026-10-28T22:49:09.640976+00:00", + "relative_seconds": 173.5, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 3 + }, + { + "at": "2026-10-28T22:49:39.651161+00:00", + "relative_seconds": 203.6, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 3 + }, + { + "at": "2026-10-28T22:50:09.661338+00:00", + "relative_seconds": 233.6, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 3 + }, + { + "at": "2026-10-28T22:50:39.670953+00:00", + "relative_seconds": 263.6, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 3 + }, + { + "at": "2026-10-28T22:51:09.676108+00:00", + "relative_seconds": 293.6, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 3 + }, + { + "at": "2026-10-28T22:51:39.683118+00:00", + "relative_seconds": 323.6, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 3 + }, + { + "at": "2026-10-28T22:52:09.691398+00:00", + "relative_seconds": 353.6, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 3 + }, + { + "at": "2026-10-28T22:52:39.694854+00:00", + "relative_seconds": 383.6, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 3 + } + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/app-trace/checkout-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/app-trace/checkout-summary.json new file mode 100644 index 000000000..a765f960a --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/app-trace/checkout-summary.json @@ -0,0 +1,19 @@ +{ + "unit": "milliseconds", + "final_replay_minute": { + "checkout_mean": { + "list": 27.177531357142858, + "list_page": 35.44172178520267, + "list_search": 43.4672156377171, + "list_updated": 6.474163999999999, + "other": 24.879902612051456 + }, + "after_query_mean": { + "list": 0.049986392857142854, + "list_page": 0.13220790630985174, + "list_search": 0.03289944168734491, + "list_updated": 0.06495604477611941, + "other": 0.046745094786729856 + } + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/app-trace/generator-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/app-trace/generator-summary.json new file mode 100644 index 000000000..8a5dc3fbd --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/app-trace/generator-summary.json @@ -0,0 +1,22 @@ +{ + "samples": 24, + "cpu_busy_mean_percent": 9.380035652408129, + "cpu_busy_max_percent": 12.673946957878314, + "network_error_drop_deltas": { + "ens4": [ + 0, + 0, + 0, + 0 + ], + "lo": [ + 0, + 0, + 0, + 0 + ] + }, + "limitations": [ + "Aggregate CPU and device counters do not exclude single-thread or external-network bottlenecks." + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/app-trace/metric-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/app-trace/metric-summary.json new file mode 100644 index 000000000..1177984d7 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/app-trace/metric-summary.json @@ -0,0 +1,250 @@ +{ + "arrivals": { + "/v0.1/servers": { + "samples": 6, + "maximum": 311.4, + "mean": 297.65 + }, + "/v0.1/servers/{serverName}/versions": { + "samples": 6, + "maximum": 3.916666666666667, + "mean": 3.4333333333333336 + }, + "/v0.1/servers/{serverName}/versions/{version}": { + "samples": 6, + "maximum": 0.925, + "mean": 0.7361111111111112 + }, + "/v0/servers": { + "samples": 6, + "maximum": 39.43333333333334, + "mean": 37.975 + }, + "/v0/servers/{serverName}/versions/{version}": { + "samples": 6, + "maximum": 8.3, + "mean": 4.872222222222223 + } + }, + "query_rate": { + "list": { + "samples": 6, + "maximum": 0.4666666666666667, + "mean": 0.4277777777777778 + }, + "list_page": { + "samples": 6, + "maximum": 341.3666666666667, + "mean": 331.26944444444445 + }, + "list_search": { + "samples": 6, + "maximum": 3.3583333333333334, + "mean": 2.413888888888889 + }, + "list_updated": { + "samples": 6, + "maximum": 0.7166666666666666, + "mean": 0.5972222222222222 + }, + "other": { + "samples": 8, + "maximum": 12.308333333333334, + "mean": 6.73125 + } + }, + "query_p95": { + "list": { + "samples": 6, + "maximum": 0.4874999999999999, + "mean": 0.4340277777777778 + }, + "list_page": { + "samples": 8, + "maximum": 0.4056010158013544, + "mean": 0.26510065315286846 + }, + "list_search": { + "samples": 8, + "maximum": 1.4649999999999999, + "mean": 0.7074782135076256 + }, + "list_updated": { + "samples": 8, + "maximum": 0.32083333333333286, + "mean": 0.18114543650793635 + }, + "other": { + "samples": 8, + "maximum": 0.46718749999999987, + "mean": 0.3665788683504789 + } + }, + "pool_p95": { + "list": { + "samples": 6, + "maximum": 1.8749999999999987, + "mean": 1.4888888888888883 + }, + "list_page": { + "samples": 8, + "maximum": 1.2051969823973154, + "mean": 0.7388997583034459 + }, + "list_search": { + "samples": 8, + "maximum": 1.6656249999999992, + "mean": 0.7282583333333327 + }, + "list_updated": { + "samples": 8, + "maximum": 1.4249999999999972, + "mean": 0.713779166666666 + }, + "other": { + "samples": 8, + "maximum": 2.2978260869565217, + "mean": 1.705681475558744 + } + }, + "pool_pending": { + "list": { + "samples": 8, + "maximum": 0.0, + "mean": 0.0 + }, + "list_page": { + "samples": 8, + "maximum": 189.0, + "mean": 47.75 + }, + "list_search": { + "samples": 8, + "maximum": 0.0, + "mean": 0.0 + }, + "list_updated": { + "samples": 8, + "maximum": 0.0, + "mean": 0.0 + }, + "other": { + "samples": 8, + "maximum": 10.0, + "mean": 2.5 + } + }, + "inflight": { + "/v0.1/servers": { + "samples": 8, + "maximum": 275.0, + "mean": 78.0 + }, + "/v0.1/servers/{serverName}/versions": { + "samples": 8, + "maximum": 12.0, + "mean": 3.0 + }, + "/v0.1/servers/{serverName}/versions/{version}": { + "samples": 8, + "maximum": 0.0, + "mean": 0.0 + }, + "/v0/servers": { + "samples": 8, + "maximum": 34.0, + "mean": 10.25 + }, + "/v0/servers/{serverName}/versions/{version}": { + "samples": 8, + "maximum": 5.0, + "mean": 1.25 + } + }, + "http_p95": { + "/v0.1/servers": { + "samples": 8, + "maximum": 1.743294020844759, + "mean": 1.131801530068524 + }, + "/v0.1/servers/{serverName}/versions": { + "samples": 6, + "maximum": 1.662142857142855, + "mean": 1.3294829424307046 + }, + "/v0.1/servers/{serverName}/versions/{version}": { + "samples": 6, + "maximum": 1.4499999999999977, + "mean": 1.1691666666666656 + }, + "/v0/servers": { + "samples": 8, + "maximum": 1.5605491329479735, + "mean": 0.970168458998299 + }, + "/v0/servers/{serverName}/versions/{version}": { + "samples": 6, + "maximum": 2.0989130434782606, + "mean": 1.1573528826246193 + } + }, + "checkout_mean": { + "list": { + "samples": 6, + "maximum": 0.10878252735999999, + "mean": 0.0719000783918254 + }, + "list_page": { + "samples": 6, + "maximum": 0.07417823184181233, + "mean": 0.05681776448060554 + }, + "list_search": { + "samples": 6, + "maximum": 0.30931861202247196, + "mean": 0.15976020422585005 + }, + "list_updated": { + "samples": 6, + "maximum": 0.053749890674418606, + "mean": 0.033903200283945986 + }, + "other": { + "samples": 8, + "maximum": 0.16398855661946904, + "mean": 0.08214665866529708 + } + }, + "after_query_mean": { + "list": { + "samples": 6, + "maximum": 6.243431249999999e-05, + "mean": 5.164908845238095e-05 + }, + "list_page": { + "samples": 6, + "maximum": 0.00019947491216678055, + "mean": 0.00017602359083540025 + }, + "list_search": { + "samples": 6, + "maximum": 0.00015387775280898878, + "mean": 8.521260302655568e-05 + }, + "list_updated": { + "samples": 6, + "maximum": 6.776674418604653e-05, + "mean": 6.287837589061446e-05 + }, + "other": { + "samples": 8, + "maximum": 7.725721172022685e-05, + "mean": 5.5738653669489895e-05 + } + }, + "limitations": [ + "App scrape interval 60s; 30s query points may repeat samples. Rates/histograms use rolling windows and are not phase-isolated distributions.", + "Pool acquisition includes scheduler delay; zero sampled pending gauges do not prove absence of transient waits.", + "Time-series maxima for per-replica means are not global request-weighted means." + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/baseline-trace-assessment.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/baseline-trace-assessment.json new file mode 100644 index 000000000..10facb013 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/baseline-trace-assessment.json @@ -0,0 +1,191 @@ +{ + "at": "2026-10-28T22:43:36.576514+00:00", + "origin_comparison_window": { + "staging_start": 1793227141.90652, + "staging_end": 1793227201.90652, + "query_samples": [ + 1793227164.908, + 1793227194.908 + ], + "method": "Compare final replay minute. Interpolate 60s production query-rate samples at matching source-relative timestamps." + }, + "workload_fidelity_pass": true, + "checks": [ + { + "name": "client_cache_HIT", + "staging": 0.6481921590682317, + "production": 0.6927664556730453, + "tolerance": 0.05, + "relative": false, + "pass": true + }, + { + "name": "client_cache_MISS", + "staging": 0.16084613849531032, + "production": 0.14796385057260986, + "tolerance": 0.05, + "relative": false, + "pass": true + }, + { + "name": "client_cache_STALE", + "staging": 0.1697900981477114, + "production": 0.1472869635212762, + "tolerance": 0.05, + "relative": false, + "pass": true + }, + { + "name": "client_cache_UPDATING", + "staging": 0.010576731191378965, + "production": 0.0005854158281804544, + "tolerance": 0.05, + "relative": false, + "pass": true + }, + { + "name": "wire_body_bytes_p50", + "staging": 93252, + "production": 93051, + "tolerance": 0.2, + "relative": true, + "pass": true + }, + { + "name": "wire_body_bytes_p95", + "staging": 162313, + "production": 160765, + "tolerance": 0.2, + "relative": true, + "pass": true + }, + { + "name": "distinct_page_keys", + "staging": 11948, + "production": 11833, + "tolerance": 0.2, + "relative": true, + "pass": true + }, + { + "name": "top_128_page_share", + "staging": 0.1352617641067427, + "production": 0.13614689878269975, + "tolerance": 0.05, + "relative": false, + "pass": true + }, + { + "name": "client_rps", + "staging": 918.6833333333333, + "production": 911.0333333333333, + "tolerance": 0.2, + "relative": true, + "pass": true + }, + { + "name": "list_page_origin_queries_rps", + "staging": 325.89583333333337, + "production": 291.554753780365, + "tolerance": 0.2, + "relative": true, + "pass": true + }, + { + "name": "list_search_origin_queries_rps", + "staging": 2.6166666666666667, + "production": 3.09142198562622, + "tolerance": 0.2, + "relative": true, + "pass": true + }, + { + "name": "list_updated_origin_queries_rps", + "staging": 0.4625, + "production": 0.5425055503845215, + "tolerance": 0.2, + "relative": true, + "pass": true + } + ], + "staging_steady_metrics": { + "arrivals": { + "/v0.1/servers": 289.9916666666667, + "/v0.1/servers/{serverName}/versions": 3.208333333333333, + "/v0.1/servers/{serverName}/versions/{version}": 0.7791666666666667, + "/v0/servers": 37.3875, + "/v0/servers/{serverName}/versions/{version}": 6.245833333333334 + }, + "query_rate": { + "list": 0.42500000000000004, + "list_page": 325.89583333333337, + "list_search": 2.6166666666666667, + "list_updated": 0.4625, + "other": 9.975 + }, + "pool_pending": { + "list": 0.5, + "list_page": 63.0, + "list_search": 0.5, + "list_updated": 0.5, + "other": 26.5 + }, + "pool_p95": { + "list": 4.482293801229507, + "list_page": 4.374166936489566, + "list_search": 4.017045454545454, + "list_updated": 4.416666666666666, + "other": 4.114678899082569 + }, + "http_p95": { + "/v0.1/servers": 4.745370502703749, + "/v0.1/servers/{serverName}/versions": 4.575627234422879, + "/v0.1/servers/{serverName}/versions/{version}": 4.668269230769227, + "/v0/servers": 4.6636364804409975, + "/v0/servers/{serverName}/versions/{version}": 3.7467950162220625 + }, + "inflight": { + "/v0.1/servers": 139.0, + "/v0.1/servers/{serverName}/versions": 0.5, + "/v0.1/servers/{serverName}/versions/{version}": 0.0, + "/v0/servers": 14.0, + "/v0/servers/{serverName}/versions/{version}": 32.5 + } + }, + "offered": { + "Scheduled": 274116, + "Completed": 268352, + "Dropped": 5764, + "Errors": { + "page:body:timeout": 8, + "page:request:timeout": 39, + "search:request:timeout": 1 + }, + "LatencyMS": { + "detail_p50": 498.266667, + "detail_p95": 4317.953071, + "detail_p99": 5583.062271, + "page_p50": 38.084056, + "page_p95": 3321.811311, + "page_p99": 5538.748918, + "search_p50": 265.915743, + "search_p95": 3514.35404, + "search_p99": 5409.575506, + "updated_p50": 655.324676, + "updated_p95": 5170.463395, + "updated_p99": 6290.706091 + }, + "Cache": { + "": 2228, + "HIT": 181144, + "MISS": 48386, + "STALE": 31252, + "UPDATING": 5342 + } + }, + "limitations": [ + "One short broad-crawl calibration, not full cyclic recovery validation", + "Search/update/detail/write fidelity remains separately required", + "Read baseline uses prior production corpus with derived schema columns present" + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/baseline-trace/app-pressure-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/baseline-trace/app-pressure-summary.json new file mode 100644 index 000000000..934a35953 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/baseline-trace/app-pressure-summary.json @@ -0,0 +1,26 @@ +[ + { + "pod": "mcp-registry-fdf44474c-d6wvz", + "metric": "container_cpu_usage_seconds_total", + "sampled_seconds": 215.88899993896484, + "rate": 1.1004152970608232 + }, + { + "pod": "mcp-registry-fdf44474c-d6wvz", + "metric": "container_pressure_cpu_stalled_seconds_total", + "sampled_seconds": 215.88899993896484, + "rate": 0.17373765690055587 + }, + { + "pod": "mcp-registry-fdf44474c-5t5wh", + "metric": "container_cpu_usage_seconds_total", + "sampled_seconds": 223.81700015068054, + "rate": 1.0289273372664305 + }, + { + "pod": "mcp-registry-fdf44474c-5t5wh", + "metric": "container_pressure_cpu_stalled_seconds_total", + "sampled_seconds": 223.81700015068054, + "rate": 0.3045389668975632 + } +] \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/baseline-trace/capacity-timeline.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/baseline-trace/capacity-timeline.json new file mode 100644 index 000000000..5c79385d9 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/baseline-trace/capacity-timeline.json @@ -0,0 +1,173 @@ +{ + "command_failures": [], + "points": [ + { + "at": "2026-10-28T22:35:54.956189+00:00", + "relative_seconds": -7.0, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T22:36:24.966365+00:00", + "relative_seconds": 23.1, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T22:36:54.973894+00:00", + "relative_seconds": 53.1, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T22:37:24.974092+00:00", + "relative_seconds": 83.1, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T22:37:54.979084+00:00", + "relative_seconds": 113.1, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T22:38:24.989258+00:00", + "relative_seconds": 143.1, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T22:38:54.999390+00:00", + "relative_seconds": 173.1, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T22:39:25.006462+00:00", + "relative_seconds": 203.1, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T22:39:55.014726+00:00", + "relative_seconds": 233.1, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T22:40:25.017826+00:00", + "relative_seconds": 263.1, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T22:40:55.026871+00:00", + "relative_seconds": 293.1, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T22:41:25.037039+00:00", + "relative_seconds": 323.1, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T22:41:55.047203+00:00", + "relative_seconds": 353.1, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T22:42:25.056700+00:00", + "relative_seconds": 383.2, + "nodes": 2, + "hpa": { + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + } + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/baseline-trace/generator-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/baseline-trace/generator-summary.json new file mode 100644 index 000000000..92909dac8 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/baseline-trace/generator-summary.json @@ -0,0 +1,22 @@ +{ + "samples": 24, + "cpu_busy_mean_percent": 9.950871641380976, + "cpu_busy_max_percent": 13.326776493730023, + "network_error_drop_deltas": { + "ens4": [ + 0, + 0, + 0, + 0 + ], + "lo": [ + 0, + 0, + 0, + 0 + ] + }, + "limitations": [ + "Aggregate CPU and device counters do not exclude single-thread or external-network bottlenecks." + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/baseline-trace/metric-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/baseline-trace/metric-summary.json new file mode 100644 index 000000000..ec0dc956d --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/baseline-trace/metric-summary.json @@ -0,0 +1,196 @@ +{ + "arrivals": { + "/v0.1/servers": { + "samples": 8, + "maximum": 299.65, + "mean": 192.22291666666666 + }, + "/v0.1/servers/{serverName}/versions": { + "samples": 5, + "maximum": 3.6333333333333333, + "mean": 3.0533333333333332 + }, + "/v0.1/servers/{serverName}/versions/{version}": { + "samples": 5, + "maximum": 0.875, + "mean": 0.6033333333333333 + }, + "/v0/servers": { + "samples": 8, + "maximum": 39.108333333333334, + "mean": 25.179166666666667 + }, + "/v0/servers/{serverName}/versions/{version}": { + "samples": 5, + "maximum": 6.9, + "mean": 4.09 + } + }, + "query_rate": { + "list": { + "samples": 8, + "maximum": 0.43333333333333335, + "mean": 0.2604166666666667 + }, + "list_page": { + "samples": 8, + "maximum": 332.775, + "mean": 214.696875 + }, + "list_search": { + "samples": 8, + "maximum": 2.908333333333333, + "mean": 1.3677083333333333 + }, + "list_updated": { + "samples": 8, + "maximum": 0.525, + "mean": 0.325 + }, + "other": { + "samples": 8, + "maximum": 10.816666666666666, + "mean": 5.179166666666666 + } + }, + "query_p95": { + "list": { + "samples": 6, + "maximum": 1.4500000000000002, + "mean": 0.9420940170940172 + }, + "list_page": { + "samples": 7, + "maximum": 0.9396345116836431, + "mean": 0.8730331175870846 + }, + "list_search": { + "samples": 7, + "maximum": 1.210714285714282, + "mean": 0.955282510752893 + }, + "list_updated": { + "samples": 7, + "maximum": 0.36250000000000004, + "mean": 0.3207291666666663 + }, + "other": { + "samples": 7, + "maximum": 0.4231617647058821, + "mean": 0.348655307688532 + } + }, + "pool_p95": { + "list": { + "samples": 6, + "maximum": 4.784883720930233, + "mean": 4.594033863763926 + }, + "list_page": { + "samples": 7, + "maximum": 4.717475944846742, + "mean": 3.974821584337703 + }, + "list_search": { + "samples": 7, + "maximum": 4.4375, + "mean": 3.714822974644403 + }, + "list_updated": { + "samples": 7, + "maximum": 4.783333333333332, + "mean": 3.683354591836734 + }, + "other": { + "samples": 7, + "maximum": 4.674698795180722, + "mean": 3.853030121694338 + } + }, + "pool_pending": { + "list": { + "samples": 8, + "maximum": 1.0, + "mean": 0.25 + }, + "list_page": { + "samples": 8, + "maximum": 397.0, + "mean": 121.5 + }, + "list_search": { + "samples": 8, + "maximum": 1.0, + "mean": 0.5 + }, + "list_updated": { + "samples": 8, + "maximum": 1.0, + "mean": 0.25 + }, + "other": { + "samples": 8, + "maximum": 38.0, + "mean": 10.25 + } + }, + "inflight": { + "/v0.1/servers": { + "samples": 8, + "maximum": 484.0, + "mean": 180.875 + }, + "/v0.1/servers/{serverName}/versions": { + "samples": 7, + "maximum": 1.0, + "mean": 0.5714285714285714 + }, + "/v0.1/servers/{serverName}/versions/{version}": { + "samples": 7, + "maximum": 1.0, + "mean": 0.2857142857142857 + }, + "/v0/servers": { + "samples": 8, + "maximum": 53.0, + "mean": 20.75 + }, + "/v0/servers/{serverName}/versions/{version}": { + "samples": 7, + "maximum": 44.0, + "mean": 12.857142857142858 + } + }, + "http_p95": { + "/v0.1/servers": { + "samples": 7, + "maximum": 6.279439252336436, + "mean": 4.604295557078723 + }, + "/v0.1/servers/{serverName}/versions": { + "samples": 5, + "maximum": 7.194444444444441, + "mean": 5.188002640690418 + }, + "/v0.1/servers/{serverName}/versions/{version}": { + "samples": 5, + "maximum": 8.999999999999996, + "mean": 6.279807692307689 + }, + "/v0/servers": { + "samples": 7, + "maximum": 5.047839506172848, + "mean": 4.370864809535785 + }, + "/v0/servers/{serverName}/versions/{version}": { + "samples": 5, + "maximum": 5.9374999999999964, + "mean": 4.4901176971649175 + } + }, + "limitations": [ + "App scrape interval 60s; 30s query points may repeat samples. Rates/histograms use rolling windows and are not phase-isolated distributions.", + "Pool acquisition includes scheduler delay; zero sampled pending gauges do not prove absence of transient waits.", + "Time-series maxima for per-replica means are not global request-weighted means." + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/broad-comparison.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/broad-comparison.json new file mode 100644 index 000000000..b4e3da293 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/broad-comparison.json @@ -0,0 +1,171 @@ +{ + "baseline-trace": { + "offered": 274116, + "attempted": 268352, + "dropped": 5764, + "drop_percent": 2.102759415721811, + "transport_errors": 48, + "status": { + "detail:200": 2160, + "detail:404": 28, + "page:0": 47, + "page:200": 265016, + "search:0": 1, + "search:200": 960, + "updated:200": 140 + }, + "latency_ms": { + "detail_p50": 498.266667, + "detail_p95": 4317.953071, + "detail_p99": 5583.062271, + "page_p50": 38.084056, + "page_p95": 3321.811311, + "page_p99": 5538.748918, + "search_p50": 265.915743, + "search_p95": 3514.35404, + "search_p99": 5409.575506, + "updated_p50": 655.324676, + "updated_p95": 5170.463395, + "updated_p99": 6290.706091 + }, + "drop_windows": [ + { + "start_relative_seconds": -1.907, + "end_relative_seconds": 8.093, + "dropped": 2503 + }, + { + "start_relative_seconds": 8.093, + "end_relative_seconds": 18.093, + "dropped": 741 + }, + { + "start_relative_seconds": 48.093, + "end_relative_seconds": 58.093, + "dropped": 1670 + }, + { + "start_relative_seconds": 58.093, + "end_relative_seconds": 68.093, + "dropped": 79 + }, + { + "start_relative_seconds": 68.093, + "end_relative_seconds": 78.093, + "dropped": 771 + } + ], + "no_drops_after_seconds": 78.093, + "ready_ingress_at_start": 2, + "terminating_ingress_at_start": 0, + "trace_sha256": "e4db9792a9b1ec06c3973851aaff4699161d18c235f5f8276e087e66611b2233", + "catalog_sha256": "9bbb7b82489608e191cdee82a096c42eb2a6d29ee43b594c9b2326e35d457e48" + }, + "app-trace": { + "offered": 274116, + "attempted": 271813, + "dropped": 2303, + "drop_percent": 0.8401552627354842, + "transport_errors": 0, + "status": { + "detail:200": 2179, + "detail:404": 28, + "page:200": 268500, + "search:200": 965, + "updated:200": 141 + }, + "latency_ms": { + "detail_p50": 13.198039, + "detail_p95": 1729.419481, + "detail_p99": 3350.082628, + "page_p50": 12.405141, + "page_p95": 1597.433442, + "page_p99": 3281.870462, + "search_p50": 17.609313, + "search_p95": 2557.167073, + "search_p99": 4124.2338, + "updated_p50": 14.233714, + "updated_p95": 1560.567641, + "updated_p99": 3313.343124 + }, + "drop_windows": [ + { + "start_relative_seconds": -6.1, + "end_relative_seconds": 3.9, + "dropped": 1194 + }, + { + "start_relative_seconds": 3.9, + "end_relative_seconds": 13.9, + "dropped": 1094 + }, + { + "start_relative_seconds": 33.9, + "end_relative_seconds": 43.9, + "dropped": 15 + } + ], + "no_drops_after_seconds": 43.9, + "ready_ingress_at_start": 2, + "terminating_ingress_at_start": 0, + "trace_sha256": "e4db9792a9b1ec06c3973851aaff4699161d18c235f5f8276e087e66611b2233", + "catalog_sha256": "9bbb7b82489608e191cdee82a096c42eb2a6d29ee43b594c9b2326e35d457e48" + }, + "infra-trace": { + "offered": 274116, + "attempted": 272191, + "dropped": 1925, + "drop_percent": 0.7022574384567117, + "transport_errors": 7, + "status": { + "detail:200": 2148, + "detail:404": 22, + "detail:429": 42, + "page:0": 7, + "page:200": 268862, + "search:200": 969, + "updated:200": 141 + }, + "latency_ms": { + "detail_p50": 10.093021, + "detail_p95": 2028.839681, + "detail_p99": 3866.843287, + "page_p50": 13.478108, + "page_p95": 1667.372591, + "page_p99": 3711.801739, + "search_p50": 17.620171, + "search_p95": 1774.427394, + "search_p99": 3305.855384, + "updated_p50": 13.02561, + "updated_p95": 2085.618734, + "updated_p99": 2645.0969 + }, + "drop_windows": [ + { + "start_relative_seconds": -6.933, + "end_relative_seconds": 3.067, + "dropped": 821 + }, + { + "start_relative_seconds": 3.067, + "end_relative_seconds": 13.067, + "dropped": 566 + }, + { + "start_relative_seconds": 43.067, + "end_relative_seconds": 53.067, + "dropped": 129 + }, + { + "start_relative_seconds": 73.067, + "end_relative_seconds": 83.067, + "dropped": 409 + } + ], + "no_drops_after_seconds": 83.067, + "ready_ingress_at_start": 2, + "terminating_ingress_at_start": 0, + "trace_sha256": "e4db9792a9b1ec06c3973851aaff4699161d18c235f5f8276e087e66611b2233", + "catalog_sha256": "9bbb7b82489608e191cdee82a096c42eb2a6d29ee43b594c9b2326e35d457e48" + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/cache-transition-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/cache-transition-summary.json new file mode 100644 index 000000000..4c8dc9969 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/cache-transition-summary.json @@ -0,0 +1,128 @@ +{ + "baseline-trace": { + "startup": { + "query_complete": true, + "unparsed": 3, + "client_count": 68451, + "cache_share": { + "none": 0.0054, + "MISS": 0.2612, + "HIT": 0.6742, + "STALE": 0.0471, + "UPDATING": 0.012 + }, + "response_bytes": { + "mean": 99663.90341996464, + "p50": 92971, + "p95": 160994, + "p99": 307465, + "max": 1326842 + }, + "zero_request_length_records": 3225 + }, + "final_minute": { + "query_complete": true, + "unparsed": 0, + "client_count": 55121, + "cache_share": { + "HIT": 0.6482, + "MISS": 0.1608, + "STALE": 0.1698, + "UPDATING": 0.0106, + "none": 0.0106 + }, + "response_bytes": { + "mean": 99283.06968306091, + "p50": 93252, + "p95": 162313, + "p99": 284389, + "max": 3515562 + }, + "zero_request_length_records": 9358 + } + }, + "app-trace": { + "startup": { + "query_complete": true, + "unparsed": 3, + "client_count": 73474, + "cache_share": { + "MISS": 0.262, + "HIT": 0.6695, + "none": 0.0055, + "STALE": 0.0542, + "UPDATING": 0.0088 + }, + "response_bytes": { + "mean": 99783.27055829273, + "p50": 93041, + "p95": 163044, + "p99": 307475, + "max": 1327783 + }, + "zero_request_length_records": 3981 + }, + "final_minute": { + "query_complete": true, + "unparsed": 0, + "client_count": 53212, + "cache_share": { + "HIT": 0.6592, + "STALE": 0.1707, + "MISS": 0.1572, + "none": 0.0103, + "UPDATING": 0.0026 + }, + "response_bytes": { + "mean": 98891.59078779223, + "p50": 93189, + "p95": 161266, + "p99": 275411, + "max": 1368331 + }, + "zero_request_length_records": 9084 + } + }, + "infra-trace": { + "startup": { + "query_complete": true, + "unparsed": 36, + "client_count": 73736, + "cache_share": { + "MISS": 0.2773, + "none": 0.0056, + "HIT": 0.665, + "STALE": 0.0486, + "UPDATING": 0.0036 + }, + "response_bytes": { + "mean": 99820.04589345775, + "p50": 93041, + "p95": 162555, + "p99": 307481, + "max": 1327784 + }, + "zero_request_length_records": 3581 + }, + "final_minute": { + "query_complete": true, + "unparsed": 42, + "client_count": 54536, + "cache_share": { + "HIT": 0.6331, + "MISS": 0.1946, + "STALE": 0.162, + "none": 0.0101, + "UPDATING": 0.0002 + }, + "response_bytes": { + "mean": 99122.02930174564, + "p50": 93327, + "p95": 161295, + "p99": 277999, + "max": 1368338 + }, + "zero_request_length_records": 8837 + } + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/cleanup-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/cleanup-summary.json new file mode 100644 index 000000000..8761239a6 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/cleanup-summary.json @@ -0,0 +1,21 @@ +{ + "at": 1793230940.671833, + "bulk_whitelist_restored": true, + "primary_ingress_unchanged": true, + "application_ingress_database_ready": true, + "owned_generator_and_disk_absent": [ + "instances/registry-workload-bench-20261029", + "disks/registry-workload-bench-20261029" + ], + "response_fixtures_equal": 48, + "corpus_names": 52366, + "corpus_versions": 187108, + "test_publications": 0, + "restored_snapshot_nodes": 4, + "node_autoscaler_bounds": [ + 2, + 4 + ], + "node_consolidation_pending": true, + "production_changed": false +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/comparison-matrix.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/comparison-matrix.json new file mode 100644 index 000000000..b7071891c --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/comparison-matrix.json @@ -0,0 +1,40 @@ +{ + "baseline": { + "image": "1.8.6", + "app_cpu": "250m", + "app_replicas": "2", + "ingress_cpu": "300m", + "ingress_replicas": "2", + "db_cpu_request": "200m", + "db_cpu_limit": "3", + "pool_per_app": "60/10", + "nodes": "2", + "ingress_memory_request": "90Mi" + }, + "app-only": { + "image": "us-central1-docker.pkg.dev/mcp-registry-staging/registry-staging-experiments/registry:staging-replicas-20261028-v1@sha256:60fb8b7d7931a257542848460f7d1c76a99f16c46ddad795249fb5c25b1c9b0e", + "app_cpu": "250m", + "app_replicas": "2", + "ingress_cpu": "300m", + "ingress_replicas": "2", + "db_cpu_request": "200m", + "db_cpu_limit": "3", + "pool_per_app": "60/10", + "nodes": "2", + "ingress_memory_request": "90Mi" + }, + "app-and-infra": { + "image": "us-central1-docker.pkg.dev/mcp-registry-staging/registry-staging-experiments/registry:staging-replicas-20261028-v1@sha256:60fb8b7d7931a257542848460f7d1c76a99f16c46ddad795249fb5c25b1c9b0e", + "app_cpu": "800m", + "app_replicas": "2\u20134", + "ingress_cpu": "800m", + "ingress_replicas": "2\u20136", + "db_cpu_request": "1500m", + "db_cpu_limit": "3", + "pool_per_app": "30/5", + "nodes": "2\u20134", + "client_connection_lifetime": "60s (candidate ingress setting)", + "idle_start_gate": "Two Ready nodes, two app replicas, two ingress replicas; DB Ready at 1500m reservation. No forced eviction.", + "ingress_memory_request": "512Mi" + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/equivalence-app-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/equivalence-app-summary.json new file mode 100644 index 000000000..a7b4617c1 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/equivalence-app-summary.json @@ -0,0 +1,5 @@ +{ + "checked": 48, + "different": 0, + "reference": "equivalence-reference.json" +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/final-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/final-summary.json new file mode 100644 index 000000000..d478f872c --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/final-summary.json @@ -0,0 +1,82 @@ +{ + "production_changed": false, + "verdict": "App improvement demonstrated in one short cold-start comparison; infrastructure delivery and two-node-within-30-minute gates not passed. Full production cycle and publishes untested.", + "short_gate": { + "baseline": "fail", + "app_only": "fail", + "app_plus_infra": "fail" + }, + "recovery": { + "scheduled": 180000, + "attempted": 180000, + "dropped": 0, + "errors": {}, + "page_p95_ms": 11.382412, + "duration_seconds": 1800, + "replica_return": { + "mcp-registry": { + "first_sample_back_at_two": "2026-10-28T23:16:20.106745+00:00", + "minutes_after_recovery_start": 12.81 + }, + "ingress-nginx-controller": { + "first_sample_back_at_two": "2026-10-28T23:19:20.162376+00:00", + "minutes_after_recovery_start": 15.82 + } + }, + "peak_nodes": 4, + "nodes_in_final_candidate_snapshot": 3, + "final_snapshot_time": "2026-10-28T23:33:44.048174+00:00", + "two_node_gate_pass": false, + "autoscaler_final": "Healthy, minimum 2 / maximum 4, one scale-down candidate remained", + "final_five_minute_gauges": { + "pool_pending": { + "samples": 9, + "maximum_sum": 0.0, + "last_sum": 0.0 + }, + "inflight": { + "samples": 9, + "maximum_sum": 0.0, + "last_sum": 0.0 + } + }, + "db_counter_delta": { + "usage_usec": 451847408, + "nr_periods": 20518, + "nr_throttled": 2, + "throttled_usec": 24131 + }, + "no_forced_consolidation": true + }, + "peak_trace_executed": false, + "publish_test_executed": false, + "limitations": [ + "Empty ingress caches at replay start; observed source was mid-cycle", + "One run per configuration; infrastructure changes tested as a package", + "DB restarted and conditioned only before infrastructure arm", + "Stale staging corpus caused detail 404s; generator route exemption missed detail 429s", + "Source successful reads only, approximate arrival timing, HTTP/1.1 transport and unrecovered TCP sessions", + "Final-minute mix qualification does not qualify complete source cycle" + ], + "restoration": { + "at": 1793230940.671833, + "bulk_whitelist_restored": true, + "primary_ingress_unchanged": true, + "application_ingress_database_ready": true, + "owned_generator_and_disk_absent": [ + "instances/registry-workload-bench-20261029", + "disks/registry-workload-bench-20261029" + ], + "response_fixtures_equal": 48, + "corpus_names": 52366, + "corpus_versions": 187108, + "test_publications": 0, + "restored_snapshot_nodes": 4, + "node_autoscaler_bounds": [ + 2, + 4 + ], + "node_consolidation_pending": true, + "production_changed": false + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/generator-cleanup.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/generator-cleanup.json new file mode 100644 index 000000000..7a525e9c6 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/generator-cleanup.json @@ -0,0 +1,9 @@ +{ + "at": 1793230802.8545501, + "verified_absent": [ + "instances/registry-workload-bench-20261029", + "disks/registry-workload-bench-20261029" + ], + "local_publish_token_removed": true, + "production_changed": false +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/infra-trace/capacity-timeline.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/infra-trace/capacity-timeline.json new file mode 100644 index 000000000..06c27fd68 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/infra-trace/capacity-timeline.json @@ -0,0 +1,1125 @@ +{ + "command_failures": [], + "points": [ + { + "at": "2026-10-28T22:59:19.855773+00:00", + "relative_seconds": -7.1, + "nodes": 2, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T22:59:49.866045+00:00", + "relative_seconds": 22.9, + "nodes": 2, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 4, + "desiredReplicas": 4 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:00:19.876295+00:00", + "relative_seconds": 52.9, + "nodes": 2, + "hpa": { + "mcp-registry": { + "currentReplicas": 3, + "desiredReplicas": 3 + }, + "ingress-nginx-controller": { + "currentReplicas": 4, + "desiredReplicas": 4 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:00:49.886551+00:00", + "relative_seconds": 83.0, + "nodes": 3, + "hpa": { + "mcp-registry": { + "currentReplicas": 3, + "desiredReplicas": 3 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:01:19.896795+00:00", + "relative_seconds": 113.0, + "nodes": 3, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:01:49.899793+00:00", + "relative_seconds": 143.0, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:02:19.902175+00:00", + "relative_seconds": 173.0, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:02:49.903835+00:00", + "relative_seconds": 203.0, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:03:19.914133+00:00", + "relative_seconds": 233.0, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:03:49.919386+00:00", + "relative_seconds": 263.0, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:04:19.919960+00:00", + "relative_seconds": 293.0, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:04:49.920439+00:00", + "relative_seconds": 323.0, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:05:19.930713+00:00", + "relative_seconds": 353.0, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:05:49.934428+00:00", + "relative_seconds": 383.0, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:06:19.938312+00:00", + "relative_seconds": 413.0, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:06:49.948578+00:00", + "relative_seconds": 443.0, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:07:19.958851+00:00", + "relative_seconds": 473.0, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:07:49.966296+00:00", + "relative_seconds": 503.0, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:08:19.976571+00:00", + "relative_seconds": 533.0, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:08:49.986841+00:00", + "relative_seconds": 563.1, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:09:19.993426+00:00", + "relative_seconds": 593.1, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:09:50.003699+00:00", + "relative_seconds": 623.1, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:10:20.013970+00:00", + "relative_seconds": 653.1, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:10:50.021821+00:00", + "relative_seconds": 683.1, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:11:20.025295+00:00", + "relative_seconds": 713.1, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:11:50.032269+00:00", + "relative_seconds": 743.1, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:12:20.042549+00:00", + "relative_seconds": 773.1, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 6, + "desiredReplicas": 6 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:12:50.044446+00:00", + "relative_seconds": 803.1, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 5, + "desiredReplicas": 5 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:13:20.054740+00:00", + "relative_seconds": 833.1, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 4 + }, + "ingress-nginx-controller": { + "currentReplicas": 5, + "desiredReplicas": 5 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:13:50.059815+00:00", + "relative_seconds": 863.1, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 4, + "desiredReplicas": 3 + }, + "ingress-nginx-controller": { + "currentReplicas": 5, + "desiredReplicas": 5 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:14:20.065962+00:00", + "relative_seconds": 893.1, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 3, + "desiredReplicas": 3 + }, + "ingress-nginx-controller": { + "currentReplicas": 5, + "desiredReplicas": 5 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:14:50.076241+00:00", + "relative_seconds": 923.1, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 3, + "desiredReplicas": 3 + }, + "ingress-nginx-controller": { + "currentReplicas": 4, + "desiredReplicas": 4 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:15:20.086522+00:00", + "relative_seconds": 953.2, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 3, + "desiredReplicas": 3 + }, + "ingress-nginx-controller": { + "currentReplicas": 4, + "desiredReplicas": 4 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:15:50.096491+00:00", + "relative_seconds": 983.2, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 3, + "desiredReplicas": 3 + }, + "ingress-nginx-controller": { + "currentReplicas": 4, + "desiredReplicas": 4 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:16:20.106745+00:00", + "relative_seconds": 1013.2, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 4, + "desiredReplicas": 4 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:16:50.110476+00:00", + "relative_seconds": 1043.2, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 3, + "desiredReplicas": 3 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:17:20.120763+00:00", + "relative_seconds": 1073.2, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 3, + "desiredReplicas": 3 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:17:50.131022+00:00", + "relative_seconds": 1103.2, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 3, + "desiredReplicas": 3 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:18:20.141770+00:00", + "relative_seconds": 1133.2, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 3, + "desiredReplicas": 3 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:18:50.152104+00:00", + "relative_seconds": 1163.2, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 3, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:19:20.162376+00:00", + "relative_seconds": 1193.2, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:19:50.171872+00:00", + "relative_seconds": 1223.2, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:20:20.182136+00:00", + "relative_seconds": 1253.2, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:20:50.183287+00:00", + "relative_seconds": 1283.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:21:20.193569+00:00", + "relative_seconds": 1313.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:21:50.197487+00:00", + "relative_seconds": 1343.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:22:20.205374+00:00", + "relative_seconds": 1373.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:22:50.207320+00:00", + "relative_seconds": 1403.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:23:20.217451+00:00", + "relative_seconds": 1433.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:23:50.217970+00:00", + "relative_seconds": 1463.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:24:20.219887+00:00", + "relative_seconds": 1493.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:24:50.223924+00:00", + "relative_seconds": 1523.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:25:20.226647+00:00", + "relative_seconds": 1553.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:25:50.236918+00:00", + "relative_seconds": 1583.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:26:20.239614+00:00", + "relative_seconds": 1613.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:26:50.242417+00:00", + "relative_seconds": 1643.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:27:20.252683+00:00", + "relative_seconds": 1673.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:27:50.259991+00:00", + "relative_seconds": 1703.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:28:20.266376+00:00", + "relative_seconds": 1733.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:28:50.276647+00:00", + "relative_seconds": 1763.3, + "nodes": 4, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:29:20.278326+00:00", + "relative_seconds": 1793.3, + "nodes": 3, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:29:50.286760+00:00", + "relative_seconds": 1823.4, + "nodes": 3, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:30:20.297034+00:00", + "relative_seconds": 1853.4, + "nodes": 3, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:30:50.305040+00:00", + "relative_seconds": 1883.4, + "nodes": 3, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:31:20.315312+00:00", + "relative_seconds": 1913.4, + "nodes": 3, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:31:50.325578+00:00", + "relative_seconds": 1943.4, + "nodes": 3, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 0 + }, + { + "at": "2026-10-28T23:32:20.335861+00:00", + "relative_seconds": 1973.4, + "nodes": 3, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 2 + }, + { + "at": "2026-10-28T23:32:50.346146+00:00", + "relative_seconds": 2003.4, + "nodes": 3, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 2 + }, + { + "at": "2026-10-28T23:33:20.356411+00:00", + "relative_seconds": 2033.4, + "nodes": 3, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 2 + }, + { + "at": "2026-10-28T23:33:50.366677+00:00", + "relative_seconds": 2063.4, + "nodes": 3, + "hpa": { + "mcp-registry": { + "currentReplicas": 2, + "desiredReplicas": 2 + }, + "ingress-nginx-controller": { + "currentReplicas": 2, + "desiredReplicas": 2 + } + }, + "db_throttled_periods": 2 + } + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/infra-trace/generator-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/infra-trace/generator-summary.json new file mode 100644 index 000000000..687ea405a --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/infra-trace/generator-summary.json @@ -0,0 +1,22 @@ +{ + "samples": 24, + "cpu_busy_mean_percent": 8.352402783676979, + "cpu_busy_max_percent": 13.968530381242614, + "network_error_drop_deltas": { + "ens4": [ + 0, + 0, + 0, + 0 + ], + "lo": [ + 0, + 0, + 0, + 0 + ] + }, + "limitations": [ + "Aggregate CPU and device counters do not exclude single-thread or external-network bottlenecks." + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/infra-trace/lb-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/infra-trace/lb-summary.json new file mode 100644 index 000000000..d36debade --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/infra-trace/lb-summary.json @@ -0,0 +1,12 @@ +{ + "samples": 30, + "first_sample_time": 1793228477.804183, + "last_sample_time": 1793228768.002134, + "health_state_observations": { + "HEALTHY": 114, + "UNHEALTHY": 2 + }, + "entries_without_health_status": 2, + "command_failures": 0, + "limitation": "Collection started after the early trace transport failures; cannot establish backend health at those failures." +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/infra-trace/metric-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/infra-trace/metric-summary.json new file mode 100644 index 000000000..c96e9468c --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/infra-trace/metric-summary.json @@ -0,0 +1,250 @@ +{ + "arrivals": { + "/v0.1/servers": { + "samples": 8, + "maximum": 412.3083333333333, + "mean": 222.89895833333333 + }, + "/v0.1/servers/{serverName}/versions": { + "samples": 5, + "maximum": 3.6750000000000003, + "mean": 3.02 + }, + "/v0.1/servers/{serverName}/versions/{version}": { + "samples": 5, + "maximum": 0.55, + "mean": 0.45166666666666666 + }, + "/v0/servers": { + "samples": 8, + "maximum": 44.333333333333336, + "mean": 25.49791666666667 + }, + "/v0/servers/{serverName}/versions/{version}": { + "samples": 5, + "maximum": 8.3, + "mean": 3.6166666666666667 + } + }, + "query_rate": { + "list": { + "samples": 8, + "maximum": 0.4833333333333333, + "mean": 0.23541666666666666 + }, + "list_page": { + "samples": 8, + "maximum": 451.51666666666665, + "mean": 245.71979166666665 + }, + "list_search": { + "samples": 8, + "maximum": 2.925, + "mean": 1.4197916666666666 + }, + "list_updated": { + "samples": 8, + "maximum": 0.8833333333333333, + "mean": 0.5447916666666667 + }, + "other": { + "samples": 8, + "maximum": 12.25, + "mean": 5.391666666666667 + } + }, + "query_p95": { + "list": { + "samples": 6, + "maximum": 0.20840909090909088, + "mean": 0.19817765974967055 + }, + "list_page": { + "samples": 8, + "maximum": 0.2130371222076215, + "mean": 0.13834787833110204 + }, + "list_search": { + "samples": 8, + "maximum": 0.47786697247706433, + "mean": 0.21957603630798633 + }, + "list_updated": { + "samples": 8, + "maximum": 0.10321428571428565, + "mean": 0.06025068240552732 + }, + "other": { + "samples": 7, + "maximum": 0.18236111111111095, + "mean": 0.13077931672185594 + } + }, + "pool_p95": { + "list": { + "samples": 6, + "maximum": 3.3749999999999942, + "mean": 1.6480555555555543 + }, + "list_page": { + "samples": 8, + "maximum": 3.2011834319526598, + "mean": 1.9392749936854938 + }, + "list_search": { + "samples": 8, + "maximum": 1.9208333333333352, + "mean": 1.2009393740031895 + }, + "list_updated": { + "samples": 8, + "maximum": 1.5659090909090914, + "mean": 0.7085616261848335 + }, + "other": { + "samples": 7, + "maximum": 3.6406249999999982, + "mean": 2.062738116133841 + } + }, + "pool_pending": { + "list": { + "samples": 8, + "maximum": 0.0, + "mean": 0.0 + }, + "list_page": { + "samples": 8, + "maximum": 52.0, + "mean": 13.0 + }, + "list_search": { + "samples": 8, + "maximum": 1.0, + "mean": 0.25 + }, + "list_updated": { + "samples": 8, + "maximum": 0.0, + "mean": 0.0 + }, + "other": { + "samples": 8, + "maximum": 0.0, + "mean": 0.0 + } + }, + "inflight": { + "/v0.1/servers": { + "samples": 8, + "maximum": 81.0, + "mean": 22.0 + }, + "/v0.1/servers/{serverName}/versions": { + "samples": 7, + "maximum": 0.0, + "mean": 0.0 + }, + "/v0.1/servers/{serverName}/versions/{version}": { + "samples": 7, + "maximum": 0.0, + "mean": 0.0 + }, + "/v0/servers": { + "samples": 8, + "maximum": 9.0, + "mean": 2.5 + }, + "/v0/servers/{serverName}/versions/{version}": { + "samples": 7, + "maximum": 0.0, + "mean": 0.0 + } + }, + "http_p95": { + "/v0.1/servers": { + "samples": 8, + "maximum": 3.383300132802122, + "mean": 2.0912901979144274 + }, + "/v0.1/servers/{serverName}/versions": { + "samples": 5, + "maximum": 2.897727272727274, + "mean": 2.0077133173843693 + }, + "/v0.1/servers/{serverName}/versions/{version}": { + "samples": 5, + "maximum": 3.7499999999999987, + "mean": 2.559416666666666 + }, + "/v0/servers": { + "samples": 8, + "maximum": 3.6505456349206327, + "mean": 2.301481974112306 + }, + "/v0/servers/{serverName}/versions/{version}": { + "samples": 5, + "maximum": 3.7272727272727253, + "mean": 1.9579199134199132 + } + }, + "checkout_mean": { + "list": { + "samples": 6, + "maximum": 0.05322223288095238, + "mean": 0.04415722856879772 + }, + "list_page": { + "samples": 8, + "maximum": 0.05743135745080425, + "mean": 0.03457606407038708 + }, + "list_search": { + "samples": 8, + "maximum": 0.12006322580911678, + "mean": 0.059798614055558294 + }, + "list_updated": { + "samples": 8, + "maximum": 0.044479515596153846, + "mean": 0.02252295406464816 + }, + "other": { + "samples": 7, + "maximum": 0.03996609454859967, + "mean": 0.027495147278027394 + } + }, + "after_query_mean": { + "list": { + "samples": 6, + "maximum": 4.5988538461538454e-05, + "mean": 3.224467750532746e-05 + }, + "list_page": { + "samples": 8, + "maximum": 0.00037998979931504927, + "mean": 0.00019555310547246507 + }, + "list_search": { + "samples": 8, + "maximum": 7.992716363636364e-05, + "mean": 5.2699164826507476e-05 + }, + "list_updated": { + "samples": 8, + "maximum": 0.0008353956538461538, + "mean": 0.00040474085032588545 + }, + "other": { + "samples": 7, + "maximum": 0.00016602758782201406, + "mean": 9.460730620584407e-05 + } + }, + "limitations": [ + "App scrape interval 60s; 30s query points may repeat samples. Rates/histograms use rolling windows and are not phase-isolated distributions.", + "Pool acquisition includes scheduler delay; zero sampled pending gauges do not prove absence of transient waits.", + "Time-series maxima for per-replica means are not global request-weighted means." + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/next-gates.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/next-gates.json new file mode 100644 index 000000000..4bd5afb41 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/next-gates.json @@ -0,0 +1,9 @@ +{ + "app_only_short_gate": "failed", + "infrastructure_short_gate": "failed", + "recovery_delivery": "passed: 180000/180000, no drops/errors", + "two_node_recovery_within_30_minutes": "not demonstrated: 3 nodes at final candidate snapshot", + "long_peak": "not run after short gate failure", + "next_acceptance": "Establish cache state from preceding production traffic, resolve corpus/rate-limit mismatches, then compare the same peak. Keep cold-start resilience separate.", + "production_changed": false +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/peak-trace-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/peak-trace-summary.json new file mode 100644 index 000000000..5a4b0a826 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/peak-trace-summary.json @@ -0,0 +1,172 @@ +{ + "source_start": "2026-10-28T17:40:00Z", + "duration_seconds": 600, + "read_count": 1227778, + "distinct_keys": 44193, + "kinds": { + "page": 1220688, + "detail": 4446, + "search": 2175, + "updated": 469 + }, + "protocols": { + "HTTP/1.1": 1218849, + "HTTP/2.0": 8929 + }, + "coverage": [ + { + "start": "2026-10-28T17:40:00+00:00", + "end": "2026-10-28T17:41:00+00:00", + "entries": 131855, + "pages": 24, + "complete": true, + "excluded": { + "outside_start_window": 890, + "background": 7816, + "unparsed": 84, + "unsupported_read": 148, + "non_successful_get": 148 + } + }, + { + "start": "2026-10-28T17:41:00+00:00", + "end": "2026-10-28T17:42:00+00:00", + "entries": 136563, + "pages": 25, + "complete": true, + "excluded": { + "unparsed": 67, + "non_successful_get": 117, + "background": 8386, + "unsupported_read": 187 + } + }, + { + "start": "2026-10-28T17:42:00+00:00", + "end": "2026-10-28T17:43:00+00:00", + "entries": 133368, + "pages": 24, + "complete": true, + "excluded": { + "background": 7658, + "unsupported_read": 193, + "non_successful_get": 168, + "unparsed": 124 + } + }, + { + "start": "2026-10-28T17:43:00+00:00", + "end": "2026-10-28T17:44:00+00:00", + "entries": 131520, + "pages": 24, + "complete": true, + "excluded": { + "background": 8320, + "unsupported_read": 92, + "non_successful_get": 103, + "unparsed": 66 + } + }, + { + "start": "2026-10-28T17:44:00+00:00", + "end": "2026-10-28T17:45:00+00:00", + "entries": 129116, + "pages": 24, + "complete": true, + "excluded": { + "background": 8377, + "unsupported_read": 34, + "non_successful_get": 105, + "unparsed": 46 + } + }, + { + "start": "2026-10-28T17:45:00+00:00", + "end": "2026-10-28T17:46:00+00:00", + "entries": 127345, + "pages": 23, + "complete": true, + "excluded": { + "background": 8268, + "unparsed": 63, + "non_successful_get": 159, + "unsupported_read": 26 + } + }, + { + "start": "2026-10-28T17:46:00+00:00", + "end": "2026-10-28T17:47:00+00:00", + "entries": 128621, + "pages": 23, + "complete": true, + "excluded": { + "background": 8856, + "unsupported_read": 27, + "unparsed": 16, + "non_successful_get": 130 + } + }, + { + "start": "2026-10-28T17:47:00+00:00", + "end": "2026-10-28T17:48:00+00:00", + "entries": 132804, + "pages": 24, + "complete": true, + "excluded": { + "background": 8864, + "non_successful_get": 250, + "unsupported_read": 32, + "unparsed": 55 + } + }, + { + "start": "2026-10-28T17:48:00+00:00", + "end": "2026-10-28T17:49:00+00:00", + "entries": 131936, + "pages": 24, + "complete": true, + "excluded": { + "background": 8767, + "unsupported_read": 22, + "non_successful_get": 219, + "unparsed": 53 + } + }, + { + "start": "2026-10-28T17:49:00+00:00", + "end": "2026-10-28T17:50:00+00:00", + "entries": 131892, + "pages": 24, + "complete": true, + "excluded": { + "background": 9194, + "unparsed": 61, + "non_successful_get": 113, + "unsupported_read": 32 + } + }, + { + "start": "2026-10-28T17:50:00+00:00", + "end": "2026-10-28T17:50:10+00:00", + "entries": 21554, + "pages": 4, + "complete": true, + "excluded": { + "background": 1585, + "outside_start_window": 18830, + "unsupported_read": 3, + "non_successful_get": 33, + "unparsed": 9 + } + } + ], + "trace_sha256": "b7db251c48d7e6b62c5607c439e2dfb4e28ce16717723ed9526b4a464058cd34", + "catalog_sha256": "52fe62adb3b8f4e793718be1957b3543427ce89f8642fa11498d1f54de157d05", + "privacy": "Only allowlisted successful public GET read paths in private staging input files; no raw log records, client IPs, identities, user agents, credentials or request IDs retained. Private paths are not committed.", + "limitations": [ + "Arrival times inferred from completion timestamp minus NGINX request duration; not packet-capture timing", + "Failed reads, writes and unsupported paths omitted and counted", + "HTTP/1.1 generator approximates minority HTTP/2 requests; no source TCP connection identity", + "Staging corpus age and initial cache state still need qualification" + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/production-image-check.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/production-image-check.json new file mode 100644 index 000000000..0060dc6f9 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/production-image-check.json @@ -0,0 +1,15 @@ +{ + "at": 1793227031.888009, + "pods": [ + { + "name": "mcp-registry-8c994bcbc-dvtkt", + "image": "ghcr.io/modelcontextprotocol/registry:1.8.6", + "image_id": "ghcr.io/modelcontextprotocol/registry@sha256:f116000c08b41a477c8c7d90ea3fc7003ff4afb65cfb60cc9ebd82a85096fea3" + }, + { + "name": "mcp-registry-8c994bcbc-kn8jc", + "image": "ghcr.io/modelcontextprotocol/registry:1.8.6", + "image_id": "ghcr.io/modelcontextprotocol/registry@sha256:f116000c08b41a477c8c7d90ea3fc7003ff4afb65cfb60cc9ebd82a85096fea3" + } + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/read-trace-summary.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/read-trace-summary.json new file mode 100644 index 000000000..3af6d79bf --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/read-trace-summary.json @@ -0,0 +1,89 @@ +{ + "source_start": "2026-10-28T18:27:00Z", + "duration_seconds": 240, + "read_count": 274116, + "distinct_keys": 34542, + "kinds": { + "page": 270766, + "detail": 2235, + "search": 974, + "updated": 141 + }, + "protocols": { + "HTTP/1.1": 269137, + "HTTP/2.0": 4979 + }, + "coverage": [ + { + "start": "2026-10-28T18:27:00+00:00", + "end": "2026-10-28T18:28:00+00:00", + "entries": 86190, + "pages": 16, + "complete": true, + "excluded": { + "outside_start_window": 55, + "background": 9561, + "unsupported_read": 770, + "non_successful_get": 54 + } + }, + { + "start": "2026-10-28T18:28:00+00:00", + "end": "2026-10-28T18:29:00+00:00", + "entries": 81300, + "pages": 15, + "complete": true, + "excluded": { + "background": 7905, + "unsupported_read": 457, + "non_successful_get": 139 + } + }, + { + "start": "2026-10-28T18:29:00+00:00", + "end": "2026-10-28T18:30:00+00:00", + "entries": 79766, + "pages": 15, + "complete": true, + "excluded": { + "background": 8658, + "unsupported_read": 36, + "non_successful_get": 105 + } + }, + { + "start": "2026-10-28T18:30:00+00:00", + "end": "2026-10-28T18:31:00+00:00", + "entries": 62714, + "pages": 12, + "complete": true, + "excluded": { + "background": 8052, + "unsupported_read": 34, + "non_successful_get": 61 + } + }, + { + "start": "2026-10-28T18:31:00+00:00", + "end": "2026-10-28T18:31:10+00:00", + "entries": 11512, + "pages": 3, + "complete": true, + "excluded": { + "background": 1349, + "outside_start_window": 10112, + "unsupported_read": 5, + "non_successful_get": 13 + } + } + ], + "trace_sha256": "e4db9792a9b1ec06c3973851aaff4699161d18c235f5f8276e087e66611b2233", + "catalog_sha256": "9bbb7b82489608e191cdee82a096c42eb2a6d29ee43b594c9b2326e35d457e48", + "privacy": "Only allowlisted successful public GET read paths in private staging input files; no raw log records, client IPs, identities, user agents, credentials or request IDs retained. Private paths are not committed.", + "limitations": [ + "Arrival times inferred from completion timestamp minus NGINX request duration; not packet-capture timing", + "Failed reads, writes and unsupported paths omitted and counted", + "HTTP/1.1 generator approximates minority HTTP/2 requests; no source TCP connection identity", + "Staging corpus age and initial cache state still need qualification" + ] +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/recovery-comparison.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/recovery-comparison.json new file mode 100644 index 000000000..1276fc3a0 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/recovery-comparison.json @@ -0,0 +1,74 @@ +{ + "baseline-trace": { + "StartedAt": "2026-10-28T22:40:06.116432761Z", + "FinishedAt": "2026-10-28T22:42:06.111490452Z", + "Scheduled": 12000, + "Completed": 12000, + "Dropped": 0, + "Errors": {}, + "Status": { + "page:200": 11760, + "search:200": 120, + "updated:200": 120 + }, + "LatencyMS": { + "page_p50": 6.8075, + "page_p95": 13.951745, + "page_p99": 22.542307, + "search_p50": 11.56263, + "search_p95": 33.783731, + "search_p99": 36.665879, + "updated_p50": 4.807329, + "updated_p95": 12.495886, + "updated_p99": 13.34996 + } + }, + "app-trace": { + "StartedAt": "2026-10-28T22:50:21.293517697Z", + "FinishedAt": "2026-10-28T22:52:21.288457055Z", + "Scheduled": 12000, + "Completed": 12000, + "Dropped": 0, + "Errors": {}, + "Status": { + "page:200": 11760, + "search:200": 120, + "updated:200": 120 + }, + "LatencyMS": { + "page_p50": 6.54282, + "page_p95": 11.546524, + "page_p99": 16.301016, + "search_p50": 11.820186, + "search_p95": 31.738718, + "search_p99": 33.707896, + "updated_p50": 5.286267, + "updated_p95": 12.189029, + "updated_p99": 14.006378 + } + }, + "infra-trace": { + "StartedAt": "2026-10-28T23:03:31.239411614Z", + "FinishedAt": "2026-10-28T23:33:31.23453527Z", + "Scheduled": 180000, + "Completed": 180000, + "Dropped": 0, + "Errors": {}, + "Status": { + "page:200": 176400, + "search:200": 1800, + "updated:200": 1800 + }, + "LatencyMS": { + "page_p50": 4.002084, + "page_p95": 11.382412, + "page_p99": 17.898149, + "search_p50": 10.449166, + "search_p95": 33.879368, + "search_p99": 44.195112, + "updated_p50": 5.345832, + "updated_p95": 12.438895, + "updated_p99": 17.020728 + } + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/restoration-scaling-events.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/restoration-scaling-events.json new file mode 100644 index 000000000..09ca42722 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/restoration-scaling-events.json @@ -0,0 +1,74 @@ +[ + { + "at": "2026-10-28T23:34:04Z", + "reason": "ScaleDown", + "object": "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-9djm", + "message": "marked the node as toBeDeleted/unschedulable" + }, + { + "at": "2026-10-28T23:34:05Z", + "reason": "ScaleDown", + "object": "collector-r2q4q", + "message": "deleting pod for node scale down" + }, + { + "at": "2026-10-28T23:35:01Z", + "reason": "FailedScheduling", + "object": "ingress-nginx-controller-54dc7ccbb8-4fmgh", + "message": "0/3 nodes are available: 1 Insufficient cpu, 1 node(s) didn't match pod topology spread constraints, 1 node(s) were unschedulable. no new claims to deallocate, preemption: 0/3 nodes are available: 1 Preemption is not helpful for scheduling, 2 No preemption victims found for incoming pod." + }, + { + "at": "2026-10-28T23:34:43Z", + "reason": "TriggeredScaleUp", + "object": "ingress-nginx-controller-54dc7ccbb8-4fmgh", + "message": "Pod triggered scale-up: [{https://www.googleapis.com/compute/v1/projects/mcp-registry-staging/zones/us-central1-b/instanceGroups/gke-mcp-registry-sta-mcp-registry-sta-5d130b39-grp 2->3 (max: 4)}]" + }, + { + "at": "2026-10-28T23:35:23Z", + "reason": "FailedScheduling", + "object": "ingress-nginx-controller-54dc7ccbb8-4fmgh", + "message": "0/4 nodes are available: 1 Insufficient cpu, 1 node(s) didn't match pod topology spread constraints, 1 node(s) had untolerated taint(s), 1 node(s) were unschedulable. no new claims to deallocate, preemption: 0/4 nodes are available: 2 No preemption victims found for incoming pod, 2 Preemption is not helpful for scheduling." + }, + { + "at": "2026-10-28T23:35:24Z", + "reason": "FailedScheduling", + "object": "ingress-nginx-controller-54dc7ccbb8-4fmgh", + "message": "0/5 nodes are available: 1 Insufficient cpu, 1 node(s) didn't match pod topology spread constraints, 1 node(s) were unschedulable, 2 node(s) had untolerated taint(s). no new claims to deallocate, preemption: 0/5 nodes are available: 2 No preemption victims found for incoming pod, 3 Preemption is not helpful for scheduling." + }, + { + "at": "2026-10-28T23:34:04Z", + "reason": "ScaleDownEmpty", + "object": "cluster-autoscaler-status", + "message": "Scale-down: removing empty node \"gke-mcp-registry-sta-mcp-registry-sta-5d130b39-9djm\"" + }, + { + "at": "2026-10-28T23:34:07Z", + "reason": "ScaleDownEmpty", + "object": "cluster-autoscaler-status", + "message": "Scale-down: empty node gke-mcp-registry-sta-mcp-registry-sta-5d130b39-9djm removed" + }, + { + "at": "2026-10-28T23:34:05Z", + "reason": "ScaleDown", + "object": "fluentbit-gke-ncxjz", + "message": "deleting pod for node scale down" + }, + { + "at": "2026-10-28T23:34:05Z", + "reason": "ScaleDown", + "object": "gke-metrics-agent-d9chh", + "message": "deleting pod for node scale down" + }, + { + "at": "2026-10-28T23:34:05Z", + "reason": "ScaleDown", + "object": "pdcsi-node-t2pkv", + "message": "deleting pod for node scale down" + }, + { + "at": "2026-10-28T23:34:05Z", + "reason": "ScaleDown", + "object": "opentelemetry-collector-agent-gfbmw", + "message": "deleting pod for node scale down" + } +] \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/restoration.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/restoration.json new file mode 100644 index 000000000..b357874b0 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/restoration.json @@ -0,0 +1,11 @@ +{ + "at": 1793230749.1713312, + "verified": [ + "deployment/mcp-registry", + "deployment/ingress-nginx-controller", + "hpa/ingress-nginx-controller", + "configmap/ingress-nginx-controller", + "cluster.postgresql.cnpg.io/registry-pg" + ], + "production_changed": false +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-comparison-2026-10-29/telemetry-integrity.json b/tools/ingressbench/evidence/workload-comparison-2026-10-29/telemetry-integrity.json new file mode 100644 index 000000000..f59b29f20 --- /dev/null +++ b/tools/ingressbench/evidence/workload-comparison-2026-10-29/telemetry-integrity.json @@ -0,0 +1,38 @@ +{ + "baseline-trace": { + "cadvisor_files": 14, + "failures": [], + "empty_node_telemetry": [], + "metric_queries": 18, + "metric_query_failures": [], + "optional_empty_series": [ + "alerts", + "raw_5xx", + "raw_cancellations" + ] + }, + "app-trace": { + "cadvisor_files": 14, + "failures": [], + "empty_node_telemetry": [], + "metric_queries": 20, + "metric_query_failures": [], + "optional_empty_series": [ + "alerts", + "raw_5xx", + "raw_cancellations" + ] + }, + "infra-trace": { + "cadvisor_files": 70, + "failures": [], + "empty_node_telemetry": [], + "metric_queries": 20, + "metric_query_failures": [], + "optional_empty_series": [ + "alerts", + "raw_5xx", + "raw_cancellations" + ] + } +} \ No newline at end of file diff --git a/tools/ingressbench/evidence/workload-profile-2026-10-28/catalog-metadata.json b/tools/ingressbench/evidence/workload-profile-2026-10-28/catalog-metadata.json new file mode 100644 index 000000000..e5f0f14fc --- /dev/null +++ b/tools/ingressbench/evidence/workload-profile-2026-10-28/catalog-metadata.json @@ -0,0 +1,11 @@ +{ + "read_only_staging_query": true, + "source_rows": 183507, + "distinct_keys": 11346, + "sha256": "410b91ddb2a4436ce549834236e282a8501cc8ab9d6748481d32e75dfc80f37b", + "mapping": "Evenly spaced valid cursors per API/limit/version class, then deterministic driver shuffle. Public catalog data only; no production request URLs.", + "limitations": [ + "Exact production cursor positions, first-page fraction and response-size distribution are not preserved.", + "This catalog enables a calibrated frequency model, not an exact crawl replay." + ] +} diff --git a/tools/ingressbench/evidence/workload-profile-2026-10-28/cycle-log-coverage.json b/tools/ingressbench/evidence/workload-profile-2026-10-28/cycle-log-coverage.json new file mode 100644 index 000000000..b5e1e3a35 --- /dev/null +++ b/tools/ingressbench/evidence/workload-profile-2026-10-28/cycle-log-coverage.json @@ -0,0 +1,155 @@ +[ + { + "phase": "cycle-1630", + "start": "2026-10-28T16:30:00Z", + "end": "2026-10-28T16:31:00Z", + "entries": 23728, + "pages": 5, + "complete": true, + "error": null + }, + { + "phase": "cycle-1645", + "start": "2026-10-28T16:45:00Z", + "end": "2026-10-28T16:46:00Z", + "entries": 18142, + "pages": 4, + "complete": true, + "error": null + }, + { + "phase": "cycle-1700", + "start": "2026-10-28T17:00:00Z", + "end": "2026-10-28T17:01:00Z", + "entries": 108267, + "pages": 20, + "complete": true, + "error": null + }, + { + "phase": "cycle-1715", + "start": "2026-10-28T17:15:00Z", + "end": "2026-10-28T17:16:00Z", + "entries": 122439, + "pages": 22, + "complete": true, + "error": null + }, + { + "phase": "cycle-1730", + "start": "2026-10-28T17:30:00Z", + "end": "2026-10-28T17:31:00Z", + "entries": 115640, + "pages": 21, + "complete": true, + "error": null + }, + { + "phase": "cycle-1745", + "start": "2026-10-28T17:45:00Z", + "end": "2026-10-28T17:46:00Z", + "entries": 127345, + "pages": 23, + "complete": true, + "error": null + }, + { + "phase": "cycle-1800", + "start": "2026-10-28T18:00:00Z", + "end": "2026-10-28T18:01:00Z", + "entries": 112640, + "pages": 21, + "complete": true, + "error": null + }, + { + "phase": "cycle-1815", + "start": "2026-10-28T18:15:00Z", + "end": "2026-10-28T18:16:00Z", + "entries": 100722, + "pages": 18, + "complete": true, + "error": null + }, + { + "phase": "cycle-1830", + "start": "2026-10-28T18:30:00Z", + "end": "2026-10-28T18:31:00Z", + "entries": 62714, + "pages": 12, + "complete": true, + "error": null + }, + { + "phase": "cycle-1845", + "start": "2026-10-28T18:45:00Z", + "end": "2026-10-28T18:46:00Z", + "entries": 49590, + "pages": 9, + "complete": true, + "error": null + }, + { + "phase": "cycle-1900", + "start": "2026-10-28T19:00:00Z", + "end": "2026-10-28T19:01:00Z", + "entries": 35776, + "pages": 7, + "complete": true, + "error": null + }, + { + "phase": "cycle-1915", + "start": "2026-10-28T19:15:00Z", + "end": "2026-10-28T19:16:00Z", + "entries": 18910, + "pages": 4, + "complete": true, + "error": null + }, + { + "phase": "cycle-1930", + "start": "2026-10-28T19:30:00Z", + "end": "2026-10-28T19:31:00Z", + "entries": 14201, + "pages": 3, + "complete": true, + "error": null + }, + { + "phase": "cycle-1945", + "start": "2026-10-28T19:45:00Z", + "end": "2026-10-28T19:46:00Z", + "entries": 11545, + "pages": 3, + "complete": true, + "error": null + }, + { + "phase": "cycle-2000", + "start": "2026-10-28T20:00:00Z", + "end": "2026-10-28T20:01:00Z", + "entries": 16792, + "pages": 4, + "complete": true, + "error": null + }, + { + "phase": "cycle-2015", + "start": "2026-10-28T20:15:00Z", + "end": "2026-10-28T20:16:00Z", + "entries": 13578, + "pages": 3, + "complete": true, + "error": null + }, + { + "phase": "cycle-2030", + "start": "2026-10-28T20:30:00Z", + "end": "2026-10-28T20:31:00Z", + "entries": 10346, + "pages": 2, + "complete": true, + "error": null + } +] diff --git a/tools/ingressbench/evidence/workload-profile-2026-10-28/local-profile-smoke.json b/tools/ingressbench/evidence/workload-profile-2026-10-28/local-profile-smoke.json new file mode 100644 index 000000000..1f23ec308 --- /dev/null +++ b/tools/ingressbench/evidence/workload-profile-2026-10-28/local-profile-smoke.json @@ -0,0 +1,144 @@ +{ + "Calibration": true, + "RequestPhases": { + "page:HIT": { + "Attempts": 300, + "NewConnections": 1, + "ReusedConnections": 299, + "SchedulerLag": { + "Count": 300, + "MeanMS": 1.7735868900000007, + "P95UpperMS": 5, + "MaximumMS": 2.072666 + }, + "QueueDelay": { + "Count": 300, + "MeanMS": 0.003675849999999997, + "P95UpperMS": 0.1, + "MaximumMS": 0.032458 + }, + "ConnectionWait": { + "Count": 300, + "MeanMS": 0.013347080000000002, + "P95UpperMS": 0.1, + "MaximumMS": 2.517458 + }, + "TLS": { + "Count": 1, + "MeanMS": 2.261042, + "P95UpperMS": 5, + "MaximumMS": 2.261042 + }, + "FirstByte": { + "Count": 300, + "MeanMS": 0.15403767999999993, + "P95UpperMS": 0.5, + "MaximumMS": 0.554292 + }, + "Body": { + "Count": 300, + "MeanMS": 0.10692536666666666, + "P95UpperMS": 0.5, + "MaximumMS": 0.365583 + }, + "HTTP": { + "Count": 300, + "MeanMS": 0.3092824766666667, + "P95UpperMS": 1, + "MaximumMS": 2.9275 + } + } + }, + "StartedAt": "2026-10-28T20:51:35.741444Z", + "FinishedAt": "2026-10-28T20:51:38.73392Z", + "CatalogSHA256": "410b91ddb2a4436ce549834236e282a8501cc8ab9d6748481d32e75dfc80f37b", + "PageWorkloadSHA256": "693e29271b933bd6cad81a4df114d111506b21d1afcb50e457f00ef217a42bbc", + "ConnectionRotation": "0s", + "ReadRate": 100, + "UpdatedRate": 0, + "SearchRate": 0, + "HotPercent": -1, + "Scheduled": 300, + "ScheduledPublishes": 0, + "Completed": 300, + "Dropped": 0, + "DroppedPublishes": 0, + "Bytes": 19660800, + "Status": { + "page:200": 300 + }, + "Cache": { + "HIT": 300 + }, + "Errors": {}, + "Protocols": { + "HTTP/1.1": 300 + }, + "ErrorDetails": {}, + "LatencyMS": { + "page_p50": 2.218083, + "page_p95": 2.610416, + "page_p99": 2.742541 + }, + "ElapsedSeconds": 2.992449416, + "Failures": null, + "OmittedFailures": 0, + "Windows": [ + { + "StartedAt": "2026-10-28T20:51:30Z", + "Seconds": 10, + "Reads": { + "Scheduled": 300, + "Dropped": 0, + "Dispatched": 300, + "Completed": 300, + "Failures": 0, + "SchedulerLag": { + "Count": 300, + "MeanMS": 1.7735868900000007, + "P95UpperMS": 5, + "MaximumMS": 2.072666 + }, + "QueueDelay": { + "Count": 300, + "MeanMS": 0.003675849999999997, + "P95UpperMS": 0.1, + "MaximumMS": 0.032458 + }, + "HTTPDuration": { + "Count": 300, + "MeanMS": 0.3092824766666667, + "P95UpperMS": 1, + "MaximumMS": 2.9275 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + } + ] +} diff --git a/tools/ingressbench/evidence/workload-profile-2026-10-28/measured-pages.json b/tools/ingressbench/evidence/workload-profile-2026-10-28/measured-pages.json new file mode 100644 index 000000000..bdc1e8c46 --- /dev/null +++ b/tools/ingressbench/evidence/workload-profile-2026-10-28/measured-pages.json @@ -0,0 +1 @@ +{"classes":[{"api_version":"v0","limit":1,"version_filter":"","key_counts":[2]},{"api_version":"v0","limit":1,"version_filter":"latest","key_counts":[1]},{"api_version":"v0","limit":2,"version_filter":"latest","key_counts":[1]},{"api_version":"v0","limit":7,"version_filter":"","key_counts":[1]},{"api_version":"v0","limit":25,"version_filter":"latest","key_counts":[1]},{"api_version":"v0","limit":30,"version_filter":"","key_counts":[2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},{"api_version":"v0","limit":30,"version_filter":"latest","key_counts":[5,5,5,5,5,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},{"api_version":"v0","limit":50,"version_filter":"","key_counts":[1]},{"api_version":"v0","limit":60,"version_filter":"latest","key_counts":[19]},{"api_version":"v0","limit":100,"version_filter":"","key_counts":[11,7,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},{"api_version":"v0","limit":100,"version_filter":"latest","key_counts":[6,5,5,5,5,5,5,5,4,4,4,4,4,4,4,4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},{"api_version":"v0.1","limit":1,"version_filter":"","key_counts":[3]},{"api_version":"v0.1","limit":24,"version_filter":"latest","key_counts":[1]},{"api_version":"v0.1","limit":30,"version_filter":"","key_counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},{"api_version":"v0.1","limit":30,"version_filter":"latest","key_counts":[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},{"api_version":"v0.1","limit":50,"version_filter":"latest","key_counts":[3]},{"api_version":"v0.1","limit":96,"version_filter":"latest","key_counts":[2]},{"api_version":"v0.1","limit":100,"version_filter":"","key_counts":[78,73,73,72,70,68,68,68,68,68,68,68,67,67,66,66,66,66,66,65,65,65,65,64,64,64,64,64,64,64,64,63,63,63,63,62,62,62,61,61,61,60,60,60,60,60,59,59,58,58,58,58,58,58,57,57,56,56,55,55,55,55,55,55,55,55,54,54,54,54,54,53,53,53,53,53,53,53,53,53,52,52,52,52,52,52,52,52,52,52,51,51,51,51,51,51,51,51,50,50,50,50,50,50,50,50,50,50,49,49,49,49,49,49,49,49,49,49,49,49,49,49,49,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,48,47,47,47,47,47,47,47,46,46,46,46,46,46,46,46,46,46,46,46,45,45,45,45,45,45,45,45,44,44,44,44,44,44,44,44,44,44,44,44,44,43,43,43,43,43,43,43,43,42,42,42,42,42,42,42,42,41,41,41,41,41,41,41,41,41,41,41,41,41,41,40,40,40,40,40,40,40,40,40,40,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,39,38,38,38,38,38,38,38,38,38,38,38,38,38,38,37,37,37,37,37,37,37,37,37,37,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,36,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,35,34,34,34,34,34,34,34,34,34,34,34,34,34,34,34,33,33,33,33,33,33,33,33,33,33,33,33,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,32,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,31,30,30,30,30,30,30,30,30,30,30,30,30,30,30,30,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,29,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,28,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,27,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,26,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,25,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,24,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,23,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,22,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,21,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,20,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,19,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,18,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,17,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,14,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,13,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,12,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,11,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,10,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]},{"api_version":"v0.1","limit":100,"version_filter":"latest","key_counts":[6,6,6,5,5,5,5,5,5,5,5,5,5,5,5,5,5,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1]}],"excluded_page_requests":16,"limitations":["Frequency model only; chronological crawl/client ordering is not replayed.","Query ordering and default limit normalized; original cache-key diversity is reported separately.","Searches, updates, details and publishing need their own measured workloads."]} diff --git a/tools/ingressbench/evidence/workload-profile-2026-10-28/origin-cycle.json b/tools/ingressbench/evidence/workload-profile-2026-10-28/origin-cycle.json new file mode 100644 index 000000000..66dcc9deb --- /dev/null +++ b/tools/ingressbench/evidence/workload-profile-2026-10-28/origin-cycle.json @@ -0,0 +1,5550 @@ +{ + "scope": "Historical origin metrics: 2-minute rate windows and 5-minute latency quantiles. Ingress/cache metrics unavailable. No interpolation of missing points.", + "from": "2026-10-28T16:30:00Z", + "to": "2026-10-28T20:30:00Z", + "points": [ + { + "at": "2026-10-28T16:30:00+00:00", + "origin_arrivals_rps": 195.8083333333333, + "origin_499_rps": 0.05, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.11666666666666667, + "list_queries_rps": 0.4, + "list_page_queries_rps": 183.1, + "list_search_queries_rps": 4.933333333333334, + "list_updated_queries_rps": 0.375, + "other_queries_rps": 7.341666666666667, + "/v0.1/servers_p95_seconds": 0.09228496449817694, + "/v0/servers_p95_seconds": 0.08980869894099845, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009584070796460176, + "list_page_acquire_p95_seconds": 0.0009558874763412224, + "list_search_acquire_p95_seconds": 0.0009674059787849566, + "list_updated_acquire_p95_seconds": 0.0009683870967741936, + "other_acquire_p95_seconds": 0.000968391411474833, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.4969408878246516, + "node_cpu_apu6": 0.5209534294619516 + }, + { + "at": "2026-10-28T16:31:00+00:00", + "origin_arrivals_rps": 192.23333333333332, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.08333333333333333, + "list_queries_rps": 0.45, + "list_page_queries_rps": 182.30833333333334, + "list_search_queries_rps": 3.7583333333333333, + "list_updated_queries_rps": 0.5416666666666667, + "other_queries_rps": 5.141666666666667, + "/v0.1/servers_p95_seconds": 0.09244796233003938, + "/v0/servers_p95_seconds": 0.08995504135139892, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009581196581196582, + "list_page_acquire_p95_seconds": 0.00095622076707203, + "list_search_acquire_p95_seconds": 0.0009683027522935779, + "list_updated_acquire_p95_seconds": 0.0009746753246753244, + "other_acquire_p95_seconds": 0.0009651426174496645, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.5446964755545196, + "node_cpu_apu6": 0.5156358042392336 + }, + { + "at": "2026-10-28T16:32:00+00:00", + "origin_arrivals_rps": 206.35, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.049999999999999996, + "list_queries_rps": 0.47500000000000003, + "list_page_queries_rps": 198.3, + "list_search_queries_rps": 2.3166666666666664, + "list_updated_queries_rps": 0.525, + "other_queries_rps": 4.6, + "/v0.1/servers_p95_seconds": 0.09220550091074683, + "/v0/servers_p95_seconds": 0.09048491165376214, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009500000000000001, + "list_page_acquire_p95_seconds": 0.000956191775583982, + "list_search_acquire_p95_seconds": 0.0009711324570273002, + "list_updated_acquire_p95_seconds": 0.0009796875, + "other_acquire_p95_seconds": 0.0009642822502424831, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.4971891721460755, + "node_cpu_apu6": 0.555959661858166 + }, + { + "at": "2026-10-28T16:33:00+00:00", + "origin_arrivals_rps": 215.2166666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.075, + "list_queries_rps": 0.42500000000000004, + "list_page_queries_rps": 206.90833333333333, + "list_search_queries_rps": 2.466666666666667, + "list_updated_queries_rps": 0.6083333333333334, + "other_queries_rps": 4.941666666666666, + "/v0.1/servers_p95_seconds": 0.0921932040531097, + "/v0/servers_p95_seconds": 0.09169855321732093, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009659663865546218, + "list_page_acquire_p95_seconds": 0.0009565366595785142, + "list_search_acquire_p95_seconds": 0.0009723310479921646, + "list_updated_acquire_p95_seconds": 0.0009679245283018867, + "other_acquire_p95_seconds": 0.000962689738919247, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.5077315760812781, + "node_cpu_apu6": 0.5668460881330247 + }, + { + "at": "2026-10-28T16:34:00+00:00", + "origin_arrivals_rps": 222.78333333333342, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.06666666666666667, + "list_queries_rps": 0.45833333333333337, + "list_page_queries_rps": 211.64999999999998, + "list_search_queries_rps": 2.375, + "list_updated_queries_rps": 0.6333333333333333, + "other_queries_rps": 7.7749999999999995, + "/v0.1/servers_p95_seconds": 0.09203316581242096, + "/v0/servers_p95_seconds": 0.09315203850509625, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009657024793388431, + "list_page_acquire_p95_seconds": 0.0009565125624746724, + "list_search_acquire_p95_seconds": 0.0009773709483793518, + "list_updated_acquire_p95_seconds": 0.0009681528662420382, + "other_acquire_p95_seconds": 0.0009666465256797583, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.4929566406647306, + "node_cpu_apu6": 0.5820851129128035 + }, + { + "at": "2026-10-28T16:35:00+00:00", + "origin_arrivals_rps": 243.23333333333332, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.475, + "list_page_queries_rps": 229.80833333333334, + "list_search_queries_rps": 2.6666666666666665, + "list_updated_queries_rps": 0.43333333333333335, + "other_queries_rps": 9.649999999999999, + "/v0.1/servers_p95_seconds": 0.09165465101741004, + "/v0/servers_p95_seconds": 0.09377511907462009, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.000964179104477612, + "list_page_acquire_p95_seconds": 0.0009559288911303032, + "list_search_acquire_p95_seconds": 0.0009777925531914893, + "list_updated_acquire_p95_seconds": 0.0009613772455089821, + "other_acquire_p95_seconds": 0.0009689030612244898, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.5132520583758984, + "node_cpu_apu6": 0.6038763303794743 + }, + { + "at": "2026-10-28T16:36:00+00:00", + "origin_arrivals_rps": 242.35833333333332, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.0, + "list_queries_rps": 0.44166666666666665, + "list_page_queries_rps": 229.66666666666666, + "list_search_queries_rps": 4.058333333333334, + "list_updated_queries_rps": 0.4666666666666667, + "other_queries_rps": 7.4, + "/v0.1/servers_p95_seconds": 0.09148353710015321, + "/v0/servers_p95_seconds": 0.09400726141078838, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009654471544715445, + "list_page_acquire_p95_seconds": 0.0009556626004990295, + "list_search_acquire_p95_seconds": 0.000981886574074074, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009704301075268818, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.4657391512317796, + "node_cpu_apu6": 0.5438290706149321 + }, + { + "at": "2026-10-28T16:37:00+00:00", + "origin_arrivals_rps": 218.7083333333334, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.008333333333333333, + "list_queries_rps": 0.4166666666666667, + "list_page_queries_rps": 209.94166666666666, + "list_search_queries_rps": 3.3833333333333333, + "list_updated_queries_rps": 0.5333333333333333, + "other_queries_rps": 4.216666666666667, + "/v0.1/servers_p95_seconds": 0.09162749930467569, + "/v0/servers_p95_seconds": 0.09409845415778254, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009724409448818898, + "list_page_acquire_p95_seconds": 0.0009556241616946487, + "list_search_acquire_p95_seconds": 0.0009827210103329504, + "list_updated_acquire_p95_seconds": 0.0009500000000000001, + "other_acquire_p95_seconds": 0.0009686563307493538, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.5050479979356168, + "node_cpu_apu6": 0.5749384107451382 + }, + { + "at": "2026-10-28T16:38:00+00:00", + "origin_arrivals_rps": 224.06666666666672, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.45833333333333337, + "list_page_queries_rps": 212.125, + "list_search_queries_rps": 2.341666666666667, + "list_updated_queries_rps": 0.5333333333333333, + "other_queries_rps": 8.508333333333333, + "/v0.1/servers_p95_seconds": 0.09166346652828233, + "/v0/servers_p95_seconds": 0.09416211470932502, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009648437500000001, + "list_page_acquire_p95_seconds": 0.0009551568241769971, + "list_search_acquire_p95_seconds": 0.0009847697756788665, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009714644351464435, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.48374744828640054, + "node_cpu_apu6": 0.577456311241743 + }, + { + "at": "2026-10-28T16:39:00+00:00", + "origin_arrivals_rps": 217.86666666666665, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.43333333333333335, + "list_page_queries_rps": 201.40833333333333, + "list_search_queries_rps": 2.158333333333333, + "list_updated_queries_rps": 0.35833333333333334, + "other_queries_rps": 13.358333333333334, + "/v0.1/servers_p95_seconds": 0.09185630153121319, + "/v0/servers_p95_seconds": 0.09359409722222224, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00096496062992126, + "list_page_acquire_p95_seconds": 0.0009544430842632264, + "list_search_acquire_p95_seconds": 0.0009814049586776858, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009714586523736599, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.4651865662196845, + "node_cpu_apu6": 0.5593828819301658 + }, + { + "at": "2026-10-28T16:40:00+00:00", + "origin_arrivals_rps": 203.27500000000003, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.4166666666666667, + "list_page_queries_rps": 185.86666666666667, + "list_search_queries_rps": 1.7416666666666667, + "list_updated_queries_rps": 0.35, + "other_queries_rps": 14.95, + "/v0.1/servers_p95_seconds": 0.09212413821405123, + "/v0/servers_p95_seconds": 0.09318691369606003, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009658333333333332, + "list_page_acquire_p95_seconds": 0.0009541206562587951, + "list_search_acquire_p95_seconds": 0.0009860433604336044, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009803769774486704, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.44919856001458053, + "node_cpu_apu6": 0.559392519370643 + }, + { + "at": "2026-10-28T16:41:00+00:00", + "origin_arrivals_rps": 218.60833333333338, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.08333333333333333, + "list_queries_rps": 0.475, + "list_page_queries_rps": 199.41666666666666, + "list_search_queries_rps": 1.9666666666666668, + "list_updated_queries_rps": 0.375, + "other_queries_rps": 16.458333333333336, + "/v0.1/servers_p95_seconds": 0.09207036183649084, + "/v0/servers_p95_seconds": 0.0929195291255188, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009647286821705426, + "list_page_acquire_p95_seconds": 0.0009539809764533725, + "list_search_acquire_p95_seconds": 0.0009913043478260869, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.000989761235955056, + "sampled_inflight": 3.0, + "node_cpu_gc71": 0.4264015825394987, + "node_cpu_apu6": 0.5701817699919047 + }, + { + "at": "2026-10-28T16:42:00+00:00", + "origin_arrivals_rps": 233.9166666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.06666666666666667, + "list_queries_rps": 0.4666666666666667, + "list_page_queries_rps": 217.825, + "list_search_queries_rps": 1.8666666666666667, + "list_updated_queries_rps": 0.35, + "other_queries_rps": 13.4, + "/v0.1/servers_p95_seconds": 0.09168889244217036, + "/v0/servers_p95_seconds": 0.0926593770709079, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009575396825396825, + "list_page_acquire_p95_seconds": 0.0009535908631757308, + "list_search_acquire_p95_seconds": 0.0009906417112299465, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009982968988307068, + "sampled_inflight": 3.0, + "node_cpu_gc71": 0.39762556248921704, + "node_cpu_apu6": 0.5189127323125527 + }, + { + "at": "2026-10-28T16:43:00+00:00", + "origin_arrivals_rps": 220.30833333333337, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.43333333333333335, + "list_page_queries_rps": 206.7, + "list_search_queries_rps": 3.4, + "list_updated_queries_rps": 0.3333333333333333, + "other_queries_rps": 9.5, + "/v0.1/servers_p95_seconds": 0.09119218181818181, + "/v0/servers_p95_seconds": 0.09216007114273013, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009573076923076922, + "list_page_acquire_p95_seconds": 0.0009532234223035964, + "list_search_acquire_p95_seconds": 0.0009773287671232874, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0013833333333333271, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.411301626831361, + "node_cpu_apu6": 0.5626186048347639 + }, + { + "at": "2026-10-28T16:44:00+00:00", + "origin_arrivals_rps": 218.45000000000005, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.4166666666666667, + "list_page_queries_rps": 201.80833333333334, + "list_search_queries_rps": 8.841666666666667, + "list_updated_queries_rps": 0.375, + "other_queries_rps": 7.0, + "/v0.1/servers_p95_seconds": 0.09103676938630324, + "/v0/servers_p95_seconds": 0.09253774796631943, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009574803149606298, + "list_page_acquire_p95_seconds": 0.0009535878819450097, + "list_search_acquire_p95_seconds": 0.0009688602941176471, + "list_updated_acquire_p95_seconds": 0.0009500000000000001, + "other_acquire_p95_seconds": 0.0017390804597701216, + "sampled_inflight": 3.0, + "node_cpu_gc71": 0.45192370098351, + "node_cpu_apu6": 0.5535946201035888 + }, + { + "at": "2026-10-28T16:45:00+00:00", + "origin_arrivals_rps": 222.05, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.3416666666666667, + "list_page_queries_rps": 209.09166666666664, + "list_search_queries_rps": 7.324999999999999, + "list_updated_queries_rps": 0.3333333333333333, + "other_queries_rps": 4.933333333333334, + "/v0.1/servers_p95_seconds": 0.09072738416049109, + "/v0/servers_p95_seconds": 0.092590793135436, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.000965702479338843, + "list_page_acquire_p95_seconds": 0.0009535842150047362, + "list_search_acquire_p95_seconds": 0.000967012893982808, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0018633093525179834, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.4418548598492634, + "node_cpu_apu6": 0.527409839474664 + }, + { + "at": "2026-10-28T16:46:00+00:00", + "origin_arrivals_rps": 197.025, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.3333333333333333, + "list_page_queries_rps": 188.53333333333333, + "list_search_queries_rps": 2.8833333333333333, + "list_updated_queries_rps": 0.35, + "other_queries_rps": 4.816666666666666, + "/v0.1/servers_p95_seconds": 0.0909136900271575, + "/v0/servers_p95_seconds": 0.09220857023411375, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009671171171171171, + "list_page_acquire_p95_seconds": 0.0009536502603170379, + "list_search_acquire_p95_seconds": 0.0009602633355840648, + "list_updated_acquire_p95_seconds": 0.0009587962962962962, + "other_acquire_p95_seconds": 0.0012484210526315792, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.4229240471337107, + "node_cpu_apu6": 0.5036670890554281 + }, + { + "at": "2026-10-28T16:47:00+00:00", + "origin_arrivals_rps": 177.7, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.4, + "list_page_queries_rps": 168.61666666666667, + "list_search_queries_rps": 2.8, + "list_updated_queries_rps": 0.39166666666666666, + "other_queries_rps": 5.358333333333333, + "/v0.1/servers_p95_seconds": 0.09145312795670438, + "/v0/servers_p95_seconds": 0.09246354627499556, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009668141592920353, + "list_page_acquire_p95_seconds": 0.0009541964554985896, + "list_search_acquire_p95_seconds": 0.0009631857237276935, + "list_updated_acquire_p95_seconds": 0.0009591346153846152, + "other_acquire_p95_seconds": 0.0009817050970873785, + "sampled_inflight": 5.0, + "node_cpu_gc71": 0.460045361774246, + "node_cpu_apu6": 0.4976123038776374 + }, + { + "at": "2026-10-28T16:48:00+00:00", + "origin_arrivals_rps": 185.525, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.42500000000000004, + "list_page_queries_rps": 177.44166666666666, + "list_search_queries_rps": 1.925, + "list_updated_queries_rps": 0.35, + "other_queries_rps": 5.266666666666667, + "/v0.1/servers_p95_seconds": 0.09180129865392336, + "/v0/servers_p95_seconds": 0.09270720989761091, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009675925925925927, + "list_page_acquire_p95_seconds": 0.0009547028040026872, + "list_search_acquire_p95_seconds": 0.0009683397683397684, + "list_updated_acquire_p95_seconds": 0.0009586363636363637, + "other_acquire_p95_seconds": 0.0009666779203241053, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.5062053281161449, + "node_cpu_apu6": 0.5536580155698175 + }, + { + "at": "2026-10-28T16:49:00+00:00", + "origin_arrivals_rps": 218.39999999999998, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.39166666666666666, + "list_page_queries_rps": 208.85833333333335, + "list_search_queries_rps": 1.8833333333333333, + "list_updated_queries_rps": 0.4583333333333333, + "other_queries_rps": 6.691666666666666, + "/v0.1/servers_p95_seconds": 0.09177966101694915, + "/v0/servers_p95_seconds": 0.09253930936613056, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009675925925925924, + "list_page_acquire_p95_seconds": 0.0009546517321581587, + "list_search_acquire_p95_seconds": 0.0009823703703703704, + "list_updated_acquire_p95_seconds": 0.0009668141592920355, + "other_acquire_p95_seconds": 0.0009693518518518517, + "sampled_inflight": 3.0, + "node_cpu_gc71": 0.5455507159827296, + "node_cpu_apu6": 0.5645053713621261 + }, + { + "at": "2026-10-28T16:50:00+00:00", + "origin_arrivals_rps": 228.52500000000003, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.4083333333333333, + "list_page_queries_rps": 215.85833333333335, + "list_search_queries_rps": 1.5666666666666667, + "list_updated_queries_rps": 0.45833333333333337, + "other_queries_rps": 10.158333333333333, + "/v0.1/servers_p95_seconds": 0.09204800352319435, + "/v0/servers_p95_seconds": 0.09251966397597146, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009582608695652173, + "list_page_acquire_p95_seconds": 0.0009549856159009676, + "list_search_acquire_p95_seconds": 0.0009876859504132232, + "list_updated_acquire_p95_seconds": 0.0009653225806451613, + "other_acquire_p95_seconds": 0.000982297231665857, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.5527978647102524, + "node_cpu_apu6": 0.5639548768456735 + }, + { + "at": "2026-10-28T16:51:00+00:00", + "origin_arrivals_rps": 225.57500000000005, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.4, + "list_page_queries_rps": 211.55, + "list_search_queries_rps": 2.0666666666666664, + "list_updated_queries_rps": 0.33333333333333337, + "other_queries_rps": 10.891666666666666, + "/v0.1/servers_p95_seconds": 0.09219788152089277, + "/v0/servers_p95_seconds": 0.09304071689577169, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009582608695652173, + "list_page_acquire_p95_seconds": 0.0009550267906324593, + "list_search_acquire_p95_seconds": 0.002230769230769221, + "list_updated_acquire_p95_seconds": 0.000958482142857143, + "other_acquire_p95_seconds": 0.0009836173001310617, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.5657446715903158, + "node_cpu_apu6": 0.5562762127755074 + }, + { + "at": "2026-10-28T16:52:00+00:00", + "origin_arrivals_rps": 226.0833333333333, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.4083333333333333, + "list_page_queries_rps": 215.75833333333333, + "list_search_queries_rps": 2.625, + "list_updated_queries_rps": 0.36666666666666664, + "other_queries_rps": 6.691666666666666, + "/v0.1/servers_p95_seconds": 0.09181100787301183, + "/v0/servers_p95_seconds": 0.09287839892665475, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009581896551724139, + "list_page_acquire_p95_seconds": 0.0009546292181200597, + "list_search_acquire_p95_seconds": 0.0009993934142114385, + "list_updated_acquire_p95_seconds": 0.0009577868852459018, + "other_acquire_p95_seconds": 0.0009845926263086027, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.46657149230644185, + "node_cpu_apu6": 0.5424180456987718 + }, + { + "at": "2026-10-28T16:53:00+00:00", + "origin_arrivals_rps": 213.84166666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.43333333333333335, + "list_page_queries_rps": 207.50833333333333, + "list_search_queries_rps": 2.0, + "list_updated_queries_rps": 0.31666666666666665, + "other_queries_rps": 3.7249999999999996, + "/v0.1/servers_p95_seconds": 0.09156086921333766, + "/v0/servers_p95_seconds": 0.09278087017791088, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009542605589475673, + "list_search_acquire_p95_seconds": 0.0009923670668953687, + "list_updated_acquire_p95_seconds": 0.0009587962962962963, + "other_acquire_p95_seconds": 0.0009857040913415793, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.43043712432214914, + "node_cpu_apu6": 0.529446722335706 + }, + { + "at": "2026-10-28T16:54:00+00:00", + "origin_arrivals_rps": 203.47500000000002, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.325, + "list_page_queries_rps": 192.23333333333335, + "list_search_queries_rps": 2.783333333333333, + "list_updated_queries_rps": 0.16666666666666666, + "other_queries_rps": 8.041666666666666, + "/v0.1/servers_p95_seconds": 0.09145557418273263, + "/v0/servers_p95_seconds": 0.09261564139246542, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009538892982887087, + "list_search_acquire_p95_seconds": 0.0009829479768786128, + "list_updated_acquire_p95_seconds": 0.0009500000000000001, + "other_acquire_p95_seconds": 0.0009821897501058872, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.44899020935699807, + "node_cpu_apu6": 0.5299762542138827 + }, + { + "at": "2026-10-28T16:55:00+00:00", + "origin_arrivals_rps": 206.92500000000004, + "origin_499_rps": 0.025, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.3666666666666667, + "list_page_queries_rps": 188.325, + "list_search_queries_rps": 2.908333333333333, + "list_updated_queries_rps": 0.23333333333333334, + "other_queries_rps": 14.975000000000001, + "/v0.1/servers_p95_seconds": 0.09127715141209525, + "/v0/servers_p95_seconds": 0.09260924696241879, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.000953418116098224, + "list_search_acquire_p95_seconds": 0.000979211229946524, + "list_updated_acquire_p95_seconds": 0.0009500000000000001, + "other_acquire_p95_seconds": 0.0009753427195257504, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.4275540658161238, + "node_cpu_apu6": 0.5113038510057474 + }, + { + "at": "2026-10-28T16:56:00+00:00", + "origin_arrivals_rps": 209.06666666666666, + "origin_499_rps": 0.03333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.5333333333333333, + "list_page_queries_rps": 186.18333333333334, + "list_search_queries_rps": 1.9666666666666668, + "list_updated_queries_rps": 0.45833333333333337, + "other_queries_rps": 19.65, + "/v0.1/servers_p95_seconds": 0.09100464311009678, + "/v0/servers_p95_seconds": 0.09247159005275059, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009575396825396825, + "list_page_acquire_p95_seconds": 0.0009529927698399036, + "list_search_acquire_p95_seconds": 0.0009692753623188407, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.000987785693084001, + "sampled_inflight": 4.0, + "node_cpu_gc71": 0.5176618823415386, + "node_cpu_apu6": 0.5181752262804367 + }, + { + "at": "2026-10-28T16:57:00+00:00", + "origin_arrivals_rps": 206.16666666666666, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.5333333333333333, + "list_page_queries_rps": 181.65833333333333, + "list_search_queries_rps": 1.575, + "list_updated_queries_rps": 1.6083333333333334, + "other_queries_rps": 20.741666666666667, + "/v0.1/servers_p95_seconds": 0.09140793536032515, + "/v0/servers_p95_seconds": 0.09260391701617735, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009573076923076922, + "list_page_acquire_p95_seconds": 0.000952836375268798, + "list_search_acquire_p95_seconds": 0.0009695721077654514, + "list_updated_acquire_p95_seconds": 0.0009541304347826087, + "other_acquire_p95_seconds": 0.0009941964285714285, + "sampled_inflight": 3.0, + "node_cpu_gc71": 0.5513029493391514, + "node_cpu_apu6": 0.5105854675416291 + }, + { + "at": "2026-10-28T16:58:00+00:00", + "origin_arrivals_rps": 194.7833333333334, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.43333333333333335, + "list_page_queries_rps": 174.49166666666667, + "list_search_queries_rps": 1.375, + "list_updated_queries_rps": 2.4833333333333334, + "other_queries_rps": 16.066666666666666, + "/v0.1/servers_p95_seconds": 0.0915393532397969, + "/v0/servers_p95_seconds": 0.09274068865916696, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009574218749999998, + "list_page_acquire_p95_seconds": 0.0009526662727574875, + "list_search_acquire_p95_seconds": 0.0009684764991896273, + "list_updated_acquire_p95_seconds": 0.0009526243093922652, + "other_acquire_p95_seconds": 0.001028820960698688, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.4814938768654732, + "node_cpu_apu6": 0.45263569960516176 + }, + { + "at": "2026-10-28T16:59:00+00:00", + "origin_arrivals_rps": 171.02499999999998, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.008333333333333333, + "list_queries_rps": 0.4, + "list_page_queries_rps": 157.73333333333335, + "list_search_queries_rps": 2.2333333333333334, + "list_updated_queries_rps": 1.4, + "other_queries_rps": 9.083333333333334, + "/v0.1/servers_p95_seconds": 0.09143680174794905, + "/v0/servers_p95_seconds": 0.09274033292873247, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009567857142857142, + "list_page_acquire_p95_seconds": 0.0009526595846675998, + "list_search_acquire_p95_seconds": 0.0009684303350970017, + "list_updated_acquire_p95_seconds": 0.0009525132275132273, + "other_acquire_p95_seconds": 0.0012369369369369379, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.40477507447819033, + "node_cpu_apu6": 0.4715039769881211 + }, + { + "at": "2026-10-28T17:00:00+00:00", + "origin_arrivals_rps": 170.7, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.0, + "list_queries_rps": 0.44166666666666665, + "list_page_queries_rps": 159.05833333333334, + "list_search_queries_rps": 5.333333333333333, + "list_updated_queries_rps": 0.5166666666666666, + "other_queries_rps": 5.141666666666667, + "/v0.1/servers_p95_seconds": 0.09145589817880792, + "/v0/servers_p95_seconds": 0.09296650532429815, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.000956934306569343, + "list_page_acquire_p95_seconds": 0.0009531522812561721, + "list_search_acquire_p95_seconds": 0.0009646961325966852, + "list_updated_acquire_p95_seconds": 0.00095239898989899, + "other_acquire_p95_seconds": 0.0013486631016042827, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.47766782998951385, + "node_cpu_apu6": 0.590499747399493 + }, + { + "at": "2026-10-28T17:01:00+00:00", + "origin_arrivals_rps": 137.61666666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.0, + "list_queries_rps": 0.4916666666666667, + "list_page_queries_rps": 120.81666666666666, + "list_search_queries_rps": 7.133333333333333, + "list_updated_queries_rps": 4.608333333333333, + "other_queries_rps": 4.35, + "/v0.1/servers_p95_seconds": 0.09269581921166259, + "/v0/servers_p95_seconds": 0.09323273043122829, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009714285714285714, + "list_page_acquire_p95_seconds": 0.0009547899520665642, + "list_search_acquire_p95_seconds": 0.00096042194092827, + "list_updated_acquire_p95_seconds": 0.0047384615384614675, + "other_acquire_p95_seconds": 0.0011433070866141643, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.9277812365282443, + "node_cpu_apu6": 0.9099739539034393 + }, + { + "at": "2026-10-28T17:02:00+00:00", + "origin_arrivals_rps": 91.33333333333333, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.0, + "list_queries_rps": 0.625, + "list_page_queries_rps": 62.25833333333333, + "list_search_queries_rps": 4.633333333333333, + "list_updated_queries_rps": 18.28333333333333, + "other_queries_rps": 5.25, + "/v0.1/servers_p95_seconds": 0.09756922095505913, + "/v0/servers_p95_seconds": 0.09547116052814457, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00098986013986014, + "list_page_acquire_p95_seconds": 0.0009591518104307385, + "list_search_acquire_p95_seconds": 0.0009634751773049644, + "list_updated_acquire_p95_seconds": 0.007043103448275851, + "other_acquire_p95_seconds": 0.0009993899204244032, + "sampled_inflight": 8.0, + "node_cpu_gc71": 0.9929007868333428, + "node_cpu_apu6": 0.9984953713261348 + }, + { + "at": "2026-10-28T17:03:00+00:00", + "origin_arrivals_rps": 114.70833333333336, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.5583333333333333, + "list_page_queries_rps": 74.55833333333334, + "list_search_queries_rps": 2.2916666666666665, + "list_updated_queries_rps": 30.083333333333332, + "other_queries_rps": 6.933333333333334, + "/v0.1/servers_p95_seconds": 0.17978608247422648, + "/v0/servers_p95_seconds": 0.09890315528243916, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.000989310344827586, + "list_page_acquire_p95_seconds": 0.0009649203766855426, + "list_search_acquire_p95_seconds": 0.0009655011655011654, + "list_updated_acquire_p95_seconds": 0.004464367816091936, + "other_acquire_p95_seconds": 0.0009844076120319215, + "sampled_inflight": 12.0, + "node_cpu_gc71": 0.9622638269684103, + "node_cpu_apu6": 0.9733181196648123 + }, + { + "at": "2026-10-28T17:04:00+00:00", + "origin_arrivals_rps": 155.29166666666663, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.42500000000000004, + "list_page_queries_rps": 125.32499999999999, + "list_search_queries_rps": 1.6749999999999998, + "list_updated_queries_rps": 19.658333333333335, + "other_queries_rps": 7.883333333333334, + "/v0.1/servers_p95_seconds": 0.1921570191431178, + "/v0/servers_p95_seconds": 0.09963736767100977, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009961805555555557, + "list_page_acquire_p95_seconds": 0.0009694600181664422, + "list_search_acquire_p95_seconds": 0.0009658333333333332, + "list_updated_acquire_p95_seconds": 0.004057142857142851, + "other_acquire_p95_seconds": 0.0009845062429057888, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.8646238272245682, + "node_cpu_apu6": 0.9299748432613948 + }, + { + "at": "2026-10-28T17:05:00+00:00", + "origin_arrivals_rps": 169.86666666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.5166666666666667, + "list_page_queries_rps": 157.08333333333334, + "list_search_queries_rps": 1.1583333333333332, + "list_updated_queries_rps": 3.9, + "other_queries_rps": 7.050000000000001, + "/v0.1/servers_p95_seconds": 0.19787246049661383, + "/v0/servers_p95_seconds": 0.09925843558282209, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009940397350993375, + "list_page_acquire_p95_seconds": 0.0009718959332042286, + "list_search_acquire_p95_seconds": 0.0009716730038022815, + "list_updated_acquire_p95_seconds": 0.004043956043956028, + "other_acquire_p95_seconds": 0.0009854555433589463, + "sampled_inflight": 3.0, + "node_cpu_gc71": 0.8256275924194294, + "node_cpu_apu6": 0.888521870085242 + }, + { + "at": "2026-10-28T17:06:00+00:00", + "origin_arrivals_rps": 170.16666666666669, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.5583333333333333, + "list_page_queries_rps": 161.24166666666667, + "list_search_queries_rps": 1.1833333333333333, + "list_updated_queries_rps": 0.725, + "other_queries_rps": 6.25, + "/v0.1/servers_p95_seconds": 0.17906515580736543, + "/v0/servers_p95_seconds": 0.09918502998092124, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009748366013071896, + "list_page_acquire_p95_seconds": 0.000970348995852434, + "list_search_acquire_p95_seconds": 0.000999376299376299, + "list_updated_acquire_p95_seconds": 0.0029670886075949103, + "other_acquire_p95_seconds": 0.0009859774820880245, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.8288004486923455, + "node_cpu_apu6": 0.8995462962212817 + }, + { + "at": "2026-10-28T17:07:00+00:00", + "origin_arrivals_rps": 182.83333333333334, + "origin_499_rps": 0.016666666666666666, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.4833333333333334, + "list_page_queries_rps": 170.675, + "list_search_queries_rps": 2.1083333333333334, + "list_updated_queries_rps": 0.7416666666666667, + "other_queries_rps": 8.416666666666668, + "/v0.1/servers_p95_seconds": 0.10478542914171632, + "/v0/servers_p95_seconds": 0.09768264088852324, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009568345323741006, + "list_page_acquire_p95_seconds": 0.0009661259499078197, + "list_search_acquire_p95_seconds": 0.000991304347826087, + "list_updated_acquire_p95_seconds": 0.000995676929565584, + "other_acquire_p95_seconds": 0.001211320754716979, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.8416922205629817, + "node_cpu_apu6": 0.8931889013714681 + }, + { + "at": "2026-10-28T17:08:00+00:00", + "origin_arrivals_rps": 192.63333333333335, + "origin_499_rps": 0.025, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.44166666666666665, + "list_page_queries_rps": 177.35833333333335, + "list_search_queries_rps": 2.833333333333333, + "list_updated_queries_rps": 0.5666666666666667, + "other_queries_rps": 11.233333333333334, + "/v0.1/servers_p95_seconds": 0.09628997490554334, + "/v0/servers_p95_seconds": 0.0967696588868941, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009566901408450704, + "list_page_acquire_p95_seconds": 0.0009635930047694751, + "list_search_acquire_p95_seconds": 0.0009916058394160584, + "list_updated_acquire_p95_seconds": 0.0009770944741532975, + "other_acquire_p95_seconds": 0.0027138461538461396, + "sampled_inflight": 3.0, + "node_cpu_gc71": 0.8249386477691587, + "node_cpu_apu6": 0.908193196542076 + }, + { + "at": "2026-10-28T17:09:00+00:00", + "origin_arrivals_rps": 208.48333333333335, + "origin_499_rps": 0.058333333333333334, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.475, + "list_page_queries_rps": 190.45833333333334, + "list_search_queries_rps": 2.5250000000000004, + "list_updated_queries_rps": 0.4833333333333334, + "other_queries_rps": 14.6, + "/v0.1/servers_p95_seconds": 0.09567497999834468, + "/v0/servers_p95_seconds": 0.0966384230711385, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009499999999999999, + "list_page_acquire_p95_seconds": 0.0009626105391829171, + "list_search_acquire_p95_seconds": 0.000990807560137457, + "list_updated_acquire_p95_seconds": 0.0009601604278074866, + "other_acquire_p95_seconds": 0.0024399999999999878, + "sampled_inflight": 5.0, + "node_cpu_gc71": 0.8672749691051043, + "node_cpu_apu6": 0.9104707340666209 + }, + { + "at": "2026-10-28T17:10:00+00:00", + "origin_arrivals_rps": 211.91666666666669, + "origin_499_rps": 0.06666666666666667, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.5333333333333333, + "list_page_queries_rps": 191.2, + "list_search_queries_rps": 2.1166666666666667, + "list_updated_queries_rps": 0.6, + "other_queries_rps": 17.508333333333333, + "/v0.1/servers_p95_seconds": 0.09595275627841525, + "/v0/servers_p95_seconds": 0.09694462534620857, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009499999999999999, + "list_page_acquire_p95_seconds": 0.0009633787701924163, + "list_search_acquire_p95_seconds": 0.0009888636363636362, + "list_updated_acquire_p95_seconds": 0.0009603825136612021, + "other_acquire_p95_seconds": 0.0023056179775280804, + "sampled_inflight": 5.0, + "node_cpu_gc71": 0.9044334971239247, + "node_cpu_apu6": 0.9006205676882625 + }, + { + "at": "2026-10-28T17:11:00+00:00", + "origin_arrivals_rps": 201.07500000000002, + "origin_499_rps": 0.03333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.058333333333333334, + "list_queries_rps": 0.5416666666666667, + "list_page_queries_rps": 178.95, + "list_search_queries_rps": 2.5, + "list_updated_queries_rps": 0.5833333333333334, + "other_queries_rps": 18.558333333333334, + "/v0.1/servers_p95_seconds": 0.09583566680903315, + "/v0/servers_p95_seconds": 0.09686572549569118, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009566901408450703, + "list_page_acquire_p95_seconds": 0.0009631086245544411, + "list_search_acquire_p95_seconds": 0.0009887755102040814, + "list_updated_acquire_p95_seconds": 0.0009555555555555552, + "other_acquire_p95_seconds": 0.0018623655913978477, + "sampled_inflight": 4.0, + "node_cpu_gc71": 0.8850831741008732, + "node_cpu_apu6": 0.9033411528408275 + }, + { + "at": "2026-10-28T17:12:00+00:00", + "origin_arrivals_rps": 200.35833333333335, + "origin_499_rps": 0.025, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.058333333333333334, + "list_queries_rps": 0.525, + "list_page_queries_rps": 182.6, + "list_search_queries_rps": 2.216666666666667, + "list_updated_queries_rps": 0.6333333333333333, + "other_queries_rps": 14.433333333333334, + "/v0.1/servers_p95_seconds": 0.09608274381000505, + "/v0/servers_p95_seconds": 0.0970444742035523, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009696551724137931, + "list_page_acquire_p95_seconds": 0.0009642121590366753, + "list_search_acquire_p95_seconds": 0.0009984984984984983, + "list_updated_acquire_p95_seconds": 0.0009555555555555553, + "other_acquire_p95_seconds": 0.0009982786885245902, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.8697924419058836, + "node_cpu_apu6": 0.8878692763069315 + }, + { + "at": "2026-10-28T17:13:00+00:00", + "origin_arrivals_rps": 190.85000000000005, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.475, + "list_page_queries_rps": 178.33333333333331, + "list_search_queries_rps": 1.7833333333333332, + "list_updated_queries_rps": 1.6666666666666667, + "other_queries_rps": 8.383333333333333, + "/v0.1/servers_p95_seconds": 0.09613929231581726, + "/v0/servers_p95_seconds": 0.09684807953772943, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009697916666666667, + "list_page_acquire_p95_seconds": 0.0009641418869309838, + "list_search_acquire_p95_seconds": 0.001840000000000006, + "list_updated_acquire_p95_seconds": 0.000965886287625418, + "other_acquire_p95_seconds": 0.0009912116679132385, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.8953125150558966, + "node_cpu_apu6": 0.9130759100972985 + }, + { + "at": "2026-10-28T17:14:00+00:00", + "origin_arrivals_rps": 197.925, + "origin_499_rps": 0.03333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.5083333333333333, + "list_page_queries_rps": 185.25, + "list_search_queries_rps": 2.1083333333333334, + "list_updated_queries_rps": 2.5583333333333336, + "other_queries_rps": 7.308333333333334, + "/v0.1/servers_p95_seconds": 0.09659855876706204, + "/v0/servers_p95_seconds": 0.09730425344158136, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009688741721854303, + "list_page_acquire_p95_seconds": 0.000965470701963117, + "list_search_acquire_p95_seconds": 0.0029999999999999797, + "list_updated_acquire_p95_seconds": 0.0009684019370460048, + "other_acquire_p95_seconds": 0.0009953943580886587, + "sampled_inflight": 7.0, + "node_cpu_gc71": 0.918298235584128, + "node_cpu_apu6": 0.9281383582630951 + }, + { + "at": "2026-10-28T17:15:00+00:00", + "origin_arrivals_rps": 219.93333333333337, + "origin_499_rps": 0.05, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.55, + "list_page_queries_rps": 206.38333333333333, + "list_search_queries_rps": 2.591666666666667, + "list_updated_queries_rps": 1.5666666666666667, + "other_queries_rps": 8.850000000000001, + "/v0.1/servers_p95_seconds": 0.09638216572318492, + "/v0/servers_p95_seconds": 0.09736327214966466, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009829861111111111, + "list_page_acquire_p95_seconds": 0.0009655701676335552, + "list_search_acquire_p95_seconds": 0.002499999999999976, + "list_updated_acquire_p95_seconds": 0.0009684466019417475, + "other_acquire_p95_seconds": 0.0009991324657991324, + "sampled_inflight": 5.0, + "node_cpu_gc71": 0.8945508597978914, + "node_cpu_apu6": 0.9545622531952782 + }, + { + "at": "2026-10-28T17:16:00+00:00", + "origin_arrivals_rps": 233.3083333333333, + "origin_499_rps": 0.058333333333333334, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.4833333333333333, + "list_page_queries_rps": 216.00833333333333, + "list_search_queries_rps": 3.1916666666666664, + "list_updated_queries_rps": 0.7666666666666666, + "other_queries_rps": 12.625, + "/v0.1/servers_p95_seconds": 0.09662527714878762, + "/v0/servers_p95_seconds": 0.09783170807884507, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.000976206896551724, + "list_page_acquire_p95_seconds": 0.0009668775499664309, + "list_search_acquire_p95_seconds": 0.0009905982905982907, + "list_updated_acquire_p95_seconds": 0.0009697004608294929, + "other_acquire_p95_seconds": 0.000995462601914215, + "sampled_inflight": 11.0, + "node_cpu_gc71": 0.8724557607657819, + "node_cpu_apu6": 0.9580080067758293 + }, + { + "at": "2026-10-28T17:17:00+00:00", + "origin_arrivals_rps": 225.78333333333333, + "origin_499_rps": 0.18333333333333335, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.475, + "list_page_queries_rps": 202.49166666666667, + "list_search_queries_rps": 2.6333333333333333, + "list_updated_queries_rps": 0.7416666666666667, + "other_queries_rps": 19.108333333333334, + "/v0.1/servers_p95_seconds": 0.09662204009353643, + "/v0/servers_p95_seconds": 0.09800321394070001, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009697916666666667, + "list_page_acquire_p95_seconds": 0.0009665857286982097, + "list_search_acquire_p95_seconds": 0.0009870990237099025, + "list_updated_acquire_p95_seconds": 0.0009678823529411765, + "other_acquire_p95_seconds": 0.0009962119094213027, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.8135984204762887, + "node_cpu_apu6": 0.9168237634820495 + }, + { + "at": "2026-10-28T17:18:00+00:00", + "origin_arrivals_rps": 194.83333333333337, + "origin_499_rps": 0.14166666666666666, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.07500000000000001, + "list_queries_rps": 0.5166666666666666, + "list_page_queries_rps": 173.925, + "list_search_queries_rps": 2.375, + "list_updated_queries_rps": 0.875, + "other_queries_rps": 17.091666666666665, + "/v0.1/servers_p95_seconds": 0.09669756835499774, + "/v0/servers_p95_seconds": 0.09866830429932205, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009756756756756756, + "list_page_acquire_p95_seconds": 0.0009671007593001234, + "list_search_acquire_p95_seconds": 0.0009907142857142857, + "list_updated_acquire_p95_seconds": 0.000961046511627907, + "other_acquire_p95_seconds": 0.0009973372001054574, + "sampled_inflight": 4.0, + "node_cpu_gc71": 0.7700191715935943, + "node_cpu_apu6": 0.8896076012131137 + }, + { + "at": "2026-10-28T17:19:00+00:00", + "origin_arrivals_rps": 177.84166666666673, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.075, + "list_queries_rps": 0.48333333333333334, + "list_page_queries_rps": 161.05833333333334, + "list_search_queries_rps": 2.916666666666667, + "list_updated_queries_rps": 0.9916666666666667, + "other_queries_rps": 12.383333333333333, + "/v0.1/servers_p95_seconds": 0.09619433527578261, + "/v0/servers_p95_seconds": 0.09839044943820226, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.000977737226277372, + "list_page_acquire_p95_seconds": 0.0009655074884171964, + "list_search_acquire_p95_seconds": 0.0009839285714285714, + "list_updated_acquire_p95_seconds": 0.0009657676348547717, + "other_acquire_p95_seconds": 0.0009918009136811734, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.8035381334656753, + "node_cpu_apu6": 0.9235292489540976 + }, + { + "at": "2026-10-28T17:20:00+00:00", + "origin_arrivals_rps": 196.1416666666667, + "origin_499_rps": 0.041666666666666664, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.45833333333333337, + "list_page_queries_rps": 177.63333333333333, + "list_search_queries_rps": 2.041666666666667, + "list_updated_queries_rps": 0.8, + "other_queries_rps": 15.183333333333334, + "/v0.1/servers_p95_seconds": 0.09642420298908015, + "/v0/servers_p95_seconds": 0.09811875445201994, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009635714285714288, + "list_page_acquire_p95_seconds": 0.0009656742556917689, + "list_search_acquire_p95_seconds": 0.000988912429378531, + "list_updated_acquire_p95_seconds": 0.0009689243027888445, + "other_acquire_p95_seconds": 0.0009869779138421168, + "sampled_inflight": 7.0, + "node_cpu_gc71": 0.8553208360477583, + "node_cpu_apu6": 0.9294203475563942 + }, + { + "at": "2026-10-28T17:21:00+00:00", + "origin_arrivals_rps": 207.5666666666667, + "origin_499_rps": 0.075, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.4916666666666667, + "list_page_queries_rps": 192.1, + "list_search_queries_rps": 1.15, + "list_updated_queries_rps": 0.825, + "other_queries_rps": 12.841666666666667, + "/v0.1/servers_p95_seconds": 0.09630610224892389, + "/v0/servers_p95_seconds": 0.09743978686706911, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009634751773049645, + "list_page_acquire_p95_seconds": 0.0009651264159149789, + "list_search_acquire_p95_seconds": 0.0010499999999999722, + "list_updated_acquire_p95_seconds": 0.0009653225806451613, + "other_acquire_p95_seconds": 0.0009870864079980957, + "sampled_inflight": 3.0, + "node_cpu_gc71": 0.891251825116827, + "node_cpu_apu6": 0.9280944828836558 + }, + { + "at": "2026-10-28T17:22:00+00:00", + "origin_arrivals_rps": 195.80000000000004, + "origin_499_rps": 0.03333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.48333333333333334, + "list_page_queries_rps": 185.50833333333333, + "list_search_queries_rps": 1.225, + "list_updated_queries_rps": 0.8416666666666667, + "other_queries_rps": 7.6, + "/v0.1/servers_p95_seconds": 0.09647488198042604, + "/v0/servers_p95_seconds": 0.097538033715628, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009567375886524825, + "list_page_acquire_p95_seconds": 0.0009654917985721668, + "list_search_acquire_p95_seconds": 0.0012000000000000222, + "list_updated_acquire_p95_seconds": 0.0009643939393939395, + "other_acquire_p95_seconds": 0.0009783189920098338, + "sampled_inflight": 3.0, + "node_cpu_gc71": 0.8763873746827205, + "node_cpu_apu6": 0.9133051316881998 + }, + { + "at": "2026-10-28T17:23:00+00:00", + "origin_arrivals_rps": 185.31666666666672, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.4166666666666667, + "list_page_queries_rps": 176.46666666666667, + "list_search_queries_rps": 1.975, + "list_updated_queries_rps": 0.575, + "other_queries_rps": 5.933333333333334, + "/v0.1/servers_p95_seconds": 0.09661827975145212, + "/v0/servers_p95_seconds": 0.09760005527915977, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009662189888190341, + "list_search_acquire_p95_seconds": 0.0009877840909090908, + "list_updated_acquire_p95_seconds": 0.0009679245283018867, + "other_acquire_p95_seconds": 0.0009758679373723623, + "sampled_inflight": 9.0, + "node_cpu_gc71": 0.9337750098369725, + "node_cpu_apu6": 0.9595076474986853 + }, + { + "at": "2026-10-28T17:24:00+00:00", + "origin_arrivals_rps": 193.13333333333335, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.3916666666666667, + "list_page_queries_rps": 183.99166666666667, + "list_search_queries_rps": 2.95, + "list_updated_queries_rps": 0.6666666666666666, + "other_queries_rps": 5.325, + "/v0.1/servers_p95_seconds": 0.09743049613655214, + "/v0/servers_p95_seconds": 0.09888120614325434, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009571969696969698, + "list_page_acquire_p95_seconds": 0.0009689264453453125, + "list_search_acquire_p95_seconds": 0.0009932604735883423, + "list_updated_acquire_p95_seconds": 0.0009583700440528634, + "other_acquire_p95_seconds": 0.0009755693581780535, + "sampled_inflight": 7.0, + "node_cpu_gc71": 0.9532100213213804, + "node_cpu_apu6": 0.9900418647545545 + }, + { + "at": "2026-10-28T17:25:00+00:00", + "origin_arrivals_rps": 204.00833333333338, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.058333333333333334, + "list_queries_rps": 0.475, + "list_page_queries_rps": 195.35000000000002, + "list_search_queries_rps": 2.7666666666666666, + "list_updated_queries_rps": 0.6416666666666666, + "other_queries_rps": 5.1, + "/v0.1/servers_p95_seconds": 0.09914919178680645, + "/v0/servers_p95_seconds": 0.1440689655172415, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009647286821705426, + "list_page_acquire_p95_seconds": 0.000971699058304949, + "list_search_acquire_p95_seconds": 0.0009904918032786884, + "list_updated_acquire_p95_seconds": 0.0009597435897435897, + "other_acquire_p95_seconds": 0.0009726190476190477, + "sampled_inflight": 4.0, + "node_cpu_gc71": 0.9772896723125692, + "node_cpu_apu6": 0.9892663765870869 + }, + { + "at": "2026-10-28T17:26:00+00:00", + "origin_arrivals_rps": 205.10000000000008, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.058333333333333334, + "list_queries_rps": 0.4583333333333333, + "list_page_queries_rps": 196.86666666666667, + "list_search_queries_rps": 2.708333333333333, + "list_updated_queries_rps": 0.2583333333333333, + "other_queries_rps": 4.891666666666667, + "/v0.1/servers_p95_seconds": 0.11310584389047783, + "/v0/servers_p95_seconds": 0.17662020905923348, + "sampled_pending_acquisitions": 1.0, + "list_acquire_p95_seconds": 0.0009655737704918032, + "list_page_acquire_p95_seconds": 0.0009743815547921552, + "list_search_acquire_p95_seconds": 0.000987636612021858, + "list_updated_acquire_p95_seconds": 0.0009680379746835443, + "other_acquire_p95_seconds": 0.0009728052126200275, + "sampled_inflight": 21.0, + "node_cpu_gc71": 0.9767639419260377, + "node_cpu_apu6": 0.9893069324995546 + }, + { + "at": "2026-10-28T17:27:00+00:00", + "origin_arrivals_rps": 204.98333333333335, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.049999999999999996, + "list_queries_rps": 0.375, + "list_page_queries_rps": 197.09166666666667, + "list_search_queries_rps": 2.7666666666666666, + "list_updated_queries_rps": 0.21666666666666667, + "other_queries_rps": 4.583333333333334, + "/v0.1/servers_p95_seconds": 0.13851674252275648, + "/v0/servers_p95_seconds": 0.18727769679300277, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009927927927927927, + "list_page_acquire_p95_seconds": 0.0009765911080149331, + "list_search_acquire_p95_seconds": 0.0009860303413400757, + "list_updated_acquire_p95_seconds": 0.0009739495798319327, + "other_acquire_p95_seconds": 0.0009818317200297841, + "sampled_inflight": 3.0, + "node_cpu_gc71": 0.9771950468384603, + "node_cpu_apu6": 0.9875204544623557 + }, + { + "at": "2026-10-28T17:28:00+00:00", + "origin_arrivals_rps": 198.58333333333331, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05833333333333333, + "list_queries_rps": 0.375, + "list_page_queries_rps": 191.375, + "list_search_queries_rps": 2.0, + "list_updated_queries_rps": 0.30833333333333335, + "other_queries_rps": 4.791666666666667, + "/v0.1/servers_p95_seconds": 0.1573863326980669, + "/v0/servers_p95_seconds": 0.1904222520107237, + "sampled_pending_acquisitions": 1.0, + "list_acquire_p95_seconds": 0.00120000000000001, + "list_page_acquire_p95_seconds": 0.0009788347521370547, + "list_search_acquire_p95_seconds": 0.0009902872777017783, + "list_updated_acquire_p95_seconds": 0.0009804000000000002, + "other_acquire_p95_seconds": 0.0009850230414746542, + "sampled_inflight": 22.0, + "node_cpu_gc71": 0.987146233170885, + "node_cpu_apu6": 0.9901967398977528 + }, + { + "at": "2026-10-28T17:29:00+00:00", + "origin_arrivals_rps": 195.31666666666672, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.058333333333333334, + "list_queries_rps": 0.3333333333333333, + "list_page_queries_rps": 188.60833333333335, + "list_search_queries_rps": 1.5083333333333333, + "list_updated_queries_rps": 0.475, + "other_queries_rps": 4.641666666666667, + "/v0.1/servers_p95_seconds": 0.16675072046109524, + "/v0/servers_p95_seconds": 0.1979993206521739, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0036000000000000064, + "list_page_acquire_p95_seconds": 0.0009800569927028034, + "list_search_acquire_p95_seconds": 0.0009946515397082658, + "list_updated_acquire_p95_seconds": 0.00099, + "other_acquire_p95_seconds": 0.000979992107340174, + "sampled_inflight": 4.0, + "node_cpu_gc71": 0.9776847327232341, + "node_cpu_apu6": 0.992896850011554 + }, + { + "at": "2026-10-28T17:30:00+00:00", + "origin_arrivals_rps": 209.3916666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.3, + "list_page_queries_rps": 202.30833333333334, + "list_search_queries_rps": 1.35, + "list_updated_queries_rps": 0.7833333333333333, + "other_queries_rps": 4.75, + "/v0.1/servers_p95_seconds": 0.1671421125781793, + "/v0/servers_p95_seconds": 0.20648437499999994, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.010799999999999912, + "list_page_acquire_p95_seconds": 0.0009791647032130828, + "list_search_acquire_p95_seconds": 0.0009918871252204587, + "list_updated_acquire_p95_seconds": 0.0015599999999999917, + "other_acquire_p95_seconds": 0.000977205882352941, + "sampled_inflight": 11.0, + "node_cpu_gc71": 0.9320104185717355, + "node_cpu_apu6": 0.9822669463976732 + }, + { + "at": "2026-10-28T17:31:00+00:00", + "origin_arrivals_rps": 231.25833333333333, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.35, + "list_page_queries_rps": 222.30833333333334, + "list_search_queries_rps": 2.225, + "list_updated_queries_rps": 1.0666666666666667, + "other_queries_rps": 5.075, + "/v0.1/servers_p95_seconds": 0.1812127515479876, + "/v0/servers_p95_seconds": 0.21659358638743442, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.010799999999999782, + "list_page_acquire_p95_seconds": 0.0009782662872345799, + "list_search_acquire_p95_seconds": 0.0009955595026642985, + "list_updated_acquire_p95_seconds": 0.0011333333333333193, + "other_acquire_p95_seconds": 0.0009818354912414318, + "sampled_inflight": 12.0, + "node_cpu_gc71": 0.8941720299868999, + "node_cpu_apu6": 0.9904873665965472 + }, + { + "at": "2026-10-28T17:32:00+00:00", + "origin_arrivals_rps": 236.75, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.3833333333333333, + "list_page_queries_rps": 226.99166666666667, + "list_search_queries_rps": 3.1916666666666664, + "list_updated_queries_rps": 1.0666666666666667, + "other_queries_rps": 4.75, + "/v0.1/servers_p95_seconds": 0.20567169117647047, + "/v0/servers_p95_seconds": 0.23594269870609963, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00099, + "list_page_acquire_p95_seconds": 0.0009801696590347555, + "list_search_acquire_p95_seconds": 0.001457142857142805, + "list_updated_acquire_p95_seconds": 0.00099390756302521, + "other_acquire_p95_seconds": 0.0009802272727272726, + "sampled_inflight": 9.0, + "node_cpu_gc71": 0.9545503403118346, + "node_cpu_apu6": 0.9977860885674696 + }, + { + "at": "2026-10-28T17:33:00+00:00", + "origin_arrivals_rps": 224.1166666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.39166666666666666, + "list_page_queries_rps": 214.1, + "list_search_queries_rps": 2.95, + "list_updated_queries_rps": 1.8083333333333333, + "other_queries_rps": 4.475, + "/v0.1/servers_p95_seconds": 0.22228820552315984, + "/v0/servers_p95_seconds": 0.24260853835021706, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009904255319148935, + "list_page_acquire_p95_seconds": 0.000981443423988617, + "list_search_acquire_p95_seconds": 0.0009981371087928465, + "list_updated_acquire_p95_seconds": 0.0009813186813186813, + "other_acquire_p95_seconds": 0.000979503105590062, + "sampled_inflight": 26.0, + "node_cpu_gc71": 0.9839566006160263, + "node_cpu_apu6": 0.9976294983593107 + }, + { + "at": "2026-10-28T17:34:00+00:00", + "origin_arrivals_rps": 213.475, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.008333333333333333, + "list_queries_rps": 0.3666666666666667, + "list_page_queries_rps": 202.80833333333334, + "list_search_queries_rps": 2.8833333333333333, + "list_updated_queries_rps": 2.9416666666666664, + "other_queries_rps": 4.1, + "/v0.1/servers_p95_seconds": 0.2308000281452293, + "/v0/servers_p95_seconds": 0.24772525380710667, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009779411764705881, + "list_page_acquire_p95_seconds": 0.0009832569508096548, + "list_search_acquire_p95_seconds": 0.0009890745501285347, + "list_updated_acquire_p95_seconds": 0.0009843809523809525, + "other_acquire_p95_seconds": 0.000990235294117647, + "sampled_inflight": 34.0, + "node_cpu_gc71": 0.9784210181724726, + "node_cpu_apu6": 0.9976321345609903 + }, + { + "at": "2026-10-28T17:35:00+00:00", + "origin_arrivals_rps": 209.5250000000001, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.0, + "list_queries_rps": 0.2833333333333333, + "list_page_queries_rps": 198.88333333333333, + "list_search_queries_rps": 3.341666666666667, + "list_updated_queries_rps": 2.7583333333333333, + "other_queries_rps": 3.9833333333333334, + "/v0.1/servers_p95_seconds": 0.23961250603573142, + "/v0/servers_p95_seconds": 0.2886627906976752, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009695876288659794, + "list_page_acquire_p95_seconds": 0.0009857397446168435, + "list_search_acquire_p95_seconds": 0.0009877067254685777, + "list_updated_acquire_p95_seconds": 0.0009803872053872052, + "other_acquire_p95_seconds": 0.0009953871499176275, + "sampled_inflight": 13.0, + "node_cpu_gc71": 0.9865738456991191, + "node_cpu_apu6": 0.9951457185634354 + }, + { + "at": "2026-10-28T17:36:00+00:00", + "origin_arrivals_rps": 209.54166666666666, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05833333333333333, + "list_queries_rps": 0.35, + "list_page_queries_rps": 199.80833333333334, + "list_search_queries_rps": 2.841666666666667, + "list_updated_queries_rps": 1.4833333333333334, + "other_queries_rps": 4.025, + "/v0.1/servers_p95_seconds": 0.24694103332085368, + "/v0/servers_p95_seconds": 0.3191129032258068, + "sampled_pending_acquisitions": 1.0, + "list_acquire_p95_seconds": 0.0009979797979797979, + "list_page_acquire_p95_seconds": 0.0009901761265677293, + "list_search_acquire_p95_seconds": 0.0009876896149358225, + "list_updated_acquire_p95_seconds": 0.0009881993006993007, + "other_acquire_p95_seconds": 0.0009966257668711659, + "sampled_inflight": 48.0, + "node_cpu_gc71": 0.9883194116710607, + "node_cpu_apu6": 0.995431966461183 + }, + { + "at": "2026-10-28T17:37:00+00:00", + "origin_arrivals_rps": 215.7, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.09166666666666666, + "list_queries_rps": 0.45000000000000007, + "list_page_queries_rps": 207.8, + "list_search_queries_rps": 2.0749999999999997, + "list_updated_queries_rps": 0.6333333333333333, + "other_queries_rps": 3.8, + "/v0.1/servers_p95_seconds": 0.24601416648363716, + "/v0/servers_p95_seconds": 0.32585125448028635, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.000997029702970297, + "list_page_acquire_p95_seconds": 0.0009901821000272554, + "list_search_acquire_p95_seconds": 0.0009879021879021879, + "list_updated_acquire_p95_seconds": 0.0009888475836431225, + "other_acquire_p95_seconds": 0.0009997661365762395, + "sampled_inflight": 24.0, + "node_cpu_gc71": 0.9854840857602539, + "node_cpu_apu6": 0.9977809914756411 + }, + { + "at": "2026-10-28T17:38:00+00:00", + "origin_arrivals_rps": 224.0416666666667, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.058333333333333334, + "list_queries_rps": 0.4083333333333333, + "list_page_queries_rps": 217.3, + "list_search_queries_rps": 1.9416666666666667, + "list_updated_queries_rps": 0.5083333333333333, + "other_queries_rps": 3.9416666666666664, + "/v0.1/servers_p95_seconds": 0.24553454129456034, + "/v0/servers_p95_seconds": 0.3649038461538464, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009865384615384614, + "list_page_acquire_p95_seconds": 0.000989887933062267, + "list_search_acquire_p95_seconds": 0.000987688098495212, + "list_updated_acquire_p95_seconds": 0.0009978417266187048, + "other_acquire_p95_seconds": 0.0009998127340823972, + "sampled_inflight": 14.0, + "node_cpu_gc71": 0.9843783268809299, + "node_cpu_apu6": 0.9979653312759182 + }, + { + "at": "2026-10-28T17:39:00+00:00", + "origin_arrivals_rps": 222.1666666666666, + "origin_499_rps": 0.016666666666666666, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.43333333333333335, + "list_page_queries_rps": 214.5, + "list_search_queries_rps": 2.2416666666666667, + "list_updated_queries_rps": 0.475, + "other_queries_rps": 4.575, + "/v0.1/servers_p95_seconds": 0.24875708289611753, + "/v0/servers_p95_seconds": 0.39725409836065717, + "sampled_pending_acquisitions": 1.0, + "list_acquire_p95_seconds": 0.006500000000000017, + "list_page_acquire_p95_seconds": 0.0009905518701246749, + "list_search_acquire_p95_seconds": 0.000990714285714286, + "list_updated_acquire_p95_seconds": 0.001999999999999978, + "other_acquire_p95_seconds": 0.0009949955317247543, + "sampled_inflight": 22.0, + "node_cpu_gc71": 0.9850918286130763, + "node_cpu_apu6": 0.9960595003370448 + }, + { + "at": "2026-10-28T17:40:00+00:00", + "origin_arrivals_rps": 219.22500000000002, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.44166666666666665, + "list_page_queries_rps": 209.36666666666667, + "list_search_queries_rps": 3.5583333333333336, + "list_updated_queries_rps": 0.375, + "other_queries_rps": 5.316666666666666, + "/v0.1/servers_p95_seconds": 0.2915962400827864, + "/v0/servers_p95_seconds": 0.45323153409090977, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.014666666666666606, + "list_page_acquire_p95_seconds": 0.0009919394219348027, + "list_search_acquire_p95_seconds": 0.0009928950863213812, + "list_updated_acquire_p95_seconds": 0.002866666666666615, + "other_acquire_p95_seconds": 0.000993822075782537, + "sampled_inflight": 14.0, + "node_cpu_gc71": 0.9780361462709455, + "node_cpu_apu6": 1.0000236510187581 + }, + { + "at": "2026-10-28T17:41:00+00:00", + "origin_arrivals_rps": 226.6666666666667, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.09166666666666667, + "list_queries_rps": 0.525, + "list_page_queries_rps": 213.775, + "list_search_queries_rps": 6.091666666666667, + "list_updated_queries_rps": 0.30833333333333335, + "other_queries_rps": 5.741666666666667, + "/v0.1/servers_p95_seconds": 0.34757382550335575, + "/v0/servers_p95_seconds": 0.6543346774193559, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0075, + "list_page_acquire_p95_seconds": 0.0009930336201428084, + "list_search_acquire_p95_seconds": 0.000995365168539326, + "list_updated_acquire_p95_seconds": 0.00097375, + "other_acquire_p95_seconds": 0.001149999999999985, + "sampled_inflight": 63.0, + "node_cpu_gc71": 0.9570650711286968, + "node_cpu_apu6": 1.000197979642447 + }, + { + "at": "2026-10-28T17:42:00+00:00", + "origin_arrivals_rps": 236.67500000000007, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.09166666666666667, + "list_queries_rps": 0.5666666666666667, + "list_page_queries_rps": 222.64999999999998, + "list_search_queries_rps": 6.308333333333334, + "list_updated_queries_rps": 0.7833333333333333, + "other_queries_rps": 6.441666666666666, + "/v0.1/servers_p95_seconds": 0.39714219807258505, + "/v0/servers_p95_seconds": 0.7917500000000011, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.010999999999999888, + "list_page_acquire_p95_seconds": 0.0009955317024463303, + "list_search_acquire_p95_seconds": 0.00099607922392886, + "list_updated_acquire_p95_seconds": 0.00099079754601227, + "other_acquire_p95_seconds": 0.0015684210526315993, + "sampled_inflight": 36.0, + "node_cpu_gc71": 0.9681942539486045, + "node_cpu_apu6": 0.9983127847815921 + }, + { + "at": "2026-10-28T17:43:00+00:00", + "origin_arrivals_rps": 239.15833333333333, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.008333333333333333, + "list_queries_rps": 0.3916666666666667, + "list_page_queries_rps": 225.91666666666666, + "list_search_queries_rps": 3.9749999999999996, + "list_updated_queries_rps": 1.1749999999999998, + "other_queries_rps": 7.966666666666667, + "/v0.1/servers_p95_seconds": 0.4049885866463761, + "/v0/servers_p95_seconds": 0.8556603773584899, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.01399999999999989, + "list_page_acquire_p95_seconds": 0.0009953535207651324, + "list_search_acquire_p95_seconds": 0.0009981936971560337, + "list_updated_acquire_p95_seconds": 0.0015333333333333186, + "other_acquire_p95_seconds": 0.002129411764705884, + "sampled_inflight": 28.0, + "node_cpu_gc71": 0.9521588683666777, + "node_cpu_apu6": 0.9989890117268563 + }, + { + "at": "2026-10-28T17:44:00+00:00", + "origin_arrivals_rps": 233.73333333333332, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.3416666666666667, + "list_page_queries_rps": 221.8, + "list_search_queries_rps": 2.5, + "list_updated_queries_rps": 0.9750000000000001, + "other_queries_rps": 8.175, + "/v0.1/servers_p95_seconds": 0.40762072434607655, + "/v0/servers_p95_seconds": 0.878445512820512, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009983050847457628, + "list_page_acquire_p95_seconds": 0.0009948402051216919, + "list_search_acquire_p95_seconds": 0.0023846153846153657, + "list_updated_acquire_p95_seconds": 0.000997716894977169, + "other_acquire_p95_seconds": 0.004909999999999943, + "sampled_inflight": 75.0, + "node_cpu_gc71": 0.9420615426798273, + "node_cpu_apu6": 0.9979496382838988 + }, + { + "at": "2026-10-28T17:45:00+00:00", + "origin_arrivals_rps": 231.30833333333334, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.4166666666666667, + "list_page_queries_rps": 219.73333333333335, + "list_search_queries_rps": 1.8, + "list_updated_queries_rps": 0.9083333333333333, + "other_queries_rps": 8.241666666666667, + "/v0.1/servers_p95_seconds": 0.40453550130450955, + "/v0/servers_p95_seconds": 0.8525547445255472, + "sampled_pending_acquisitions": 3.0, + "list_acquire_p95_seconds": 0.0009983050847457628, + "list_page_acquire_p95_seconds": 0.0009937799720130187, + "list_search_acquire_p95_seconds": 0.0030714285714285063, + "list_updated_acquire_p95_seconds": 0.0009984313725490196, + "other_acquire_p95_seconds": 0.00965909090909086, + "sampled_inflight": 30.0, + "node_cpu_gc71": 0.9749315300611973, + "node_cpu_apu6": 0.9989993325390969 + }, + { + "at": "2026-10-28T17:46:00+00:00", + "origin_arrivals_rps": 228.73333333333335, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.4083333333333333, + "list_page_queries_rps": 215.68333333333334, + "list_search_queries_rps": 2.1, + "list_updated_queries_rps": 1.2000000000000002, + "other_queries_rps": 9.125, + "/v0.1/servers_p95_seconds": 0.38318143281051253, + "/v0/servers_p95_seconds": 0.6527616279069762, + "sampled_pending_acquisitions": 7.0, + "list_acquire_p95_seconds": 0.000985185185185185, + "list_page_acquire_p95_seconds": 0.0009922995547415288, + "list_search_acquire_p95_seconds": 0.0009988095238095238, + "list_updated_acquire_p95_seconds": 0.0026599999999999983, + "other_acquire_p95_seconds": 0.003944827586206857, + "sampled_inflight": 43.0, + "node_cpu_gc71": 0.9812965980837665, + "node_cpu_apu6": 0.9993745183024051 + }, + { + "at": "2026-10-28T17:47:00+00:00", + "origin_arrivals_rps": 224.71666666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.06666666666666667, + "list_queries_rps": 0.4833333333333334, + "list_page_queries_rps": 213.05833333333334, + "list_search_queries_rps": 2.158333333333333, + "list_updated_queries_rps": 1.0333333333333332, + "other_queries_rps": 7.883333333333333, + "/v0.1/servers_p95_seconds": 0.3773993549822059, + "/v0/servers_p95_seconds": 0.4757241813602014, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009759090909090906, + "list_page_acquire_p95_seconds": 0.0009923436376707403, + "list_search_acquire_p95_seconds": 0.0009934542586750787, + "list_updated_acquire_p95_seconds": 0.00354999999999998, + "other_acquire_p95_seconds": 0.0035933333333333147, + "sampled_inflight": 50.0, + "node_cpu_gc71": 0.9820598367009855, + "node_cpu_apu6": 0.9979195742952773 + }, + { + "at": "2026-10-28T17:48:00+00:00", + "origin_arrivals_rps": 230.01666666666668, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.45833333333333337, + "list_page_queries_rps": 218.46666666666667, + "list_search_queries_rps": 2.708333333333333, + "list_updated_queries_rps": 0.5333333333333333, + "other_queries_rps": 7.966666666666667, + "/v0.1/servers_p95_seconds": 0.4353290833907649, + "/v0/servers_p95_seconds": 0.6164948453608273, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009833333333333335, + "list_page_acquire_p95_seconds": 0.0009978523954360385, + "list_search_acquire_p95_seconds": 0.0011999999999999255, + "list_updated_acquire_p95_seconds": 0.002333333333333341, + "other_acquire_p95_seconds": 0.00811666666666658, + "sampled_inflight": 21.0, + "node_cpu_gc71": 0.9564734046745191, + "node_cpu_apu6": 0.9963588963762352 + }, + { + "at": "2026-10-28T17:49:00+00:00", + "origin_arrivals_rps": 237.81666666666663, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.39166666666666666, + "list_page_queries_rps": 225.24166666666667, + "list_search_queries_rps": 2.8083333333333336, + "list_updated_queries_rps": 0.3416666666666667, + "other_queries_rps": 8.991666666666667, + "/v0.1/servers_p95_seconds": 0.44091020608439635, + "/v0/servers_p95_seconds": 0.6112944162436591, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.009750000000000026, + "list_page_acquire_p95_seconds": 0.0009983135744871222, + "list_search_acquire_p95_seconds": 0.0009946212121212123, + "list_updated_acquire_p95_seconds": 0.002933333333333318, + "other_acquire_p95_seconds": 0.006062499999999942, + "sampled_inflight": 25.0, + "node_cpu_gc71": 0.936846133141565, + "node_cpu_apu6": 0.9998457946120956 + }, + { + "at": "2026-10-28T17:50:00+00:00", + "origin_arrivals_rps": 240.37500000000003, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.3583333333333333, + "list_page_queries_rps": 230.51666666666665, + "list_search_queries_rps": 1.5833333333333335, + "list_updated_queries_rps": 0.35833333333333334, + "other_queries_rps": 7.191666666666666, + "/v0.1/servers_p95_seconds": 0.432024092297252, + "/v0/servers_p95_seconds": 0.5436827956989253, + "sampled_pending_acquisitions": 2.0, + "list_acquire_p95_seconds": 0.003, + "list_page_acquire_p95_seconds": 0.0009993909952606636, + "list_search_acquire_p95_seconds": 0.0009992924528301889, + "list_updated_acquire_p95_seconds": 0.002840000000000017, + "other_acquire_p95_seconds": 0.004442424242424209, + "sampled_inflight": 48.0, + "node_cpu_gc71": 0.9517792655560091, + "node_cpu_apu6": 0.9984890658694591 + }, + { + "at": "2026-10-28T17:51:00+00:00", + "origin_arrivals_rps": 240.8166666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.35, + "list_page_queries_rps": 231.28333333333333, + "list_search_queries_rps": 1.875, + "list_updated_queries_rps": 0.43333333333333335, + "other_queries_rps": 6.8, + "/v0.1/servers_p95_seconds": 0.44723451327433633, + "/v0/servers_p95_seconds": 0.6424358974358956, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0059999999999999, + "list_page_acquire_p95_seconds": 0.001193055555555569, + "list_search_acquire_p95_seconds": 0.013300000000000027, + "list_updated_acquire_p95_seconds": 0.0009724409448818898, + "other_acquire_p95_seconds": 0.006749999999999956, + "sampled_inflight": 29.0, + "node_cpu_gc71": 0.9695414728953134, + "node_cpu_apu6": 0.998616118033533 + }, + { + "at": "2026-10-28T17:52:00+00:00", + "origin_arrivals_rps": 245.15000000000003, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.325, + "list_page_queries_rps": 235.52499999999998, + "list_search_queries_rps": 2.1083333333333334, + "list_updated_queries_rps": 0.31666666666666665, + "other_queries_rps": 6.8, + "/v0.1/servers_p95_seconds": 0.4540266259275424, + "/v0/servers_p95_seconds": 0.6848557692307692, + "sampled_pending_acquisitions": 3.0, + "list_acquire_p95_seconds": 0.017333333333333298, + "list_page_acquire_p95_seconds": 0.0009997682969445584, + "list_search_acquire_p95_seconds": 0.023181818181818317, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0062692307692307665, + "sampled_inflight": 54.0, + "node_cpu_gc71": 0.9329418855908443, + "node_cpu_apu6": 0.9974723671815203 + }, + { + "at": "2026-10-28T17:53:00+00:00", + "origin_arrivals_rps": 252.79166666666669, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.21666666666666667, + "list_page_queries_rps": 244.25, + "list_search_queries_rps": 1.5, + "list_updated_queries_rps": 0.22499999999999998, + "other_queries_rps": 6.283333333333333, + "/v0.1/servers_p95_seconds": 0.4380209862048723, + "/v0/servers_p95_seconds": 0.4790890269151137, + "sampled_pending_acquisitions": 1.0, + "list_acquire_p95_seconds": 0.013600000000000004, + "list_page_acquire_p95_seconds": 0.000996149289099526, + "list_search_acquire_p95_seconds": 0.017375000000000036, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0025124999999999735, + "sampled_inflight": 66.0, + "node_cpu_gc71": 0.9273138167724634, + "node_cpu_apu6": 1.0010041571236798 + }, + { + "at": "2026-10-28T17:54:00+00:00", + "origin_arrivals_rps": 254.2, + "origin_499_rps": 0.05, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.29166666666666663, + "list_page_queries_rps": 245.11666666666667, + "list_search_queries_rps": 1.775, + "list_updated_queries_rps": 0.2583333333333333, + "other_queries_rps": 6.283333333333333, + "/v0.1/servers_p95_seconds": 0.4774644250631168, + "/v0/servers_p95_seconds": 0.5799180327868855, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009969135802469135, + "list_page_acquire_p95_seconds": 0.0028502890173410225, + "list_search_acquire_p95_seconds": 0.02853333333333323, + "list_updated_acquire_p95_seconds": 0.0009697916666666668, + "other_acquire_p95_seconds": 0.004292857142857112, + "sampled_inflight": 93.0, + "node_cpu_gc71": 0.905021579114206, + "node_cpu_apu6": 0.9999908739999227 + }, + { + "at": "2026-10-28T17:55:00+00:00", + "origin_arrivals_rps": 255.68333333333334, + "origin_499_rps": 0.325, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.42500000000000004, + "list_page_queries_rps": 242.2, + "list_search_queries_rps": 2.2666666666666666, + "list_updated_queries_rps": 0.3666666666666667, + "other_queries_rps": 10.141666666666667, + "/v0.1/servers_p95_seconds": 0.605180826636051, + "/v0/servers_p95_seconds": 0.8243838028169014, + "sampled_pending_acquisitions": 8.0, + "list_acquire_p95_seconds": 0.02099999999999993, + "list_page_acquire_p95_seconds": 0.018658711217183703, + "list_search_acquire_p95_seconds": 0.04106666666666659, + "list_updated_acquire_p95_seconds": 0.0021999999999999936, + "other_acquire_p95_seconds": 0.035774193548387095, + "sampled_inflight": 85.0, + "node_cpu_gc71": 0.8958274754882344, + "node_cpu_apu6": 0.9977129111401154 + }, + { + "at": "2026-10-28T17:56:00+00:00", + "origin_arrivals_rps": 254.55, + "origin_499_rps": 0.4083333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.44166666666666665, + "list_page_queries_rps": 236.075, + "list_search_queries_rps": 2.9416666666666664, + "list_updated_queries_rps": 0.7416666666666667, + "other_queries_rps": 13.258333333333333, + "/v0.1/servers_p95_seconds": 0.8309647577092514, + "/v0/servers_p95_seconds": 1.1564655172413776, + "sampled_pending_acquisitions": 90.0, + "list_acquire_p95_seconds": 0.14499999999999993, + "list_page_acquire_p95_seconds": 0.10960616438356179, + "list_search_acquire_p95_seconds": 0.3911290322580644, + "list_updated_acquire_p95_seconds": 0.14687499999999992, + "other_acquire_p95_seconds": 0.16911931818181825, + "sampled_inflight": 204.0, + "node_cpu_gc71": 0.8802312277058566, + "node_cpu_apu6": 0.9999685207837804 + }, + { + "at": "2026-10-28T17:57:00+00:00", + "origin_arrivals_rps": 246.55000000000004, + "origin_499_rps": 0.14166666666666666, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.4, + "list_page_queries_rps": 232.08333333333331, + "list_search_queries_rps": 3.041666666666667, + "list_updated_queries_rps": 1.2916666666666667, + "other_queries_rps": 9.858333333333334, + "/v0.1/servers_p95_seconds": 1.260285752979413, + "/v0/servers_p95_seconds": 1.8285377358490584, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.6749999999999985, + "list_page_acquire_p95_seconds": 0.44202510917030513, + "list_search_acquire_p95_seconds": 0.8945945945945952, + "list_updated_acquire_p95_seconds": 0.9541666666666687, + "other_acquire_p95_seconds": 0.3713068181818182, + "sampled_inflight": 38.0, + "node_cpu_gc71": 0.9346397007117048, + "node_cpu_apu6": 1.0003495282677857 + }, + { + "at": "2026-10-28T17:58:00+00:00", + "origin_arrivals_rps": 252.22500000000002, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.375, + "list_page_queries_rps": 240.33333333333331, + "list_search_queries_rps": 2.466666666666667, + "list_updated_queries_rps": 2.166666666666667, + "other_queries_rps": 8.158333333333333, + "/v0.1/servers_p95_seconds": 1.26262155591572, + "/v0/servers_p95_seconds": 1.8082167832167817, + "sampled_pending_acquisitions": 3.0, + "list_acquire_p95_seconds": 0.6399999999999991, + "list_page_acquire_p95_seconds": 0.44218613775065285, + "list_search_acquire_p95_seconds": 0.8472972972972976, + "list_updated_acquire_p95_seconds": 0.34687499999999705, + "other_acquire_p95_seconds": 0.3546348314606724, + "sampled_inflight": 56.0, + "node_cpu_gc71": 0.9181247124681249, + "node_cpu_apu6": 0.9978496388621958 + }, + { + "at": "2026-10-28T17:59:00+00:00", + "origin_arrivals_rps": 259.575, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.008333333333333333, + "list_queries_rps": 0.325, + "list_page_queries_rps": 245.04166666666669, + "list_search_queries_rps": 3.925, + "list_updated_queries_rps": 2.0999999999999996, + "other_queries_rps": 7.991666666666667, + "/v0.1/servers_p95_seconds": 1.2422743533296683, + "/v0/servers_p95_seconds": 1.7918454935622314, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.6500000000000001, + "list_page_acquire_p95_seconds": 0.4396583143507966, + "list_search_acquire_p95_seconds": 0.7175675675675649, + "list_updated_acquire_p95_seconds": 0.2199999999999995, + "other_acquire_p95_seconds": 0.33963068181818096, + "sampled_inflight": 66.0, + "node_cpu_gc71": 0.8755866299499757, + "node_cpu_apu6": 0.9989918617010214 + }, + { + "at": "2026-10-28T18:00:00+00:00", + "origin_arrivals_rps": 252.12500000000003, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.32499999999999996, + "list_page_queries_rps": 236.91666666666669, + "list_search_queries_rps": 6.575, + "list_updated_queries_rps": 1.1416666666666666, + "other_queries_rps": 6.616666666666667, + "/v0.1/servers_p95_seconds": 1.2826954069298946, + "/v0/servers_p95_seconds": 1.998913043478261, + "sampled_pending_acquisitions": 30.0, + "list_acquire_p95_seconds": 0.6949999999999985, + "list_page_acquire_p95_seconds": 0.43922039323273876, + "list_search_acquire_p95_seconds": 0.4980769230769224, + "list_updated_acquire_p95_seconds": 0.20425000000000018, + "other_acquire_p95_seconds": 0.3972560975609747, + "sampled_inflight": 124.0, + "node_cpu_gc71": 0.8764508339450306, + "node_cpu_apu6": 1.000608010857104 + }, + { + "at": "2026-10-28T18:01:00+00:00", + "origin_arrivals_rps": 251.12500000000003, + "origin_499_rps": 0.025, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.4583333333333333, + "list_page_queries_rps": 236.44166666666666, + "list_search_queries_rps": 5.875, + "list_updated_queries_rps": 1.8583333333333334, + "other_queries_rps": 6.333333333333334, + "/v0.1/servers_p95_seconds": 0.9975818507998206, + "/v0/servers_p95_seconds": 1.9143534482758606, + "sampled_pending_acquisitions": 7.0, + "list_acquire_p95_seconds": 0.5624999999999994, + "list_page_acquire_p95_seconds": 0.3136859418282536, + "list_search_acquire_p95_seconds": 0.23608695652173833, + "list_updated_acquire_p95_seconds": 0.08477272727272712, + "other_acquire_p95_seconds": 0.24412650602409644, + "sampled_inflight": 66.0, + "node_cpu_gc71": 0.9170841431325243, + "node_cpu_apu6": 0.9999623830405956 + }, + { + "at": "2026-10-28T18:02:00+00:00", + "origin_arrivals_rps": 259.3833333333333, + "origin_499_rps": 0.041666666666666664, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.44166666666666665, + "list_page_queries_rps": 246.8, + "list_search_queries_rps": 3.2333333333333334, + "list_updated_queries_rps": 2.216666666666667, + "other_queries_rps": 6.433333333333334, + "/v0.1/servers_p95_seconds": 0.9385970090515541, + "/v0/servers_p95_seconds": 1.8252678571428578, + "sampled_pending_acquisitions": 29.0, + "list_acquire_p95_seconds": 0.22999999999999962, + "list_page_acquire_p95_seconds": 0.23357019064124768, + "list_search_acquire_p95_seconds": 0.19238636363636363, + "list_updated_acquire_p95_seconds": 0.0946666666666666, + "other_acquire_p95_seconds": 0.18065068493150666, + "sampled_inflight": 112.0, + "node_cpu_gc71": 0.8719731041712899, + "node_cpu_apu6": 1.0003784062258394 + }, + { + "at": "2026-10-28T18:03:00+00:00", + "origin_arrivals_rps": 264.37499999999994, + "origin_499_rps": 0.025, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.4, + "list_page_queries_rps": 253.88333333333333, + "list_search_queries_rps": 2.2666666666666666, + "list_updated_queries_rps": 1.25, + "other_queries_rps": 6.324999999999999, + "/v0.1/servers_p95_seconds": 1.234582590990072, + "/v0/servers_p95_seconds": 1.9526642335766402, + "sampled_pending_acquisitions": 3.0, + "list_acquire_p95_seconds": 0.41249999999999964, + "list_page_acquire_p95_seconds": 0.4300635686102583, + "list_search_acquire_p95_seconds": 0.2312500000000004, + "list_updated_acquire_p95_seconds": 0.34545454545454507, + "other_acquire_p95_seconds": 0.4758928571428574, + "sampled_inflight": 104.0, + "node_cpu_gc71": 0.8847533887235234, + "node_cpu_apu6": 0.9994104314348078 + }, + { + "at": "2026-10-28T18:04:00+00:00", + "origin_arrivals_rps": 263.97499999999997, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.058333333333333334, + "list_queries_rps": 0.45833333333333337, + "list_page_queries_rps": 255.18333333333334, + "list_search_queries_rps": 2.1333333333333337, + "list_updated_queries_rps": 0.8333333333333333, + "other_queries_rps": 6.033333333333333, + "/v0.1/servers_p95_seconds": 1.488802615447487, + "/v0/servers_p95_seconds": 1.9973489932885913, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.4249999999999999, + "list_page_acquire_p95_seconds": 0.4978874719730939, + "list_search_acquire_p95_seconds": 0.40294117647058814, + "list_updated_acquire_p95_seconds": 0.4404411764705879, + "other_acquire_p95_seconds": 0.5885416666666677, + "sampled_inflight": 81.0, + "node_cpu_gc71": 0.9301071660302114, + "node_cpu_apu6": 1.0000916561485307 + }, + { + "at": "2026-10-28T18:05:00+00:00", + "origin_arrivals_rps": 257.475, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.058333333333333334, + "list_queries_rps": 0.45833333333333337, + "list_page_queries_rps": 247.66666666666669, + "list_search_queries_rps": 2.966666666666667, + "list_updated_queries_rps": 0.75, + "other_queries_rps": 5.941666666666666, + "/v0.1/servers_p95_seconds": 1.7063447275072552, + "/v0/servers_p95_seconds": 1.9851644736842111, + "sampled_pending_acquisitions": 28.0, + "list_acquire_p95_seconds": 0.47019230769230747, + "list_page_acquire_p95_seconds": 0.642104361726421, + "list_search_acquire_p95_seconds": 0.5895348837209295, + "list_updated_acquire_p95_seconds": 0.5407894736842095, + "other_acquire_p95_seconds": 0.7067460317460317, + "sampled_inflight": 97.0, + "node_cpu_gc71": 0.9815379925222563, + "node_cpu_apu6": 0.9995739076807998 + }, + { + "at": "2026-10-28T18:06:00+00:00", + "origin_arrivals_rps": 256.2166666666667, + "origin_499_rps": 0.19166666666666665, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.4083333333333333, + "list_page_queries_rps": 242.21666666666667, + "list_search_queries_rps": 3.05, + "list_updated_queries_rps": 0.65, + "other_queries_rps": 9.066666666666666, + "/v0.1/servers_p95_seconds": 1.7347098736546571, + "/v0/servers_p95_seconds": 1.9566896551724153, + "sampled_pending_acquisitions": 53.0, + "list_acquire_p95_seconds": 0.48303571428571407, + "list_page_acquire_p95_seconds": 0.6540891987605133, + "list_search_acquire_p95_seconds": 0.6465116279069764, + "list_updated_acquire_p95_seconds": 0.702777777777777, + "other_acquire_p95_seconds": 0.6436507936507927, + "sampled_inflight": 163.0, + "node_cpu_gc71": 0.9059935075032021, + "node_cpu_apu6": 0.9973411355308296 + }, + { + "at": "2026-10-28T18:07:00+00:00", + "origin_arrivals_rps": 264.44166666666666, + "origin_499_rps": 0.31666666666666665, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.44166666666666665, + "list_page_queries_rps": 250.71666666666667, + "list_search_queries_rps": 1.9833333333333334, + "list_updated_queries_rps": 0.6583333333333333, + "other_queries_rps": 10.308333333333334, + "/v0.1/servers_p95_seconds": 1.5910248161764713, + "/v0/servers_p95_seconds": 1.8510815307820296, + "sampled_pending_acquisitions": 1.0, + "list_acquire_p95_seconds": 0.4883928571428572, + "list_page_acquire_p95_seconds": 0.5122174709835058, + "list_search_acquire_p95_seconds": 0.4949519230769227, + "list_updated_acquire_p95_seconds": 0.48472222222222205, + "other_acquire_p95_seconds": 0.5036082474226793, + "sampled_inflight": 122.0, + "node_cpu_gc71": 0.8677776171144311, + "node_cpu_apu6": 1.0010854549926687 + }, + { + "at": "2026-10-28T18:08:00+00:00", + "origin_arrivals_rps": 266.59166666666664, + "origin_499_rps": 0.5416666666666666, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.5083333333333333, + "list_page_queries_rps": 252.0, + "list_search_queries_rps": 1.8583333333333334, + "list_updated_queries_rps": 1.0083333333333333, + "other_queries_rps": 10.725, + "/v0.1/servers_p95_seconds": 1.5981284658040678, + "/v0/servers_p95_seconds": 1.893113522537563, + "sampled_pending_acquisitions": 61.0, + "list_acquire_p95_seconds": 0.5843750000000001, + "list_page_acquire_p95_seconds": 0.4859094589124274, + "list_search_acquire_p95_seconds": 0.49207746478873254, + "list_updated_acquire_p95_seconds": 0.4769230769230769, + "other_acquire_p95_seconds": 0.4686418685121102, + "sampled_inflight": 181.0, + "node_cpu_gc71": 0.8537294173300324, + "node_cpu_apu6": 1.0004928442028662 + }, + { + "at": "2026-10-28T18:09:00+00:00", + "origin_arrivals_rps": 265.28333333333336, + "origin_499_rps": 1.125, + "origin_5xx_rps": 0.016666666666666666, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.5166666666666666, + "list_page_queries_rps": 242.05, + "list_search_queries_rps": 2.4916666666666667, + "list_updated_queries_rps": 1.4833333333333334, + "other_queries_rps": 15.766666666666666, + "/v0.1/servers_p95_seconds": 1.9693680794196247, + "/v0/servers_p95_seconds": 2.1812982005141395, + "sampled_pending_acquisitions": 290.0, + "list_acquire_p95_seconds": 0.8571428571428568, + "list_page_acquire_p95_seconds": 0.8312536162005784, + "list_search_acquire_p95_seconds": 0.9296874999999994, + "list_updated_acquire_p95_seconds": 0.9338709677419363, + "other_acquire_p95_seconds": 0.9045627376425847, + "sampled_inflight": 382.0, + "node_cpu_gc71": 0.8642108937509572, + "node_cpu_apu6": 0.9975471548369872 + }, + { + "at": "2026-10-28T18:10:00+00:00", + "origin_arrivals_rps": 276.0, + "origin_499_rps": 1.2083333333333335, + "origin_5xx_rps": 0.016666666666666666, + "publish_completions_rps": 0.08333333333333333, + "list_queries_rps": 0.5166666666666666, + "list_page_queries_rps": 252.22500000000002, + "list_search_queries_rps": 2.1916666666666664, + "list_updated_queries_rps": 1.725, + "other_queries_rps": 19.016666666666666, + "/v0.1/servers_p95_seconds": 1.9957003768844217, + "/v0/servers_p95_seconds": 2.1659090909090897, + "sampled_pending_acquisitions": 23.0, + "list_acquire_p95_seconds": 0.9326923076923075, + "list_page_acquire_p95_seconds": 0.9063278008298747, + "list_search_acquire_p95_seconds": 1.2475000000000032, + "list_updated_acquire_p95_seconds": 1.3600000000000017, + "other_acquire_p95_seconds": 0.9619318181818172, + "sampled_inflight": 148.0, + "node_cpu_gc71": 0.8741889067578692, + "node_cpu_apu6": 1.0003719362114982 + }, + { + "at": "2026-10-28T18:11:00+00:00", + "origin_arrivals_rps": 287.6416666666666, + "origin_499_rps": 0.5416666666666666, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.11666666666666667, + "list_queries_rps": 0.5166666666666666, + "list_page_queries_rps": 270.2833333333333, + "list_search_queries_rps": 1.4166666666666665, + "list_updated_queries_rps": 1.35, + "other_queries_rps": 16.333333333333336, + "/v0.1/servers_p95_seconds": 1.9741698473282439, + "/v0/servers_p95_seconds": 2.1059191176470575, + "sampled_pending_acquisitions": 4.0, + "list_acquire_p95_seconds": 0.9211538461538458, + "list_page_acquire_p95_seconds": 0.8930772666368914, + "list_search_acquire_p95_seconds": 1.4306451612903228, + "list_updated_acquire_p95_seconds": 1.2849999999999986, + "other_acquire_p95_seconds": 0.9544507575757564, + "sampled_inflight": 104.0, + "node_cpu_gc71": 0.875577121158858, + "node_cpu_apu6": 1.000256129335321 + }, + { + "at": "2026-10-28T18:12:00+00:00", + "origin_arrivals_rps": 284.1916666666666, + "origin_499_rps": 0.049999999999999996, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.075, + "list_queries_rps": 0.45000000000000007, + "list_page_queries_rps": 272.2416666666667, + "list_search_queries_rps": 1.3, + "list_updated_queries_rps": 0.9416666666666667, + "other_queries_rps": 9.716666666666667, + "/v0.1/servers_p95_seconds": 1.9475092862828325, + "/v0/servers_p95_seconds": 2.0373883928571432, + "sampled_pending_acquisitions": 3.0, + "list_acquire_p95_seconds": 0.9250000000000007, + "list_page_acquire_p95_seconds": 0.8823456644674832, + "list_search_acquire_p95_seconds": 1.4862903225806434, + "list_updated_acquire_p95_seconds": 1.236249999999999, + "other_acquire_p95_seconds": 0.9683593749999998, + "sampled_inflight": 98.0, + "node_cpu_gc71": 0.8295930167158075, + "node_cpu_apu6": 0.9996161107034519 + }, + { + "at": "2026-10-28T18:13:00+00:00", + "origin_arrivals_rps": 277.7, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.375, + "list_page_queries_rps": 269.04166666666663, + "list_search_queries_rps": 1.85, + "list_updated_queries_rps": 1.15, + "other_queries_rps": 5.75, + "/v0.1/servers_p95_seconds": 1.8511654306039036, + "/v0/servers_p95_seconds": 1.863235294117647, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.8999999999999997, + "list_page_acquire_p95_seconds": 0.849322455752213, + "list_search_acquire_p95_seconds": 1.4354838709677415, + "list_updated_acquire_p95_seconds": 1.2324999999999986, + "other_acquire_p95_seconds": 1.0355491329479787, + "sampled_inflight": 36.0, + "node_cpu_gc71": 0.8394482436213972, + "node_cpu_apu6": 0.9983855623002454 + }, + { + "at": "2026-10-28T18:14:00+00:00", + "origin_arrivals_rps": 277.7416666666666, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.3833333333333333, + "list_page_queries_rps": 268.98333333333335, + "list_search_queries_rps": 2.591666666666667, + "list_updated_queries_rps": 1.175, + "other_queries_rps": 5.091666666666667, + "/v0.1/servers_p95_seconds": 0.9913928904549687, + "/v0/servers_p95_seconds": 0.9972931562819203, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.1174999999999989, + "list_page_acquire_p95_seconds": 0.27118355293048524, + "list_search_acquire_p95_seconds": 0.34999999999999987, + "list_updated_acquire_p95_seconds": 0.5124999999999986, + "other_acquire_p95_seconds": 0.4388000000000005, + "sampled_inflight": 39.0, + "node_cpu_gc71": 0.8371481576655545, + "node_cpu_apu6": 0.9980533813567793 + }, + { + "at": "2026-10-28T18:15:00+00:00", + "origin_arrivals_rps": 278.3416666666665, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.41666666666666663, + "list_page_queries_rps": 269.2416666666667, + "list_search_queries_rps": 2.2416666666666667, + "list_updated_queries_rps": 1.25, + "other_queries_rps": 4.95, + "/v0.1/servers_p95_seconds": 0.8693558632956423, + "/v0/servers_p95_seconds": 0.8974828375286039, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.07099999999999987, + "list_page_acquire_p95_seconds": 0.09213899999999978, + "list_search_acquire_p95_seconds": 0.07303571428571415, + "list_updated_acquire_p95_seconds": 0.0947727272727272, + "other_acquire_p95_seconds": 0.09204861111111103, + "sampled_inflight": 72.0, + "node_cpu_gc71": 0.8521318850922398, + "node_cpu_apu6": 0.9977024207624102 + }, + { + "at": "2026-10-28T18:16:00+00:00", + "origin_arrivals_rps": 278.2166666666667, + "origin_499_rps": 0.075, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.5, + "list_page_queries_rps": 264.69166666666666, + "list_search_queries_rps": 1.775, + "list_updated_queries_rps": 1.6583333333333332, + "other_queries_rps": 9.358333333333333, + "/v0.1/servers_p95_seconds": 0.8238622618687632, + "/v0/servers_p95_seconds": 0.8819811320754711, + "sampled_pending_acquisitions": 1.0, + "list_acquire_p95_seconds": 0.04950000000000006, + "list_page_acquire_p95_seconds": 0.0766256830601092, + "list_search_acquire_p95_seconds": 0.05714285714285708, + "list_updated_acquire_p95_seconds": 0.06968749999999979, + "other_acquire_p95_seconds": 0.07750000000000004, + "sampled_inflight": 39.0, + "node_cpu_gc71": 0.9274286124336262, + "node_cpu_apu6": 1.0017928615146956 + }, + { + "at": "2026-10-28T18:17:00+00:00", + "origin_arrivals_rps": 278.6083333333333, + "origin_499_rps": 0.18333333333333335, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.09166666666666666, + "list_queries_rps": 0.525, + "list_page_queries_rps": 255.58333333333334, + "list_search_queries_rps": 1.9249999999999998, + "list_updated_queries_rps": 1.1416666666666666, + "other_queries_rps": 19.9, + "/v0.1/servers_p95_seconds": 0.7135900216919739, + "/v0/servers_p95_seconds": 0.8339285714285716, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.03500000000000012, + "list_page_acquire_p95_seconds": 0.04747448979591824, + "list_search_acquire_p95_seconds": 0.043399999999999904, + "list_updated_acquire_p95_seconds": 0.0542857142857134, + "other_acquire_p95_seconds": 0.045037735849056376, + "sampled_inflight": 30.0, + "node_cpu_gc71": 0.873367496063535, + "node_cpu_apu6": 0.9977367941321669 + }, + { + "at": "2026-10-28T18:18:00+00:00", + "origin_arrivals_rps": 278.4833333333333, + "origin_499_rps": 0.19166666666666665, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.08333333333333333, + "list_queries_rps": 0.5166666666666667, + "list_page_queries_rps": 254.00833333333333, + "list_search_queries_rps": 2.1166666666666667, + "list_updated_queries_rps": 0.6666666666666667, + "other_queries_rps": 21.21666666666667, + "/v0.1/servers_p95_seconds": 0.5259418324291733, + "/v0/servers_p95_seconds": 0.7577427821522307, + "sampled_pending_acquisitions": 6.0, + "list_acquire_p95_seconds": 0.026000000000000224, + "list_page_acquire_p95_seconds": 0.01368135184067586, + "list_search_acquire_p95_seconds": 0.05799999999999997, + "list_updated_acquire_p95_seconds": 0.0010999999999999866, + "other_acquire_p95_seconds": 0.04570588235294115, + "sampled_inflight": 76.0, + "node_cpu_gc71": 0.9005961564881242, + "node_cpu_apu6": 0.9984610742309693 + }, + { + "at": "2026-10-28T18:19:00+00:00", + "origin_arrivals_rps": 280.1, + "origin_499_rps": 0.3416666666666667, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.5333333333333333, + "list_page_queries_rps": 260.45833333333337, + "list_search_queries_rps": 1.8416666666666668, + "list_updated_queries_rps": 0.925, + "other_queries_rps": 16.25, + "/v0.1/servers_p95_seconds": 0.4915099557522124, + "/v0/servers_p95_seconds": 0.6181734317343149, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00359999999999994, + "list_page_acquire_p95_seconds": 0.007980327868852253, + "list_search_acquire_p95_seconds": 0.05999999999999996, + "list_updated_acquire_p95_seconds": 0.0009941176470588233, + "other_acquire_p95_seconds": 0.03846774193548389, + "sampled_inflight": 41.0, + "node_cpu_gc71": 0.9238628656588844, + "node_cpu_apu6": 1.000003670186076 + }, + { + "at": "2026-10-28T18:20:00+00:00", + "origin_arrivals_rps": 287.8416666666666, + "origin_499_rps": 0.35, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.575, + "list_page_queries_rps": 271.55, + "list_search_queries_rps": 1.4416666666666667, + "list_updated_queries_rps": 0.7416666666666667, + "other_queries_rps": 13.883333333333333, + "/v0.1/servers_p95_seconds": 0.4824619168517932, + "/v0/servers_p95_seconds": 0.5328740157480317, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.013333333333332963, + "list_page_acquire_p95_seconds": 0.005290697674418625, + "list_search_acquire_p95_seconds": 0.06416666666666676, + "list_updated_acquire_p95_seconds": 0.0009928571428571428, + "other_acquire_p95_seconds": 0.03905109489051096, + "sampled_inflight": 36.0, + "node_cpu_gc71": 0.930527372386617, + "node_cpu_apu6": 0.9972668942347809 + }, + { + "at": "2026-10-28T18:21:00+00:00", + "origin_arrivals_rps": 294.67499999999995, + "origin_499_rps": 0.2, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.06666666666666668, + "list_queries_rps": 0.5166666666666667, + "list_page_queries_rps": 281.75, + "list_search_queries_rps": 1.75, + "list_updated_queries_rps": 0.35833333333333334, + "other_queries_rps": 10.2, + "/v0.1/servers_p95_seconds": 0.4760047804532578, + "/v0/servers_p95_seconds": 0.4893998951781971, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009961805555555557, + "list_page_acquire_p95_seconds": 0.006809049079754667, + "list_search_acquire_p95_seconds": 0.06342105263157885, + "list_updated_acquire_p95_seconds": 0.0009768361581920905, + "other_acquire_p95_seconds": 0.040596774193548256, + "sampled_inflight": 53.0, + "node_cpu_gc71": 0.9037680286931087, + "node_cpu_apu6": 0.9987551016005455 + }, + { + "at": "2026-10-28T18:22:00+00:00", + "origin_arrivals_rps": 298.3083333333333, + "origin_499_rps": 0.19166666666666665, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.08333333333333333, + "list_queries_rps": 0.4666666666666667, + "list_page_queries_rps": 280.79999999999995, + "list_search_queries_rps": 2.841666666666667, + "list_updated_queries_rps": 0.39166666666666666, + "other_queries_rps": 13.625, + "/v0.1/servers_p95_seconds": 0.4787658397287166, + "/v0/servers_p95_seconds": 0.4996839887640452, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009965034965034964, + "list_page_acquire_p95_seconds": 0.0044588652482269245, + "list_search_acquire_p95_seconds": 0.06119047619047601, + "list_updated_acquire_p95_seconds": 0.0009705405405405405, + "other_acquire_p95_seconds": 0.0355221238938051, + "sampled_inflight": 43.0, + "node_cpu_gc71": 0.8652500385086266, + "node_cpu_apu6": 0.9987022793359452 + }, + { + "at": "2026-10-28T18:23:00+00:00", + "origin_arrivals_rps": 304.6666666666665, + "origin_499_rps": 0.275, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.08333333333333334, + "list_queries_rps": 0.5, + "list_page_queries_rps": 284.05, + "list_search_queries_rps": 2.7416666666666667, + "list_updated_queries_rps": 0.55, + "other_queries_rps": 17.166666666666664, + "/v0.1/servers_p95_seconds": 0.45853107625099293, + "/v0/servers_p95_seconds": 0.470430232558139, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009965034965034964, + "list_page_acquire_p95_seconds": 0.0013126728110599492, + "list_search_acquire_p95_seconds": 0.011999999999999775, + "list_updated_acquire_p95_seconds": 0.0009791411042944786, + "other_acquire_p95_seconds": 0.019925925925925916, + "sampled_inflight": 15.0, + "node_cpu_gc71": 0.8581246217361088, + "node_cpu_apu6": 0.9957292107707568 + }, + { + "at": "2026-10-28T18:24:00+00:00", + "origin_arrivals_rps": 308.0583333333334, + "origin_499_rps": 0.24166666666666664, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.5, + "list_page_queries_rps": 291.725, + "list_search_queries_rps": 2.05, + "list_updated_queries_rps": 0.625, + "other_queries_rps": 12.6, + "/v0.1/servers_p95_seconds": 0.4464232632390178, + "/v0/servers_p95_seconds": 0.4646795475966067, + "sampled_pending_acquisitions": 26.0, + "list_acquire_p95_seconds": 0.0009907142857142857, + "list_page_acquire_p95_seconds": 0.001238686987104342, + "list_search_acquire_p95_seconds": 0.006666666666666666, + "list_updated_acquire_p95_seconds": 0.000989041095890411, + "other_acquire_p95_seconds": 0.02399999999999996, + "sampled_inflight": 105.0, + "node_cpu_gc71": 0.8589548749179812, + "node_cpu_apu6": 0.9962493195217879 + }, + { + "at": "2026-10-28T18:25:00+00:00", + "origin_arrivals_rps": 302.54999999999995, + "origin_499_rps": 0.19166666666666665, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.5, + "list_page_queries_rps": 287.35, + "list_search_queries_rps": 2.4749999999999996, + "list_updated_queries_rps": 0.6833333333333333, + "other_queries_rps": 11.133333333333333, + "/v0.1/servers_p95_seconds": 0.45740960451977375, + "/v0/servers_p95_seconds": 0.4865822490706318, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.02519999999999998, + "list_page_acquire_p95_seconds": 0.0041920481927710095, + "list_search_acquire_p95_seconds": 0.024333333333333262, + "list_updated_acquire_p95_seconds": 0.019499999999999892, + "other_acquire_p95_seconds": 0.03245312499999967, + "sampled_inflight": 15.0, + "node_cpu_gc71": 0.8770617908148173, + "node_cpu_apu6": 0.9944423665394424 + }, + { + "at": "2026-10-28T18:26:00+00:00", + "origin_arrivals_rps": 301.9833333333333, + "origin_499_rps": 0.19999999999999998, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.43333333333333335, + "list_page_queries_rps": 286.4083333333333, + "list_search_queries_rps": 3.475, + "list_updated_queries_rps": 0.6666666666666666, + "other_queries_rps": 11.325, + "/v0.1/servers_p95_seconds": 0.45829617656894406, + "/v0/servers_p95_seconds": 0.4882423580786024, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.024, + "list_page_acquire_p95_seconds": 0.003082871536523975, + "list_search_acquire_p95_seconds": 0.004244444444444415, + "list_updated_acquire_p95_seconds": 0.0088749999999999, + "other_acquire_p95_seconds": 0.03110743801652905, + "sampled_inflight": 31.0, + "node_cpu_gc71": 0.8790382323692029, + "node_cpu_apu6": 0.9915229241400084 + }, + { + "at": "2026-10-28T18:27:00+00:00", + "origin_arrivals_rps": 314.1416666666667, + "origin_499_rps": 0.058333333333333334, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.375, + "list_page_queries_rps": 300.65, + "list_search_queries_rps": 3.4916666666666667, + "list_updated_queries_rps": 0.6166666666666667, + "other_queries_rps": 8.841666666666667, + "/v0.1/servers_p95_seconds": 0.43632326820603873, + "/v0/servers_p95_seconds": 0.45189888164026093, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.025666666666666706, + "list_page_acquire_p95_seconds": 0.0016910485933503787, + "list_search_acquire_p95_seconds": 0.0029714285714284605, + "list_updated_acquire_p95_seconds": 0.009125000000000005, + "other_acquire_p95_seconds": 0.03005172413793091, + "sampled_inflight": 30.0, + "node_cpu_gc71": 0.8785136586326795, + "node_cpu_apu6": 0.9913608708089062 + }, + { + "at": "2026-10-28T18:28:00+00:00", + "origin_arrivals_rps": 333.0499999999999, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.4666666666666667, + "list_page_queries_rps": 321.10833333333335, + "list_search_queries_rps": 2.5583333333333336, + "list_updated_queries_rps": 0.5833333333333334, + "other_queries_rps": 8.408333333333333, + "/v0.1/servers_p95_seconds": 0.4238106276747499, + "/v0/servers_p95_seconds": 0.42915846456692874, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.027999999999999817, + "list_page_acquire_p95_seconds": 0.0009992719349215574, + "list_search_acquire_p95_seconds": 0.0016857142857142206, + "list_updated_acquire_p95_seconds": 0.004133333333333341, + "other_acquire_p95_seconds": 0.028879120879120687, + "sampled_inflight": 7.0, + "node_cpu_gc71": 0.7966253685990523, + "node_cpu_apu6": 0.9802873560782852 + }, + { + "at": "2026-10-28T18:29:00+00:00", + "origin_arrivals_rps": 334.1833333333334, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.4083333333333333, + "list_page_queries_rps": 323.275, + "list_search_queries_rps": 1.8916666666666666, + "list_updated_queries_rps": 0.5333333333333333, + "other_queries_rps": 8.108333333333334, + "/v0.1/servers_p95_seconds": 0.39946347416259864, + "/v0/servers_p95_seconds": 0.3962443310657598, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.02519999999999993, + "list_page_acquire_p95_seconds": 0.0009933364505166162, + "list_search_acquire_p95_seconds": 0.0009985583224115335, + "list_updated_acquire_p95_seconds": 0.005249999999999973, + "other_acquire_p95_seconds": 0.01605797101449273, + "sampled_inflight": 15.0, + "node_cpu_gc71": 0.7734848230325345, + "node_cpu_apu6": 0.9544046597346703 + }, + { + "at": "2026-10-28T18:30:00+00:00", + "origin_arrivals_rps": 318.65, + "origin_499_rps": 0.058333333333333334, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.058333333333333334, + "list_queries_rps": 0.3833333333333333, + "list_page_queries_rps": 305.0083333333333, + "list_search_queries_rps": 2.2416666666666667, + "list_updated_queries_rps": 0.4, + "other_queries_rps": 10.491666666666667, + "/v0.1/servers_p95_seconds": 0.31558131407269363, + "/v0/servers_p95_seconds": 0.3147459499263621, + "sampled_pending_acquisitions": 2.0, + "list_acquire_p95_seconds": 0.000991304347826087, + "list_page_acquire_p95_seconds": 0.0009833428312828935, + "list_search_acquire_p95_seconds": 0.0009819029850746271, + "list_updated_acquire_p95_seconds": 0.0023999999999999777, + "other_acquire_p95_seconds": 0.00313846153846149, + "sampled_inflight": 19.0, + "node_cpu_gc71": 0.8214326251099313, + "node_cpu_apu6": 0.9312813295301263 + }, + { + "at": "2026-10-28T18:31:00+00:00", + "origin_arrivals_rps": 301.50833333333327, + "origin_499_rps": 0.1, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.55, + "list_page_queries_rps": 283.76666666666665, + "list_search_queries_rps": 3.583333333333333, + "list_updated_queries_rps": 0.625, + "other_queries_rps": 12.833333333333334, + "/v0.1/servers_p95_seconds": 0.24374360286591584, + "/v0/servers_p95_seconds": 0.24087367624810893, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.000978787878787879, + "list_page_acquire_p95_seconds": 0.0009819839636297837, + "list_search_acquire_p95_seconds": 0.0009822236180904523, + "list_updated_acquire_p95_seconds": 0.0009903030303030302, + "other_acquire_p95_seconds": 0.0017409090909090677, + "sampled_inflight": 4.0, + "node_cpu_gc71": 0.7749536529193088, + "node_cpu_apu6": 0.8853493901289766 + }, + { + "at": "2026-10-28T18:32:00+00:00", + "origin_arrivals_rps": 289.9749999999999, + "origin_499_rps": 0.05, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.4666666666666667, + "list_page_queries_rps": 275.6166666666667, + "list_search_queries_rps": 3.6166666666666663, + "list_updated_queries_rps": 0.9666666666666666, + "other_queries_rps": 9.091666666666667, + "/v0.1/servers_p95_seconds": 0.21427456183658358, + "/v0/servers_p95_seconds": 0.2094498444790045, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009647286821705426, + "list_page_acquire_p95_seconds": 0.0009770730755546376, + "list_search_acquire_p95_seconds": 0.0009838868388683888, + "list_updated_acquire_p95_seconds": 0.0009842783505154638, + "other_acquire_p95_seconds": 0.0009977987421383645, + "sampled_inflight": 6.0, + "node_cpu_gc71": 0.7777605629904428, + "node_cpu_apu6": 0.8941615871054334 + }, + { + "at": "2026-10-28T18:33:00+00:00", + "origin_arrivals_rps": 307.23333333333335, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.4, + "list_page_queries_rps": 295.81666666666666, + "list_search_queries_rps": 2.341666666666667, + "list_updated_queries_rps": 1.025, + "other_queries_rps": 7.375, + "/v0.1/servers_p95_seconds": 0.17745869686281796, + "/v0/servers_p95_seconds": 0.16536284953395436, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.000974576949860724, + "list_search_acquire_p95_seconds": 0.0009871093750000001, + "list_updated_acquire_p95_seconds": 0.0009850230414746542, + "other_acquire_p95_seconds": 0.0009964692901802135, + "sampled_inflight": 9.0, + "node_cpu_gc71": 0.7474708862452341, + "node_cpu_apu6": 0.9272823517455738 + }, + { + "at": "2026-10-28T18:34:00+00:00", + "origin_arrivals_rps": 332.1583333333333, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.45833333333333337, + "list_page_queries_rps": 316.6, + "list_search_queries_rps": 2.375, + "list_updated_queries_rps": 0.75, + "other_queries_rps": 11.733333333333334, + "/v0.1/servers_p95_seconds": 0.16756019007391715, + "/v0/servers_p95_seconds": 0.1594493392070485, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009743190346977656, + "list_search_acquire_p95_seconds": 0.0009941347270615562, + "list_updated_acquire_p95_seconds": 0.0009802272727272726, + "other_acquire_p95_seconds": 0.000995541706615532, + "sampled_inflight": 7.0, + "node_cpu_gc71": 0.7316727331912262, + "node_cpu_apu6": 0.9268798320275285 + }, + { + "at": "2026-10-28T18:35:00+00:00", + "origin_arrivals_rps": 326.875, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.4, + "list_page_queries_rps": 307.7, + "list_search_queries_rps": 2.5666666666666664, + "list_updated_queries_rps": 0.6416666666666666, + "other_queries_rps": 15.425, + "/v0.1/servers_p95_seconds": 0.1253915929203541, + "/v0/servers_p95_seconds": 0.1155147058823526, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009733634441890282, + "list_search_acquire_p95_seconds": 0.002933333333333281, + "list_updated_acquire_p95_seconds": 0.0009769230769230768, + "other_acquire_p95_seconds": 0.0009948932926829269, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.6951592313246996, + "node_cpu_apu6": 0.885483784401246 + }, + { + "at": "2026-10-28T18:36:00+00:00", + "origin_arrivals_rps": 310.75833333333327, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.049999999999999996, + "list_queries_rps": 0.4083333333333333, + "list_page_queries_rps": 292.08333333333337, + "list_search_queries_rps": 2.041666666666667, + "list_updated_queries_rps": 0.6833333333333333, + "other_queries_rps": 15.316666666666666, + "/v0.1/servers_p95_seconds": 0.09882009528782142, + "/v0/servers_p95_seconds": 0.0990328233657858, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009499999999999999, + "list_page_acquire_p95_seconds": 0.0009703511245537036, + "list_search_acquire_p95_seconds": 0.009666666666666648, + "list_updated_acquire_p95_seconds": 0.0009792951541850222, + "other_acquire_p95_seconds": 0.0009936781609195403, + "sampled_inflight": 6.0, + "node_cpu_gc71": 0.711020009535908, + "node_cpu_apu6": 0.870956100349819 + }, + { + "at": "2026-10-28T18:37:00+00:00", + "origin_arrivals_rps": 310.15833333333325, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.38333333333333336, + "list_page_queries_rps": 292.56666666666666, + "list_search_queries_rps": 1.5666666666666667, + "list_updated_queries_rps": 0.5333333333333333, + "other_queries_rps": 14.875, + "/v0.1/servers_p95_seconds": 0.09881932044355984, + "/v0/servers_p95_seconds": 0.0990103350515464, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009705122143420017, + "list_search_acquire_p95_seconds": 0.011933333333333386, + "list_updated_acquire_p95_seconds": 0.0009790816326530612, + "other_acquire_p95_seconds": 0.0009984446710861804, + "sampled_inflight": 7.0, + "node_cpu_gc71": 0.7903318489714972, + "node_cpu_apu6": 0.8483030832721852 + }, + { + "at": "2026-10-28T18:38:00+00:00", + "origin_arrivals_rps": 312.95833333333337, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.4, + "list_page_queries_rps": 296.825, + "list_search_queries_rps": 1.8583333333333334, + "list_updated_queries_rps": 0.4, + "other_queries_rps": 13.483333333333334, + "/v0.1/servers_p95_seconds": 0.0985635611805988, + "/v0/servers_p95_seconds": 0.09974984831937872, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009695762933039818, + "list_search_acquire_p95_seconds": 0.008375000000000014, + "list_updated_acquire_p95_seconds": 0.0009808441558441558, + "other_acquire_p95_seconds": 0.0009972787447903898, + "sampled_inflight": 5.0, + "node_cpu_gc71": 0.7682739735816236, + "node_cpu_apu6": 0.864312540859343 + }, + { + "at": "2026-10-28T18:39:00+00:00", + "origin_arrivals_rps": 320.8833333333333, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.43333333333333335, + "list_page_queries_rps": 307.2166666666667, + "list_search_queries_rps": 2.558333333333333, + "list_updated_queries_rps": 0.41666666666666663, + "other_queries_rps": 10.433333333333334, + "/v0.1/servers_p95_seconds": 0.09737714680666874, + "/v0/servers_p95_seconds": 0.09895176452458007, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009582608695652173, + "list_page_acquire_p95_seconds": 0.0009675775966904204, + "list_search_acquire_p95_seconds": 0.007116666666666676, + "list_updated_acquire_p95_seconds": 0.0009740506329113925, + "other_acquire_p95_seconds": 0.0009931016184664366, + "sampled_inflight": 4.0, + "node_cpu_gc71": 0.6784719093566915, + "node_cpu_apu6": 0.8715613933173396 + }, + { + "at": "2026-10-28T18:40:00+00:00", + "origin_arrivals_rps": 320.9916666666666, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.06666666666666667, + "list_queries_rps": 0.41666666666666663, + "list_page_queries_rps": 309.23333333333335, + "list_search_queries_rps": 1.9166666666666667, + "list_updated_queries_rps": 0.5583333333333333, + "other_queries_rps": 8.908333333333333, + "/v0.1/servers_p95_seconds": 0.0968573107727633, + "/v0/servers_p95_seconds": 0.09850004701899569, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009577868852459017, + "list_page_acquire_p95_seconds": 0.0009663491812713708, + "list_search_acquire_p95_seconds": 0.00607142857142855, + "list_updated_acquire_p95_seconds": 0.0009695205479452055, + "other_acquire_p95_seconds": 0.0009860509931811443, + "sampled_inflight": 5.0, + "node_cpu_gc71": 0.7374679050238532, + "node_cpu_apu6": 0.8687238466030976 + }, + { + "at": "2026-10-28T18:41:00+00:00", + "origin_arrivals_rps": 316.7416666666666, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.08333333333333333, + "list_queries_rps": 0.5666666666666667, + "list_page_queries_rps": 305.3666666666667, + "list_search_queries_rps": 2.0250000000000004, + "list_updated_queries_rps": 1.0083333333333333, + "other_queries_rps": 7.833333333333334, + "/v0.1/servers_p95_seconds": 0.0969146038102172, + "/v0/servers_p95_seconds": 0.09826395048041392, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.000957142857142857, + "list_page_acquire_p95_seconds": 0.0009661758474576271, + "list_search_acquire_p95_seconds": 0.001724999999999961, + "list_updated_acquire_p95_seconds": 0.0009692893401015229, + "other_acquire_p95_seconds": 0.0009863760683760682, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.7064621220447104, + "node_cpu_apu6": 0.8849317444580568 + }, + { + "at": "2026-10-28T18:42:00+00:00", + "origin_arrivals_rps": 321.7249999999999, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.06666666666666667, + "list_queries_rps": 0.48333333333333334, + "list_page_queries_rps": 310.2083333333333, + "list_search_queries_rps": 3.45, + "list_updated_queries_rps": 1.3833333333333333, + "other_queries_rps": 6.375, + "/v0.1/servers_p95_seconds": 0.09658147263162123, + "/v0/servers_p95_seconds": 0.09817598738798541, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009722656249999999, + "list_page_acquire_p95_seconds": 0.0009655971086673865, + "list_search_acquire_p95_seconds": 0.0009970052083333333, + "list_updated_acquire_p95_seconds": 0.0009693089430894308, + "other_acquire_p95_seconds": 0.0009775420168067228, + "sampled_inflight": 4.0, + "node_cpu_gc71": 0.6678881494811018, + "node_cpu_apu6": 0.8427215684848003 + }, + { + "at": "2026-10-28T18:43:00+00:00", + "origin_arrivals_rps": 308.89166666666665, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.35, + "list_page_queries_rps": 297.125, + "list_search_queries_rps": 3.466666666666667, + "list_updated_queries_rps": 0.975, + "other_queries_rps": 6.866666666666667, + "/v0.1/servers_p95_seconds": 0.09633950676387311, + "/v0/servers_p95_seconds": 0.09773442331478045, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009733606557377048, + "list_page_acquire_p95_seconds": 0.0009654688317907936, + "list_search_acquire_p95_seconds": 0.003228571428571424, + "list_updated_acquire_p95_seconds": 0.0009642857142857142, + "other_acquire_p95_seconds": 0.0009695205479452055, + "sampled_inflight": 5.0, + "node_cpu_gc71": 0.6389033578860108, + "node_cpu_apu6": 0.824688447726415 + }, + { + "at": "2026-10-28T18:44:00+00:00", + "origin_arrivals_rps": 296.1166666666667, + "origin_499_rps": 0.016666666666666666, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.43333333333333335, + "list_page_queries_rps": 281.7916666666667, + "list_search_queries_rps": 2.5166666666666666, + "list_updated_queries_rps": 0.5333333333333333, + "other_queries_rps": 10.75, + "/v0.1/servers_p95_seconds": 0.09625799635104126, + "/v0/servers_p95_seconds": 0.09688928227691462, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009649606299212599, + "list_page_acquire_p95_seconds": 0.0009654666013466553, + "list_search_acquire_p95_seconds": 0.0012714285714285594, + "list_updated_acquire_p95_seconds": 0.0009608778625954199, + "other_acquire_p95_seconds": 0.0009726283367556468, + "sampled_inflight": 6.0, + "node_cpu_gc71": 0.6135700219684319, + "node_cpu_apu6": 0.824758788421847 + }, + { + "at": "2026-10-28T18:45:00+00:00", + "origin_arrivals_rps": 295.13333333333327, + "origin_499_rps": 0.05, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.4666666666666667, + "list_page_queries_rps": 275.43333333333334, + "list_search_queries_rps": 2.9333333333333336, + "list_updated_queries_rps": 0.5666666666666667, + "other_queries_rps": 15.475000000000001, + "/v0.1/servers_p95_seconds": 0.09618063226907363, + "/v0/servers_p95_seconds": 0.09678312233642516, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009649606299212599, + "list_page_acquire_p95_seconds": 0.0009654810057233442, + "list_search_acquire_p95_seconds": 0.0009985244040862655, + "list_updated_acquire_p95_seconds": 0.0009570631970260223, + "other_acquire_p95_seconds": 0.000978474114441417, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.5919386680254926, + "node_cpu_apu6": 0.8243106256173632 + }, + { + "at": "2026-10-28T18:46:00+00:00", + "origin_arrivals_rps": 289.14166666666665, + "origin_499_rps": 0.049999999999999996, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.225, + "list_queries_rps": 0.6083333333333334, + "list_page_queries_rps": 266.39166666666665, + "list_search_queries_rps": 3.8166666666666664, + "list_updated_queries_rps": 0.6416666666666666, + "other_queries_rps": 18.383333333333333, + "/v0.1/servers_p95_seconds": 0.09566274907575663, + "/v0/servers_p95_seconds": 0.0968654051172708, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009674311926605504, + "list_page_acquire_p95_seconds": 0.0009641561201700246, + "list_search_acquire_p95_seconds": 0.0009889344262295082, + "list_updated_acquire_p95_seconds": 0.000958675799086758, + "other_acquire_p95_seconds": 0.0009789683670561413, + "sampled_inflight": 3.0, + "node_cpu_gc71": 0.5687202070352214, + "node_cpu_apu6": 0.8101839359638688 + }, + { + "at": "2026-10-28T18:47:00+00:00", + "origin_arrivals_rps": 288.79999999999995, + "origin_499_rps": 0.016666666666666666, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 3.5583333333333336, + "list_queries_rps": 3.9583333333333335, + "list_page_queries_rps": 265.34166666666664, + "list_search_queries_rps": 4.016666666666667, + "list_updated_queries_rps": 0.6166666666666667, + "other_queries_rps": 42.50833333333333, + "/v0.1/servers_p95_seconds": 0.09561063638315087, + "/v0/servers_p95_seconds": 0.09724577620828141, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009636520315929125, + "list_search_acquire_p95_seconds": 0.0009950474183350899, + "list_updated_acquire_p95_seconds": 0.0009553370786516853, + "other_acquire_p95_seconds": 0.0009815777153558052, + "sampled_inflight": 7.0, + "node_cpu_gc71": 0.6107268864000313, + "node_cpu_apu6": 0.8156504052911866 + }, + { + "at": "2026-10-28T18:48:00+00:00", + "origin_arrivals_rps": 301.8916666666666, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 3.7, + "list_queries_rps": 4.133333333333333, + "list_page_queries_rps": 281.94166666666666, + "list_search_queries_rps": 2.75, + "list_updated_queries_rps": 0.6083333333333334, + "other_queries_rps": 41.74166666666667, + "/v0.1/servers_p95_seconds": 0.09560273532650407, + "/v0/servers_p95_seconds": 0.09662644030399609, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009653225806451614, + "list_page_acquire_p95_seconds": 0.0009628302857983233, + "list_search_acquire_p95_seconds": 0.0009838530066815143, + "list_updated_acquire_p95_seconds": 0.0009553977272727273, + "other_acquire_p95_seconds": 0.0009853089949417198, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.6096558430304867, + "node_cpu_apu6": 0.8309602851688396 + }, + { + "at": "2026-10-28T18:49:00+00:00", + "origin_arrivals_rps": 306.31666666666666, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.35000000000000003, + "list_queries_rps": 0.775, + "list_page_queries_rps": 293.04999999999995, + "list_search_queries_rps": 1.8083333333333333, + "list_updated_queries_rps": 0.675, + "other_queries_rps": 12.633333333333333, + "/v0.1/servers_p95_seconds": 0.09552563701536952, + "/v0/servers_p95_seconds": 0.09670278365045805, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009886178861788616, + "list_page_acquire_p95_seconds": 0.000962242834695543, + "list_search_acquire_p95_seconds": 0.000985787671232877, + "list_updated_acquire_p95_seconds": 0.0009597938144329895, + "other_acquire_p95_seconds": 0.0009851101634683723, + "sampled_inflight": 3.0, + "node_cpu_gc71": 0.5439708611479243, + "node_cpu_apu6": 0.7470540050470542 + }, + { + "at": "2026-10-28T18:50:00+00:00", + "origin_arrivals_rps": 274.15000000000003, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.008333333333333333, + "list_queries_rps": 0.4083333333333333, + "list_page_queries_rps": 260.06666666666666, + "list_search_queries_rps": 2.308333333333333, + "list_updated_queries_rps": 0.975, + "other_queries_rps": 10.308333333333334, + "/v0.1/servers_p95_seconds": 0.09509041268601061, + "/v0/servers_p95_seconds": 0.0963652941176471, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009905982905982903, + "list_page_acquire_p95_seconds": 0.0009607291601525859, + "list_search_acquire_p95_seconds": 0.0009868932038834951, + "list_updated_acquire_p95_seconds": 0.0009627802690582957, + "other_acquire_p95_seconds": 0.0009842019133937563, + "sampled_inflight": 3.0, + "node_cpu_gc71": 0.510148069151425, + "node_cpu_apu6": 0.692511598077787 + }, + { + "at": "2026-10-28T18:51:00+00:00", + "origin_arrivals_rps": 246.54166666666666, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.375, + "list_page_queries_rps": 229.14166666666665, + "list_search_queries_rps": 2.6666666666666665, + "list_updated_queries_rps": 1.5, + "other_queries_rps": 12.691666666666666, + "/v0.1/servers_p95_seconds": 0.09491845827967825, + "/v0/servers_p95_seconds": 0.09610602723267922, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009895833333333332, + "list_page_acquire_p95_seconds": 0.0009610034837830542, + "list_search_acquire_p95_seconds": 0.0009978882833787468, + "list_updated_acquire_p95_seconds": 0.0009563973063973064, + "other_acquire_p95_seconds": 0.0009860371267150926, + "sampled_inflight": 3.0, + "node_cpu_gc71": 0.5325677972463988, + "node_cpu_apu6": 0.6760676787491926 + }, + { + "at": "2026-10-28T18:52:00+00:00", + "origin_arrivals_rps": 243.70833333333337, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.3416666666666667, + "list_page_queries_rps": 229.41666666666666, + "list_search_queries_rps": 2.1416666666666666, + "list_updated_queries_rps": 1.7833333333333332, + "other_queries_rps": 9.991666666666667, + "/v0.1/servers_p95_seconds": 0.09464180838441764, + "/v0/servers_p95_seconds": 0.0951797892580126, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009939814814814813, + "list_page_acquire_p95_seconds": 0.0009605793999920739, + "list_search_acquire_p95_seconds": 0.0009876237623762374, + "list_updated_acquire_p95_seconds": 0.0009552197802197803, + "other_acquire_p95_seconds": 0.0009834747145187602, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.5175185351081988, + "node_cpu_apu6": 0.6509219090466407 + }, + { + "at": "2026-10-28T18:53:00+00:00", + "origin_arrivals_rps": 235.77500000000003, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.3333333333333333, + "list_page_queries_rps": 224.04166666666669, + "list_search_queries_rps": 1.6166666666666667, + "list_updated_queries_rps": 1.3666666666666667, + "other_queries_rps": 8.408333333333333, + "/v0.1/servers_p95_seconds": 0.0944560050541359, + "/v0/servers_p95_seconds": 0.09502636054421768, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009768867924528303, + "list_page_acquire_p95_seconds": 0.0009605507949035968, + "list_search_acquire_p95_seconds": 0.0009889344262295082, + "list_updated_acquire_p95_seconds": 0.0009548969072164947, + "other_acquire_p95_seconds": 0.000979614711033275, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.5284526528286126, + "node_cpu_apu6": 0.6367984576662518 + }, + { + "at": "2026-10-28T18:54:00+00:00", + "origin_arrivals_rps": 236.45000000000005, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.39166666666666666, + "list_page_queries_rps": 219.625, + "list_search_queries_rps": 2.3666666666666667, + "list_updated_queries_rps": 2.333333333333333, + "other_queries_rps": 11.608333333333334, + "/v0.1/servers_p95_seconds": 0.09420687596074258, + "/v0/servers_p95_seconds": 0.09466570881226054, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009499999999999999, + "list_page_acquire_p95_seconds": 0.0009606798942983832, + "list_search_acquire_p95_seconds": 0.0009957142857142855, + "list_updated_acquire_p95_seconds": 0.0009567736185383245, + "other_acquire_p95_seconds": 0.0009847018420231033, + "sampled_inflight": 4.0, + "node_cpu_gc71": 0.5455785380690941, + "node_cpu_apu6": 0.673798481705065 + }, + { + "at": "2026-10-28T18:55:00+00:00", + "origin_arrivals_rps": 260.05, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.5, + "list_page_queries_rps": 239.40833333333333, + "list_search_queries_rps": 2.916666666666667, + "list_updated_queries_rps": 4.008333333333334, + "other_queries_rps": 13.266666666666666, + "/v0.1/servers_p95_seconds": 0.0942122445404648, + "/v0/servers_p95_seconds": 0.09490195738783996, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009611704303117059, + "list_search_acquire_p95_seconds": 0.0009979940564635958, + "list_updated_acquire_p95_seconds": 0.0009563417890520697, + "other_acquire_p95_seconds": 0.0009837456390738978, + "sampled_inflight": 7.0, + "node_cpu_gc71": 0.5790002270087522, + "node_cpu_apu6": 0.7212871549192641 + }, + { + "at": "2026-10-28T18:56:00+00:00", + "origin_arrivals_rps": 273.7833333333333, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.4916666666666667, + "list_page_queries_rps": 258.1, + "list_search_queries_rps": 2.125, + "list_updated_queries_rps": 3.325, + "other_queries_rps": 9.7, + "/v0.1/servers_p95_seconds": 0.09410173282778073, + "/v0/servers_p95_seconds": 0.09459010114532203, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009581896551724139, + "list_page_acquire_p95_seconds": 0.0009604394122240365, + "list_search_acquire_p95_seconds": 0.0009891268533772652, + "list_updated_acquire_p95_seconds": 0.0009610322580645162, + "other_acquire_p95_seconds": 0.000980216707444949, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.4892253059268114, + "node_cpu_apu6": 0.6610451677540065 + }, + { + "at": "2026-10-28T18:57:00+00:00", + "origin_arrivals_rps": 250.35833333333338, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.35, + "list_page_queries_rps": 239.25, + "list_search_queries_rps": 1.3666666666666667, + "list_updated_queries_rps": 1.6083333333333334, + "other_queries_rps": 7.608333333333333, + "/v0.1/servers_p95_seconds": 0.09408444998812072, + "/v0/servers_p95_seconds": 0.0946217859892225, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009580508474576271, + "list_page_acquire_p95_seconds": 0.0009607474086197489, + "list_search_acquire_p95_seconds": 0.000997013651877133, + "list_updated_acquire_p95_seconds": 0.0009618093922651935, + "other_acquire_p95_seconds": 0.0009830406336088155, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.45398227054805224, + "node_cpu_apu6": 0.5875010730055995 + }, + { + "at": "2026-10-28T18:58:00+00:00", + "origin_arrivals_rps": 227.09166666666673, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.35833333333333334, + "list_page_queries_rps": 213.36666666666667, + "list_search_queries_rps": 1.5750000000000002, + "list_updated_queries_rps": 0.85, + "other_queries_rps": 10.833333333333332, + "/v0.1/servers_p95_seconds": 0.09372158358933706, + "/v0/servers_p95_seconds": 0.09454462383215867, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.000957983193277311, + "list_page_acquire_p95_seconds": 0.0009601608683712257, + "list_search_acquire_p95_seconds": 0.00099275, + "list_updated_acquire_p95_seconds": 0.0009619915848527349, + "other_acquire_p95_seconds": 0.0009889244558258642, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.4400149823445342, + "node_cpu_apu6": 0.5910984235000797 + }, + { + "at": "2026-10-28T18:59:00+00:00", + "origin_arrivals_rps": 236.65833333333333, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.43333333333333335, + "list_page_queries_rps": 220.56666666666666, + "list_search_queries_rps": 2.0833333333333335, + "list_updated_queries_rps": 1.2000000000000002, + "other_queries_rps": 12.291666666666666, + "/v0.1/servers_p95_seconds": 0.09318592906707787, + "/v0/servers_p95_seconds": 0.09426724137931036, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009655737704918032, + "list_page_acquire_p95_seconds": 0.000958860976796086, + "list_search_acquire_p95_seconds": 0.0009821174377224197, + "list_updated_acquire_p95_seconds": 0.0009629032258064516, + "other_acquire_p95_seconds": 0.0009831210191082803, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.44968841865144593, + "node_cpu_apu6": 0.6046100839873058 + }, + { + "at": "2026-10-28T19:00:00+00:00", + "origin_arrivals_rps": 244.67500000000007, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.4166666666666667, + "list_page_queries_rps": 231.60833333333335, + "list_search_queries_rps": 1.8916666666666666, + "list_updated_queries_rps": 1.475, + "other_queries_rps": 9.433333333333334, + "/v0.1/servers_p95_seconds": 0.09268263968696985, + "/v0/servers_p95_seconds": 0.09372701335055987, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009672727272727271, + "list_page_acquire_p95_seconds": 0.0009577048413730618, + "list_search_acquire_p95_seconds": 0.0009711967545638946, + "list_updated_acquire_p95_seconds": 0.0009638349514563107, + "other_acquire_p95_seconds": 0.0009805658284023668, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.4912431370932609, + "node_cpu_apu6": 0.6242204168382159 + }, + { + "at": "2026-10-28T19:01:00+00:00", + "origin_arrivals_rps": 253.1833333333334, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.5333333333333333, + "list_page_queries_rps": 237.95833333333334, + "list_search_queries_rps": 1.7999999999999998, + "list_updated_queries_rps": 2.466666666666667, + "other_queries_rps": 10.516666666666666, + "/v0.1/servers_p95_seconds": 0.09258164532239617, + "/v0/servers_p95_seconds": 0.0937687390734266, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009648437500000001, + "list_page_acquire_p95_seconds": 0.0009572916974770033, + "list_search_acquire_p95_seconds": 0.0009624299065420562, + "list_updated_acquire_p95_seconds": 0.0009616803278688524, + "other_acquire_p95_seconds": 0.0009824601366742597, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.5815088514612095, + "node_cpu_apu6": 0.6584837241173469 + }, + { + "at": "2026-10-28T19:02:00+00:00", + "origin_arrivals_rps": 258.52499999999986, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.6166666666666667, + "list_page_queries_rps": 237.97500000000002, + "list_search_queries_rps": 2.4166666666666665, + "list_updated_queries_rps": 2.375, + "other_queries_rps": 14.95, + "/v0.1/servers_p95_seconds": 0.09235492749921963, + "/v0/servers_p95_seconds": 0.09388280995769606, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009634751773049645, + "list_page_acquire_p95_seconds": 0.0009564607891888728, + "list_search_acquire_p95_seconds": 0.0009700162074554294, + "list_updated_acquire_p95_seconds": 0.000963220675944334, + "other_acquire_p95_seconds": 0.000984359988697372, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.4959133255276053, + "node_cpu_apu6": 0.6202346465458201 + }, + { + "at": "2026-10-28T19:03:00+00:00", + "origin_arrivals_rps": 237.05833333333337, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.008333333333333333, + "list_queries_rps": 0.42500000000000004, + "list_page_queries_rps": 219.18333333333334, + "list_search_queries_rps": 2.8166666666666664, + "list_updated_queries_rps": 1.1333333333333333, + "other_queries_rps": 13.25, + "/v0.1/servers_p95_seconds": 0.0924219621441988, + "/v0/servers_p95_seconds": 0.09390908108886646, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009638686131386863, + "list_page_acquire_p95_seconds": 0.0009565310241933124, + "list_search_acquire_p95_seconds": 0.0009754086181277861, + "list_updated_acquire_p95_seconds": 0.0009627639155470249, + "other_acquire_p95_seconds": 0.0009781398104265402, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.4333884377398742, + "node_cpu_apu6": 0.5664877033559605 + }, + { + "at": "2026-10-28T19:04:00+00:00", + "origin_arrivals_rps": 221.35, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.008333333333333333, + "list_queries_rps": 0.35, + "list_page_queries_rps": 206.75, + "list_search_queries_rps": 2.2083333333333335, + "list_updated_queries_rps": 1.0416666666666667, + "other_queries_rps": 10.866666666666667, + "/v0.1/servers_p95_seconds": 0.0923957078313253, + "/v0/servers_p95_seconds": 0.09379042597187756, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.000957089552238806, + "list_page_acquire_p95_seconds": 0.0009563488020138753, + "list_search_acquire_p95_seconds": 0.0009757165605095544, + "list_updated_acquire_p95_seconds": 0.0009597736625514403, + "other_acquire_p95_seconds": 0.0009767213595077643, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.4326978479490208, + "node_cpu_apu6": 0.5665451353170889 + }, + { + "at": "2026-10-28T19:05:00+00:00", + "origin_arrivals_rps": 219.70833333333334, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.4083333333333333, + "list_page_queries_rps": 202.64166666666665, + "list_search_queries_rps": 1.3833333333333333, + "list_updated_queries_rps": 0.7166666666666667, + "other_queries_rps": 14.558333333333334, + "/v0.1/servers_p95_seconds": 0.09222841370199115, + "/v0/servers_p95_seconds": 0.09372438552885944, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009569343065693429, + "list_page_acquire_p95_seconds": 0.0009555700040011079, + "list_search_acquire_p95_seconds": 0.0009779411764705881, + "list_updated_acquire_p95_seconds": 0.0009609953703703704, + "other_acquire_p95_seconds": 0.0009802608477552043, + "sampled_inflight": 4.0, + "node_cpu_gc71": 0.43300550729278864, + "node_cpu_apu6": 0.574233664850423 + }, + { + "at": "2026-10-28T19:06:00+00:00", + "origin_arrivals_rps": 215.91666666666669, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.44999999999999996, + "list_page_queries_rps": 196.51666666666668, + "list_search_queries_rps": 1.9833333333333334, + "list_updated_queries_rps": 0.6083333333333334, + "other_queries_rps": 16.283333333333335, + "/v0.1/servers_p95_seconds": 0.09161513780408079, + "/v0/servers_p95_seconds": 0.0935759878419453, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009576612903225806, + "list_page_acquire_p95_seconds": 0.0009548040261083109, + "list_search_acquire_p95_seconds": 0.000991627543035994, + "list_updated_acquire_p95_seconds": 0.0009535580524344569, + "other_acquire_p95_seconds": 0.0009788301517376404, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.43000395892959203, + "node_cpu_apu6": 0.5442524982521718 + }, + { + "at": "2026-10-28T19:07:00+00:00", + "origin_arrivals_rps": 204.05000000000004, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.4166666666666667, + "list_page_queries_rps": 187.88333333333333, + "list_search_queries_rps": 2.4, + "list_updated_queries_rps": 0.9, + "other_queries_rps": 12.425, + "/v0.1/servers_p95_seconds": 0.09111534575662324, + "/v0/servers_p95_seconds": 0.0928031754032258, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009668141592920353, + "list_page_acquire_p95_seconds": 0.000954341791221626, + "list_search_acquire_p95_seconds": 0.0009796387520525451, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009720334412081986, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.3904740223593156, + "node_cpu_apu6": 0.5002507871273943 + }, + { + "at": "2026-10-28T19:08:00+00:00", + "origin_arrivals_rps": 196.70833333333331, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.4083333333333333, + "list_page_queries_rps": 183.83333333333331, + "list_search_queries_rps": 1.8416666666666668, + "list_updated_queries_rps": 0.9666666666666667, + "other_queries_rps": 9.858333333333334, + "/v0.1/servers_p95_seconds": 0.09086167512690355, + "/v0/servers_p95_seconds": 0.09224831259228011, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009657024793388431, + "list_page_acquire_p95_seconds": 0.0009540157998683344, + "list_search_acquire_p95_seconds": 0.0009748134328358209, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009705794746818306, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.4173864255208173, + "node_cpu_apu6": 0.5588948679787185 + }, + { + "at": "2026-10-28T19:09:00+00:00", + "origin_arrivals_rps": 206.81666666666666, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.4666666666666667, + "list_page_queries_rps": 192.75, + "list_search_queries_rps": 2.1666666666666665, + "list_updated_queries_rps": 1.3166666666666667, + "other_queries_rps": 10.274999999999999, + "/v0.1/servers_p95_seconds": 0.09113979972511292, + "/v0/servers_p95_seconds": 0.0924405328694378, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009651999999999999, + "list_page_acquire_p95_seconds": 0.0009544189125624988, + "list_search_acquire_p95_seconds": 0.00097183908045977, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009688741721854304, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.48349323521591336, + "node_cpu_apu6": 0.571329199810619 + }, + { + "at": "2026-10-28T19:10:00+00:00", + "origin_arrivals_rps": 221.30833333333334, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.049999999999999996, + "list_queries_rps": 0.4166666666666667, + "list_page_queries_rps": 207.61666666666667, + "list_search_queries_rps": 2.8, + "list_updated_queries_rps": 1.6583333333333332, + "other_queries_rps": 8.95, + "/v0.1/servers_p95_seconds": 0.09095263389112984, + "/v0/servers_p95_seconds": 0.0925854256233878, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009655737704918032, + "list_page_acquire_p95_seconds": 0.0009546952224052716, + "list_search_acquire_p95_seconds": 0.000971590909090909, + "list_updated_acquire_p95_seconds": 0.0009526388888888887, + "other_acquire_p95_seconds": 0.0009618283093053737, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.44883550513671555, + "node_cpu_apu6": 0.5631745147021623 + }, + { + "at": "2026-10-28T19:11:00+00:00", + "origin_arrivals_rps": 219.66666666666663, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.058333333333333334, + "list_queries_rps": 0.35833333333333334, + "list_page_queries_rps": 207.45, + "list_search_queries_rps": 3.225, + "list_updated_queries_rps": 1.2916666666666665, + "other_queries_rps": 7.458333333333333, + "/v0.1/servers_p95_seconds": 0.09081594711836397, + "/v0/servers_p95_seconds": 0.09281807286673056, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009583333333333335, + "list_page_acquire_p95_seconds": 0.0009550848035455552, + "list_search_acquire_p95_seconds": 0.0009778666666666667, + "list_updated_acquire_p95_seconds": 0.0009525401069518717, + "other_acquire_p95_seconds": 0.0009572964669738864, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.43581609848867325, + "node_cpu_apu6": 0.5322242149459877 + }, + { + "at": "2026-10-28T19:12:00+00:00", + "origin_arrivals_rps": 204.65833333333333, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.06666666666666667, + "list_queries_rps": 0.3833333333333333, + "list_page_queries_rps": 193.75, + "list_search_queries_rps": 2.8666666666666667, + "list_updated_queries_rps": 0.8916666666666666, + "other_queries_rps": 6.958333333333333, + "/v0.1/servers_p95_seconds": 0.09072432740066226, + "/v0/servers_p95_seconds": 0.0929063244875258, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009551893194001493, + "list_search_acquire_p95_seconds": 0.0009764238410596027, + "list_updated_acquire_p95_seconds": 0.0009553072625698325, + "other_acquire_p95_seconds": 0.0009583403010033444, + "sampled_inflight": 3.0, + "node_cpu_gc71": 0.39929077811951474, + "node_cpu_apu6": 0.5019778084730205 + }, + { + "at": "2026-10-28T19:13:00+00:00", + "origin_arrivals_rps": 200.45833333333334, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.06666666666666667, + "list_queries_rps": 0.3833333333333333, + "list_page_queries_rps": 191.10000000000002, + "list_search_queries_rps": 1.8333333333333335, + "list_updated_queries_rps": 0.9, + "other_queries_rps": 6.574999999999999, + "/v0.1/servers_p95_seconds": 0.09051217403685774, + "/v0/servers_p95_seconds": 0.09308379573435062, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009549941215989251, + "list_search_acquire_p95_seconds": 0.0009805220883534135, + "list_updated_acquire_p95_seconds": 0.0009552054794520547, + "other_acquire_p95_seconds": 0.0009599680656934306, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.39327265196916417, + "node_cpu_apu6": 0.5146326006135495 + }, + { + "at": "2026-10-28T19:14:00+00:00", + "origin_arrivals_rps": 196.85000000000008, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.3833333333333333, + "list_page_queries_rps": 187.175, + "list_search_queries_rps": 1.8333333333333335, + "list_updated_queries_rps": 0.7916666666666667, + "other_queries_rps": 6.841666666666667, + "/v0.1/servers_p95_seconds": 0.08994307836297112, + "/v0/servers_p95_seconds": 0.09284984110169492, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009499999999999999, + "list_page_acquire_p95_seconds": 0.00095457748893995, + "list_search_acquire_p95_seconds": 0.0009820224719101123, + "list_updated_acquire_p95_seconds": 0.0009564406779661017, + "other_acquire_p95_seconds": 0.0009610242179616548, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.35944865540780785, + "node_cpu_apu6": 0.49887906849229086 + }, + { + "at": "2026-10-28T19:15:00+00:00", + "origin_arrivals_rps": 180.325, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.4083333333333333, + "list_page_queries_rps": 168.79166666666666, + "list_search_queries_rps": 2.125, + "list_updated_queries_rps": 0.5083333333333333, + "other_queries_rps": 8.533333333333333, + "/v0.1/servers_p95_seconds": 0.0896444797661036, + "/v0/servers_p95_seconds": 0.09215084388185657, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009541141255890969, + "list_search_acquire_p95_seconds": 0.0009798206278026904, + "list_updated_acquire_p95_seconds": 0.0009541666666666665, + "other_acquire_p95_seconds": 0.0009614670380687095, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.33169298775076944, + "node_cpu_apu6": 0.4674397527213538 + }, + { + "at": "2026-10-28T19:16:00+00:00", + "origin_arrivals_rps": 171.75833333333333, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.45833333333333337, + "list_page_queries_rps": 156.675, + "list_search_queries_rps": 1.8666666666666667, + "list_updated_queries_rps": 0.6916666666666667, + "other_queries_rps": 12.133333333333333, + "/v0.1/servers_p95_seconds": 0.08929931291516653, + "/v0/servers_p95_seconds": 0.09127335224792665, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009532860664666422, + "list_search_acquire_p95_seconds": 0.0009651595744680852, + "list_updated_acquire_p95_seconds": 0.0009542410714285713, + "other_acquire_p95_seconds": 0.000965745856353591, + "sampled_inflight": 5.0, + "node_cpu_gc71": 0.3851265666967568, + "node_cpu_apu6": 0.4693075519666308 + }, + { + "at": "2026-10-28T19:17:00+00:00", + "origin_arrivals_rps": 170.67499999999995, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.44999999999999996, + "list_page_queries_rps": 150.88333333333333, + "list_search_queries_rps": 1.55, + "list_updated_queries_rps": 0.8666666666666667, + "other_queries_rps": 16.858333333333334, + "/v0.1/servers_p95_seconds": 0.08938724394785845, + "/v0/servers_p95_seconds": 0.09146519671422394, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009500000000000001, + "list_page_acquire_p95_seconds": 0.0009530658675432971, + "list_search_acquire_p95_seconds": 0.0009700575815738964, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009740942028985507, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.4179968696664825, + "node_cpu_apu6": 0.43968395427946233 + }, + { + "at": "2026-10-28T19:18:00+00:00", + "origin_arrivals_rps": 171.9416666666667, + "origin_499_rps": 0.016666666666666666, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.008333333333333333, + "list_queries_rps": 0.3916666666666667, + "list_page_queries_rps": 149.64166666666668, + "list_search_queries_rps": 2.3249999999999997, + "list_updated_queries_rps": 1.25, + "other_queries_rps": 18.2, + "/v0.1/servers_p95_seconds": 0.089560620247418, + "/v0/servers_p95_seconds": 0.09153828654404647, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.000952987358335268, + "list_search_acquire_p95_seconds": 0.0009652733118971061, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009810473494595985, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.42341986068495313, + "node_cpu_apu6": 0.4779916485989815 + }, + { + "at": "2026-10-28T19:19:00+00:00", + "origin_arrivals_rps": 183.26666666666668, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.4916666666666667, + "list_page_queries_rps": 161.95833333333331, + "list_search_queries_rps": 3.5250000000000004, + "list_updated_queries_rps": 1.525, + "other_queries_rps": 15.808333333333334, + "/v0.1/servers_p95_seconds": 0.08952115836395397, + "/v0/servers_p95_seconds": 0.09160794430794429, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009499999999999999, + "list_page_acquire_p95_seconds": 0.000952816585969328, + "list_search_acquire_p95_seconds": 0.0009697916666666665, + "list_updated_acquire_p95_seconds": 0.0009500000000000001, + "other_acquire_p95_seconds": 0.0009838031423290204, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.3740447819021938, + "node_cpu_apu6": 0.49930434450458655 + }, + { + "at": "2026-10-28T19:20:00+00:00", + "origin_arrivals_rps": 190.47500000000002, + "origin_499_rps": 0.016666666666666666, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.55, + "list_page_queries_rps": 171.7, + "list_search_queries_rps": 2.9916666666666667, + "list_updated_queries_rps": 0.9666666666666667, + "other_queries_rps": 14.316666666666666, + "/v0.1/servers_p95_seconds": 0.0896967612496417, + "/v0/servers_p95_seconds": 0.09171252739841514, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.000957142857142857, + "list_page_acquire_p95_seconds": 0.0009528459201843511, + "list_search_acquire_p95_seconds": 0.0009711111111111107, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009876716880815241, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.39345583381315613, + "node_cpu_apu6": 0.49194567045075627 + }, + { + "at": "2026-10-28T19:21:00+00:00", + "origin_arrivals_rps": 188.4166666666667, + "origin_499_rps": 0.025, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.5583333333333333, + "list_page_queries_rps": 167.9, + "list_search_queries_rps": 2.0, + "list_updated_queries_rps": 1.1416666666666666, + "other_queries_rps": 16.716666666666665, + "/v0.1/servers_p95_seconds": 0.0897631082505687, + "/v0/servers_p95_seconds": 0.09159086031121448, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009567375886524823, + "list_page_acquire_p95_seconds": 0.0009530146220056, + "list_search_acquire_p95_seconds": 0.0009680461329715058, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009936395759717314, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.3655184074891925, + "node_cpu_apu6": 0.4792212982465571 + }, + { + "at": "2026-10-28T19:22:00+00:00", + "origin_arrivals_rps": 179.02499999999998, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.5583333333333333, + "list_page_queries_rps": 158.475, + "list_search_queries_rps": 1.6833333333333333, + "list_updated_queries_rps": 1.2583333333333333, + "other_queries_rps": 17.108333333333334, + "/v0.1/servers_p95_seconds": 0.08956366556743232, + "/v0/servers_p95_seconds": 0.09115707236842106, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009565068493150684, + "list_page_acquire_p95_seconds": 0.0009527142299233315, + "list_search_acquire_p95_seconds": 0.0009641025641025642, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009938818565400844, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.3418365026785371, + "node_cpu_apu6": 0.43769777691632045 + }, + { + "at": "2026-10-28T19:23:00+00:00", + "origin_arrivals_rps": 157.28333333333336, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.4666666666666667, + "list_page_queries_rps": 142.66666666666669, + "list_search_queries_rps": 1.9416666666666667, + "list_updated_queries_rps": 0.6166666666666667, + "other_queries_rps": 11.616666666666667, + "/v0.1/servers_p95_seconds": 0.08944233295515615, + "/v0/servers_p95_seconds": 0.09064760614272814, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009563758389261744, + "list_page_acquire_p95_seconds": 0.0009527260259521045, + "list_search_acquire_p95_seconds": 0.0009622844827586206, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009881310022162029, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.32271727376816367, + "node_cpu_apu6": 0.4362611324094663 + }, + { + "at": "2026-10-28T19:24:00+00:00", + "origin_arrivals_rps": 159.1500000000001, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.525, + "list_page_queries_rps": 147.24166666666667, + "list_search_queries_rps": 2.325, + "list_updated_queries_rps": 0.8166666666666667, + "other_queries_rps": 8.275, + "/v0.1/servers_p95_seconds": 0.0894840977183683, + "/v0/servers_p95_seconds": 0.09035844370860929, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009564189189189188, + "list_page_acquire_p95_seconds": 0.0009527555024739805, + "list_search_acquire_p95_seconds": 0.0009626666666666667, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009850409836065575, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.3656906607488054, + "node_cpu_apu6": 0.4659430308107403 + }, + { + "at": "2026-10-28T19:25:00+00:00", + "origin_arrivals_rps": 179.325, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.049999999999999996, + "list_queries_rps": 0.55, + "list_page_queries_rps": 166.35833333333335, + "list_search_queries_rps": 2.0166666666666666, + "list_updated_queries_rps": 0.8583333333333334, + "other_queries_rps": 9.708333333333332, + "/v0.1/servers_p95_seconds": 0.08912505314303067, + "/v0/servers_p95_seconds": 0.09038350951374208, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009693877551020408, + "list_page_acquire_p95_seconds": 0.0009526613489585561, + "list_search_acquire_p95_seconds": 0.0009614457831325302, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009802735476593344, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.3605612604148822, + "node_cpu_apu6": 0.4873373721956644 + }, + { + "at": "2026-10-28T19:26:00+00:00", + "origin_arrivals_rps": 187.66666666666663, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.5833333333333334, + "list_page_queries_rps": 170.74166666666667, + "list_search_queries_rps": 2.5666666666666664, + "list_updated_queries_rps": 0.4166666666666667, + "other_queries_rps": 13.3, + "/v0.1/servers_p95_seconds": 0.08879609158478206, + "/v0/servers_p95_seconds": 0.09081945775052688, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009688741721854304, + "list_page_acquire_p95_seconds": 0.0009527356073598846, + "list_search_acquire_p95_seconds": 0.000974506828528073, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009709326894053727, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.38271375071635705, + "node_cpu_apu6": 0.4867479438369628 + }, + { + "at": "2026-10-28T19:27:00+00:00", + "origin_arrivals_rps": 187.05833333333334, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.5, + "list_page_queries_rps": 170.39166666666665, + "list_search_queries_rps": 3.0083333333333337, + "list_updated_queries_rps": 0.36666666666666664, + "other_queries_rps": 12.783333333333335, + "/v0.1/servers_p95_seconds": 0.08847671344955628, + "/v0/servers_p95_seconds": 0.09119169510807731, + "sampled_pending_acquisitions": 1.0, + "list_acquire_p95_seconds": 0.0009773381294964027, + "list_page_acquire_p95_seconds": 0.0009529186473750855, + "list_search_acquire_p95_seconds": 0.0009721232876712327, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009697338526452451, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.35781119905295783, + "node_cpu_apu6": 0.44428565069902104 + }, + { + "at": "2026-10-28T19:28:00+00:00", + "origin_arrivals_rps": 188.0333333333334, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.33333333333333337, + "list_page_queries_rps": 176.6, + "list_search_queries_rps": 2.1333333333333333, + "list_updated_queries_rps": 0.5166666666666666, + "other_queries_rps": 8.666666666666666, + "/v0.1/servers_p95_seconds": 0.08808359746434227, + "/v0/servers_p95_seconds": 0.091609989181392, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009790076335877862, + "list_page_acquire_p95_seconds": 0.0009529828188076525, + "list_search_acquire_p95_seconds": 0.0009736803519061582, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009715762711864407, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.3947381778777829, + "node_cpu_apu6": 0.5001208076947175 + }, + { + "at": "2026-10-28T19:29:00+00:00", + "origin_arrivals_rps": 202.95, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.30833333333333335, + "list_page_queries_rps": 193.68333333333334, + "list_search_queries_rps": 1.5499999999999998, + "list_updated_queries_rps": 0.85, + "other_queries_rps": 6.5, + "/v0.1/servers_p95_seconds": 0.08799615081529978, + "/v0/servers_p95_seconds": 0.09192568214456678, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009833333333333335, + "list_page_acquire_p95_seconds": 0.000953343201876885, + "list_search_acquire_p95_seconds": 0.0009784251968503936, + "list_updated_acquire_p95_seconds": 0.0009500000000000001, + "other_acquire_p95_seconds": 0.0009746062303115156, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.37049375600666584, + "node_cpu_apu6": 0.5258029977274419 + }, + { + "at": "2026-10-28T19:30:00+00:00", + "origin_arrivals_rps": 181.6, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.325, + "list_page_queries_rps": 169.25, + "list_search_queries_rps": 1.8833333333333333, + "list_updated_queries_rps": 1.2416666666666667, + "other_queries_rps": 8.708333333333332, + "/v0.1/servers_p95_seconds": 0.08872177670023623, + "/v0/servers_p95_seconds": 0.09241334355828221, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009674311926605504, + "list_page_acquire_p95_seconds": 0.0009541272304111714, + "list_search_acquire_p95_seconds": 0.0009906106870229007, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009870881088825216, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.3041652846271366, + "node_cpu_apu6": 0.4390652900649002 + }, + { + "at": "2026-10-28T19:31:00+00:00", + "origin_arrivals_rps": 147.49166666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.525, + "list_page_queries_rps": 131.89166666666665, + "list_search_queries_rps": 1.9, + "list_updated_queries_rps": 1.775, + "other_queries_rps": 11.341666666666667, + "/v0.1/servers_p95_seconds": 0.08888670411985014, + "/v0/servers_p95_seconds": 0.09195075820009892, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009855140186915887, + "list_page_acquire_p95_seconds": 0.0009540499745028046, + "list_search_acquire_p95_seconds": 0.0009825225225225224, + "list_updated_acquire_p95_seconds": 0.000952810650887574, + "other_acquire_p95_seconds": 0.0009924601632335793, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.3366986189559005, + "node_cpu_apu6": 0.45459804947580174 + }, + { + "at": "2026-10-28T19:32:00+00:00", + "origin_arrivals_rps": 161.44166666666663, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.008333333333333333, + "list_queries_rps": 0.5333333333333333, + "list_page_queries_rps": 146.05833333333334, + "list_search_queries_rps": 1.95, + "list_updated_queries_rps": 1.5583333333333333, + "other_queries_rps": 11.008333333333333, + "/v0.1/servers_p95_seconds": 0.08918376553936641, + "/v0/servers_p95_seconds": 0.09190336071695296, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009924107142857143, + "list_page_acquire_p95_seconds": 0.0009541787424718914, + "list_search_acquire_p95_seconds": 0.0009984877126654064, + "list_updated_acquire_p95_seconds": 0.0009525469168900803, + "other_acquire_p95_seconds": 0.0009951856148491876, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.330818387325174, + "node_cpu_apu6": 0.44167638596263714 + }, + { + "at": "2026-10-28T19:33:00+00:00", + "origin_arrivals_rps": 164.275, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.425, + "list_page_queries_rps": 149.725, + "list_search_queries_rps": 2.033333333333333, + "list_updated_queries_rps": 0.7833333333333333, + "other_queries_rps": 11.108333333333334, + "/v0.1/servers_p95_seconds": 0.08964688422208497, + "/v0/servers_p95_seconds": 0.09191936587591244, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009892561983471076, + "list_page_acquire_p95_seconds": 0.0009542934806340335, + "list_search_acquire_p95_seconds": 0.0015846153846153627, + "list_updated_acquire_p95_seconds": 0.0009525675675675677, + "other_acquire_p95_seconds": 0.0010678260869565305, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.34720784582095804, + "node_cpu_apu6": 0.4288354563652926 + }, + { + "at": "2026-10-28T19:34:00+00:00", + "origin_arrivals_rps": 145.14999999999998, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.45, + "list_page_queries_rps": 133.0, + "list_search_queries_rps": 2.175, + "list_updated_queries_rps": 0.5416666666666666, + "other_queries_rps": 9.1, + "/v0.1/servers_p95_seconds": 0.0898164375928678, + "/v0/servers_p95_seconds": 0.09159141637945181, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009865384615384614, + "list_page_acquire_p95_seconds": 0.0009539688481043227, + "list_search_acquire_p95_seconds": 0.0009924668874172187, + "list_updated_acquire_p95_seconds": 0.0009528273809523808, + "other_acquire_p95_seconds": 0.0011636363636363526, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.32972387871535225, + "node_cpu_apu6": 0.39156694152043203 + }, + { + "at": "2026-10-28T19:35:00+00:00", + "origin_arrivals_rps": 140.77499999999998, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.44166666666666665, + "list_page_queries_rps": 130.06666666666666, + "list_search_queries_rps": 2.875, + "list_updated_queries_rps": 0.65, + "other_queries_rps": 6.633333333333333, + "/v0.1/servers_p95_seconds": 0.08924973723723725, + "/v0/servers_p95_seconds": 0.09134498374361356, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009781481481481483, + "list_page_acquire_p95_seconds": 0.0009531381312088605, + "list_search_acquire_p95_seconds": 0.0009787443267776097, + "list_updated_acquire_p95_seconds": 0.0009531772575250837, + "other_acquire_p95_seconds": 0.000990555345674348, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.42348646120080957, + "node_cpu_apu6": 0.4297340806663366 + }, + { + "at": "2026-10-28T19:36:00+00:00", + "origin_arrivals_rps": 157.10000000000002, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.3833333333333333, + "list_page_queries_rps": 146.34166666666667, + "list_search_queries_rps": 2.9, + "list_updated_queries_rps": 0.6833333333333333, + "other_queries_rps": 5.75, + "/v0.1/servers_p95_seconds": 0.0896461220727794, + "/v0/servers_p95_seconds": 0.09187752705011389, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009661016949152544, + "list_page_acquire_p95_seconds": 0.0009533673894912425, + "list_search_acquire_p95_seconds": 0.0009813617606602475, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009862977602108037, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.4905626984264624, + "node_cpu_apu6": 0.46099929391954597 + }, + { + "at": "2026-10-28T19:37:00+00:00", + "origin_arrivals_rps": 165.49166666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.29166666666666663, + "list_page_queries_rps": 154.90833333333333, + "list_search_queries_rps": 2.8833333333333333, + "list_updated_queries_rps": 0.5, + "other_queries_rps": 5.091666666666667, + "/v0.1/servers_p95_seconds": 0.09000070332480818, + "/v0/servers_p95_seconds": 0.09211421257922968, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009534881792770172, + "list_search_acquire_p95_seconds": 0.0009796488946684004, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.000977744209466264, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.3763683853581218, + "node_cpu_apu6": 0.4250590488013889 + }, + { + "at": "2026-10-28T19:38:00+00:00", + "origin_arrivals_rps": 149.7583333333333, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.2833333333333333, + "list_page_queries_rps": 139.40833333333333, + "list_search_queries_rps": 2.783333333333333, + "list_updated_queries_rps": 0.4, + "other_queries_rps": 5.166666666666667, + "/v0.1/servers_p95_seconds": 0.09002950349301397, + "/v0/servers_p95_seconds": 0.09223886867636868, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009593137254901962, + "list_page_acquire_p95_seconds": 0.0009534896479343572, + "list_search_acquire_p95_seconds": 0.0009863580246913578, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.000962767257177764, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.39057205352267677, + "node_cpu_apu6": 0.39798488157830436 + }, + { + "at": "2026-10-28T19:39:00+00:00", + "origin_arrivals_rps": 141.29166666666663, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.3416666666666667, + "list_page_queries_rps": 132.225, + "list_search_queries_rps": 1.8333333333333335, + "list_updated_queries_rps": 0.31666666666666665, + "other_queries_rps": 5.5, + "/v0.1/servers_p95_seconds": 0.0899507784876869, + "/v0/servers_p95_seconds": 0.0923954245451302, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009595959595959597, + "list_page_acquire_p95_seconds": 0.0009536318088996533, + "list_search_acquire_p95_seconds": 0.0009867403314917128, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.000956442663378545, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.4158715812586403, + "node_cpu_apu6": 0.36564853919893114 + }, + { + "at": "2026-10-28T19:40:00+00:00", + "origin_arrivals_rps": 136.48333333333335, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.275, + "list_page_queries_rps": 128.24166666666667, + "list_search_queries_rps": 1.2416666666666667, + "list_updated_queries_rps": 0.41666666666666663, + "other_queries_rps": 5.975, + "/v0.1/servers_p95_seconds": 0.08981151424379508, + "/v0/servers_p95_seconds": 0.09250864629747509, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.000961176470588235, + "list_page_acquire_p95_seconds": 0.0009534393056654441, + "list_search_acquire_p95_seconds": 0.0009931118314424636, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009565723270440248, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.42531897083414755, + "node_cpu_apu6": 0.3415961692522008 + }, + { + "at": "2026-10-28T19:41:00+00:00", + "origin_arrivals_rps": 122.78333333333335, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.0, + "list_queries_rps": 0.31666666666666665, + "list_page_queries_rps": 111.8, + "list_search_queries_rps": 1.6166666666666667, + "list_updated_queries_rps": 0.55, + "other_queries_rps": 8.325, + "/v0.1/servers_p95_seconds": 0.08964488858483188, + "/v0/servers_p95_seconds": 0.09282949101796405, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009603260869565218, + "list_page_acquire_p95_seconds": 0.0009533001736933524, + "list_search_acquire_p95_seconds": 0.001911111111111099, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009603154084798344, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.36557432324286004, + "node_cpu_apu6": 0.3427331927998845 + }, + { + "at": "2026-10-28T19:42:00+00:00", + "origin_arrivals_rps": 124.25000000000003, + "origin_499_rps": 0.041666666666666664, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.45, + "list_page_queries_rps": 106.46666666666667, + "list_search_queries_rps": 2.4583333333333335, + "list_updated_queries_rps": 0.475, + "other_queries_rps": 14.325, + "/v0.1/servers_p95_seconds": 0.08921672153438774, + "/v0/servers_p95_seconds": 0.09249534161490681, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009686274509803923, + "list_page_acquire_p95_seconds": 0.0009529875045223054, + "list_search_acquire_p95_seconds": 0.0009952380952380953, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009712230215827335, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.38065242602460786, + "node_cpu_apu6": 0.3844865385988669 + }, + { + "at": "2026-10-28T19:43:00+00:00", + "origin_arrivals_rps": 147.66666666666669, + "origin_499_rps": 0.058333333333333334, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.48333333333333334, + "list_page_queries_rps": 125.14166666666667, + "list_search_queries_rps": 3.033333333333333, + "list_updated_queries_rps": 0.4, + "other_queries_rps": 18.633333333333333, + "/v0.1/servers_p95_seconds": 0.08971757196407101, + "/v0/servers_p95_seconds": 0.09257758843273946, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009582608695652173, + "list_page_acquire_p95_seconds": 0.0009532456261193001, + "list_search_acquire_p95_seconds": 0.0009828665568369028, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009774407856730214, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.47043278367758223, + "node_cpu_apu6": 0.434088829682878 + }, + { + "at": "2026-10-28T19:44:00+00:00", + "origin_arrivals_rps": 158.72500000000002, + "origin_499_rps": 0.016666666666666666, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.4083333333333333, + "list_page_queries_rps": 138.54166666666669, + "list_search_queries_rps": 2.0166666666666666, + "list_updated_queries_rps": 0.4166666666666667, + "other_queries_rps": 17.275, + "/v0.1/servers_p95_seconds": 0.09003779891117084, + "/v0/servers_p95_seconds": 0.09223138427734374, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009585585585585585, + "list_page_acquire_p95_seconds": 0.0009531335187002507, + "list_search_acquire_p95_seconds": 0.0009851851851851853, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009746138515196811, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.3575597021636592, + "node_cpu_apu6": 0.39356898387753364 + }, + { + "at": "2026-10-28T19:45:00+00:00", + "origin_arrivals_rps": 143.24166666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.3, + "list_page_queries_rps": 129.38333333333333, + "list_search_queries_rps": 1.0416666666666667, + "list_updated_queries_rps": 0.41666666666666663, + "other_queries_rps": 12.166666666666668, + "/v0.1/servers_p95_seconds": 0.09040145367299413, + "/v0/servers_p95_seconds": 0.09199743560519717, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009581196581196582, + "list_page_acquire_p95_seconds": 0.0009537048225859606, + "list_search_acquire_p95_seconds": 0.0009825342465753426, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009738294314381273, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.30960771899359923, + "node_cpu_apu6": 0.3677980862276551 + }, + { + "at": "2026-10-28T19:46:00+00:00", + "origin_arrivals_rps": 130.2, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.06666666666666667, + "list_queries_rps": 0.3416666666666667, + "list_page_queries_rps": 119.14166666666667, + "list_search_queries_rps": 1.7166666666666666, + "list_updated_queries_rps": 0.7, + "other_queries_rps": 8.375, + "/v0.1/servers_p95_seconds": 0.0904662157450666, + "/v0/servers_p95_seconds": 0.09194686676130602, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009674311926605505, + "list_page_acquire_p95_seconds": 0.0009534408373059204, + "list_search_acquire_p95_seconds": 0.0009828665568369028, + "list_updated_acquire_p95_seconds": 0.0009499999999999998, + "other_acquire_p95_seconds": 0.000972093023255814, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.35685395387239016, + "node_cpu_apu6": 0.3870221738228186 + }, + { + "at": "2026-10-28T19:47:00+00:00", + "origin_arrivals_rps": 135.13333333333335, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.058333333333333334, + "list_queries_rps": 0.43333333333333335, + "list_page_queries_rps": 121.88333333333333, + "list_search_queries_rps": 2.375, + "list_updated_queries_rps": 0.9333333333333333, + "other_queries_rps": 9.508333333333333, + "/v0.1/servers_p95_seconds": 0.09082785981558199, + "/v0/servers_p95_seconds": 0.0920497372007008, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009585585585585585, + "list_page_acquire_p95_seconds": 0.0009536502169480891, + "list_search_acquire_p95_seconds": 0.0009831010452961673, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009675103163686381, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.40188926898384425, + "node_cpu_apu6": 0.36430582878820134 + }, + { + "at": "2026-10-28T19:48:00+00:00", + "origin_arrivals_rps": 138.15833333333333, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.525, + "list_page_queries_rps": 121.48333333333333, + "list_search_queries_rps": 2.6416666666666666, + "list_updated_queries_rps": 0.95, + "other_queries_rps": 12.608333333333334, + "/v0.1/servers_p95_seconds": 0.09078446000812017, + "/v0/servers_p95_seconds": 0.0918085256968641, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009583333333333333, + "list_page_acquire_p95_seconds": 0.000953378778897451, + "list_search_acquire_p95_seconds": 0.0009821174377224197, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009617213361420402, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.43018563814645555, + "node_cpu_apu6": 0.39559169624747786 + }, + { + "at": "2026-10-28T19:49:00+00:00", + "origin_arrivals_rps": 138.20833333333337, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.4, + "list_page_queries_rps": 122.89166666666667, + "list_search_queries_rps": 2.9333333333333336, + "list_updated_queries_rps": 0.9166666666666667, + "other_queries_rps": 11.266666666666666, + "/v0.1/servers_p95_seconds": 0.09111360379418587, + "/v0/servers_p95_seconds": 0.09202862595419849, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009587155963302752, + "list_page_acquire_p95_seconds": 0.0009536550511157526, + "list_search_acquire_p95_seconds": 0.0009763119533527696, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009633710950909713, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.39787014476557186, + "node_cpu_apu6": 0.38331391172347745 + }, + { + "at": "2026-10-28T19:50:00+00:00", + "origin_arrivals_rps": 142.91666666666669, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.35, + "list_page_queries_rps": 131.25, + "list_search_queries_rps": 2.7583333333333337, + "list_updated_queries_rps": 0.6833333333333333, + "other_queries_rps": 8.108333333333334, + "/v0.1/servers_p95_seconds": 0.09098559656997388, + "/v0/servers_p95_seconds": 0.09202922661870501, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009662393162393161, + "list_page_acquire_p95_seconds": 0.0009530008566227647, + "list_search_acquire_p95_seconds": 0.0009735026041666667, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009641489361702127, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.41911644913173707, + "node_cpu_apu6": 0.43812850826943756 + }, + { + "at": "2026-10-28T19:51:00+00:00", + "origin_arrivals_rps": 178.96666666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.39166666666666666, + "list_page_queries_rps": 158.51666666666665, + "list_search_queries_rps": 3.35, + "list_updated_queries_rps": 0.8166666666666667, + "other_queries_rps": 15.891666666666666, + "/v0.1/servers_p95_seconds": 0.09057082064981052, + "/v0/servers_p95_seconds": 0.09186848657300377, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009581196581196582, + "list_page_acquire_p95_seconds": 0.0009532813602198924, + "list_search_acquire_p95_seconds": 0.0009628378378378378, + "list_updated_acquire_p95_seconds": 0.0009536821705426356, + "other_acquire_p95_seconds": 0.000962646828504307, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.3839533730795665, + "node_cpu_apu6": 0.49129586441146483 + }, + { + "at": "2026-10-28T19:52:00+00:00", + "origin_arrivals_rps": 199.6083333333334, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.29166666666666663, + "list_page_queries_rps": 171.45833333333331, + "list_search_queries_rps": 3.15, + "list_updated_queries_rps": 1.1833333333333333, + "other_queries_rps": 23.433333333333334, + "/v0.1/servers_p95_seconds": 0.09014562451233417, + "/v0/servers_p95_seconds": 0.09187020905923346, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009589622641509435, + "list_page_acquire_p95_seconds": 0.0009530949630314233, + "list_search_acquire_p95_seconds": 0.0009675925925925927, + "list_updated_acquire_p95_seconds": 0.0009533216783216783, + "other_acquire_p95_seconds": 0.0009594831410825199, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.3109609522941658, + "node_cpu_apu6": 0.4315877668467389 + }, + { + "at": "2026-10-28T19:53:00+00:00", + "origin_arrivals_rps": 168.4666666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.3416666666666667, + "list_page_queries_rps": 143.56666666666666, + "list_search_queries_rps": 1.775, + "list_updated_queries_rps": 1.275, + "other_queries_rps": 21.458333333333336, + "/v0.1/servers_p95_seconds": 0.08959837139648073, + "/v0/servers_p95_seconds": 0.09174065819423373, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009596938775510203, + "list_page_acquire_p95_seconds": 0.0009531262854792264, + "list_search_acquire_p95_seconds": 0.000965772669220945, + "list_updated_acquire_p95_seconds": 0.0009531986531986532, + "other_acquire_p95_seconds": 0.000957154488092815, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.32273777820034105, + "node_cpu_apu6": 0.39696440608095146 + }, + { + "at": "2026-10-28T19:54:00+00:00", + "origin_arrivals_rps": 149.02500000000003, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.31666666666666665, + "list_page_queries_rps": 125.41666666666666, + "list_search_queries_rps": 1.5499999999999998, + "list_updated_queries_rps": 1.6416666666666666, + "other_queries_rps": 20.058333333333334, + "/v0.1/servers_p95_seconds": 0.08916072109881724, + "/v0/servers_p95_seconds": 0.09158063498738436, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009597938144329895, + "list_page_acquire_p95_seconds": 0.0009529642545771576, + "list_search_acquire_p95_seconds": 0.0009662624821683309, + "list_updated_acquire_p95_seconds": 0.0009525469168900803, + "other_acquire_p95_seconds": 0.0009556032171581769, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.3779308981511955, + "node_cpu_apu6": 0.3758964225923895 + }, + { + "at": "2026-10-28T19:55:00+00:00", + "origin_arrivals_rps": 142.41666666666669, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.30000000000000004, + "list_page_queries_rps": 120.4, + "list_search_queries_rps": 1.6833333333333333, + "list_updated_queries_rps": 2.5083333333333333, + "other_queries_rps": 17.508333333333333, + "/v0.1/servers_p95_seconds": 0.08933421300659752, + "/v0/servers_p95_seconds": 0.09163406388435534, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.000953110132366103, + "list_search_acquire_p95_seconds": 0.0009688837920489298, + "list_updated_acquire_p95_seconds": 0.0009536893203883494, + "other_acquire_p95_seconds": 0.0009551672985000824, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.4013918836322167, + "node_cpu_apu6": 0.39483068894357226 + }, + { + "at": "2026-10-28T19:56:00+00:00", + "origin_arrivals_rps": 140.80833333333337, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.3833333333333333, + "list_page_queries_rps": 122.4, + "list_search_queries_rps": 1.9166666666666667, + "list_updated_queries_rps": 1.5750000000000002, + "other_queries_rps": 14.533333333333333, + "/v0.1/servers_p95_seconds": 0.0898077117444916, + "/v0/servers_p95_seconds": 0.09168362687540771, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009601063829787232, + "list_page_acquire_p95_seconds": 0.0009530243989405052, + "list_search_acquire_p95_seconds": 0.0009807619047619049, + "list_updated_acquire_p95_seconds": 0.0009541036717062633, + "other_acquire_p95_seconds": 0.000956853496115427, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.3820477411080598, + "node_cpu_apu6": 0.37422277762088924 + }, + { + "at": "2026-10-28T19:57:00+00:00", + "origin_arrivals_rps": 143.21666666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.075, + "list_queries_rps": 0.42500000000000004, + "list_page_queries_rps": 126.125, + "list_search_queries_rps": 1.875, + "list_updated_queries_rps": 0.275, + "other_queries_rps": 14.7, + "/v0.1/servers_p95_seconds": 0.08985292737510828, + "/v0/servers_p95_seconds": 0.09137118362831859, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009590476190476191, + "list_page_acquire_p95_seconds": 0.0009527947209327433, + "list_search_acquire_p95_seconds": 0.0009785571142284567, + "list_updated_acquire_p95_seconds": 0.0009546798029556647, + "other_acquire_p95_seconds": 0.0009591916952227373, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.35132444792519285, + "node_cpu_apu6": 0.35699119981970096 + }, + { + "at": "2026-10-28T19:58:00+00:00", + "origin_arrivals_rps": 143.0166666666667, + "origin_499_rps": 0.016666666666666666, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.075, + "list_queries_rps": 0.375, + "list_page_queries_rps": 124.97500000000001, + "list_search_queries_rps": 2.5166666666666666, + "list_updated_queries_rps": 0.3, + "other_queries_rps": 15.091666666666667, + "/v0.1/servers_p95_seconds": 0.08956700207931086, + "/v0/servers_p95_seconds": 0.09131702501096972, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009597938144329895, + "list_page_acquire_p95_seconds": 0.0009523836503439903, + "list_search_acquire_p95_seconds": 0.0009747154471544717, + "list_updated_acquire_p95_seconds": 0.0009554913294797689, + "other_acquire_p95_seconds": 0.0009638822246455835, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.31575796415878965, + "node_cpu_apu6": 0.3914295194769996 + }, + { + "at": "2026-10-28T19:59:00+00:00", + "origin_arrivals_rps": 144.90000000000003, + "origin_499_rps": 0.03333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.06666666666666667, + "list_queries_rps": 0.3, + "list_page_queries_rps": 124.69999999999999, + "list_search_queries_rps": 4.033333333333333, + "list_updated_queries_rps": 0.25, + "other_queries_rps": 15.825, + "/v0.1/servers_p95_seconds": 0.08916369825061349, + "/v0/servers_p95_seconds": 0.09122242609582061, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009593137254901963, + "list_page_acquire_p95_seconds": 0.0009522602536026172, + "list_search_acquire_p95_seconds": 0.0009715365239294709, + "list_updated_acquire_p95_seconds": 0.000957949790794979, + "other_acquire_p95_seconds": 0.0009733035304906006, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.2856181042472599, + "node_cpu_apu6": 0.3710075898881769 + }, + { + "at": "2026-10-28T20:00:00+00:00", + "origin_arrivals_rps": 136.75833333333335, + "origin_499_rps": 0.016666666666666666, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.3833333333333333, + "list_page_queries_rps": 118.15, + "list_search_queries_rps": 4.925, + "list_updated_queries_rps": 0.575, + "other_queries_rps": 12.875, + "/v0.1/servers_p95_seconds": 0.08851478778162183, + "/v0/servers_p95_seconds": 0.09114347290640393, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009682692307692308, + "list_page_acquire_p95_seconds": 0.0009522458628841606, + "list_search_acquire_p95_seconds": 0.0009719157472417253, + "list_updated_acquire_p95_seconds": 0.0009582608695652173, + "other_acquire_p95_seconds": 0.0009775535939470367, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.2884827134051496, + "node_cpu_apu6": 0.39332704148449316 + }, + { + "at": "2026-10-28T20:01:00+00:00", + "origin_arrivals_rps": 136.13333333333338, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.5, + "list_page_queries_rps": 115.06666666666666, + "list_search_queries_rps": 5.591666666666667, + "list_updated_queries_rps": 5.358333333333333, + "other_queries_rps": 9.491666666666667, + "/v0.1/servers_p95_seconds": 0.08840939267273173, + "/v0/servers_p95_seconds": 0.09091879853814527, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009662393162393161, + "list_page_acquire_p95_seconds": 0.0009521308706247231, + "list_search_acquire_p95_seconds": 0.0009622877930476959, + "list_updated_acquire_p95_seconds": 0.0009527417027417027, + "other_acquire_p95_seconds": 0.000977926377118644, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.3196951954373314, + "node_cpu_apu6": 0.407155996390793 + }, + { + "at": "2026-10-28T20:02:00+00:00", + "origin_arrivals_rps": 144.3, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.5083333333333333, + "list_page_queries_rps": 123.49166666666667, + "list_search_queries_rps": 4.291666666666666, + "list_updated_queries_rps": 6.233333333333333, + "other_queries_rps": 9.45, + "/v0.1/servers_p95_seconds": 0.08863307704565375, + "/v0/servers_p95_seconds": 0.09108580508474579, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009662393162393161, + "list_page_acquire_p95_seconds": 0.0009523697916666668, + "list_search_acquire_p95_seconds": 0.0009692367601246109, + "list_updated_acquire_p95_seconds": 0.0009534420289855073, + "other_acquire_p95_seconds": 0.0009774226804123711, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.33409031783448884, + "node_cpu_apu6": 0.41971843852904084 + }, + { + "at": "2026-10-28T20:03:00+00:00", + "origin_arrivals_rps": 147.675, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.4666666666666667, + "list_page_queries_rps": 133.125, + "list_search_queries_rps": 2.458333333333333, + "list_updated_queries_rps": 1.7833333333333332, + "other_queries_rps": 9.558333333333334, + "/v0.1/servers_p95_seconds": 0.08954671401236385, + "/v0/servers_p95_seconds": 0.09150960555896176, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00096484375, + "list_page_acquire_p95_seconds": 0.0009527938914820712, + "list_search_acquire_p95_seconds": 0.000982323651452282, + "list_updated_acquire_p95_seconds": 0.0009532758620689654, + "other_acquire_p95_seconds": 0.000972981464027647, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.393336287305768, + "node_cpu_apu6": 0.44035434738301166 + }, + { + "at": "2026-10-28T20:04:00+00:00", + "origin_arrivals_rps": 158.3083333333334, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.4, + "list_page_queries_rps": 141.675, + "list_search_queries_rps": 1.6583333333333332, + "list_updated_queries_rps": 0.9583333333333333, + "other_queries_rps": 13.466666666666667, + "/v0.1/servers_p95_seconds": 0.09029019434628975, + "/v0/servers_p95_seconds": 0.09199552772808586, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009645038167938933, + "list_page_acquire_p95_seconds": 0.0009530857781358, + "list_search_acquire_p95_seconds": 0.0009936224489795918, + "list_updated_acquire_p95_seconds": 0.0009531215772179627, + "other_acquire_p95_seconds": 0.0009709514807813484, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.5172916165630643, + "node_cpu_apu6": 0.43190105950974006 + }, + { + "at": "2026-10-28T20:05:00+00:00", + "origin_arrivals_rps": 164.125, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 1.5, + "list_queries_rps": 1.875, + "list_page_queries_rps": 139.88333333333333, + "list_search_queries_rps": 1.1833333333333333, + "list_updated_queries_rps": 0.6666666666666667, + "other_queries_rps": 32.208333333333336, + "/v0.1/servers_p95_seconds": 0.09073411192065145, + "/v0/servers_p95_seconds": 0.09244580452251919, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009569852941176469, + "list_page_acquire_p95_seconds": 0.0009531868741645017, + "list_search_acquire_p95_seconds": 0.003504761904761901, + "list_updated_acquire_p95_seconds": 0.0009532349602724176, + "other_acquire_p95_seconds": 0.0009766313309776207, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.5117930636193099, + "node_cpu_apu6": 0.4197962527970958 + }, + { + "at": "2026-10-28T20:06:00+00:00", + "origin_arrivals_rps": 149.2583333333333, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 3.8916666666666666, + "list_queries_rps": 4.341666666666667, + "list_page_queries_rps": 121.1, + "list_search_queries_rps": 2.1166666666666667, + "list_updated_queries_rps": 0.575, + "other_queries_rps": 51.875, + "/v0.1/servers_p95_seconds": 0.09095866362898694, + "/v0/servers_p95_seconds": 0.09265194314501617, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009531016451983166, + "list_search_acquire_p95_seconds": 0.006352941176470543, + "list_updated_acquire_p95_seconds": 0.0009527859237536656, + "other_acquire_p95_seconds": 0.0009792663153627887, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.4075442732787148, + "node_cpu_apu6": 0.3844439912762027 + }, + { + "at": "2026-10-28T20:07:00+00:00", + "origin_arrivals_rps": 135.36666666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 2.4416666666666664, + "list_queries_rps": 2.791666666666667, + "list_page_queries_rps": 111.45833333333334, + "list_search_queries_rps": 3.8083333333333336, + "list_updated_queries_rps": 0.8416666666666667, + "other_queries_rps": 35.53333333333333, + "/v0.1/servers_p95_seconds": 0.09116133036693104, + "/v0/servers_p95_seconds": 0.09248751037344397, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009499999999999999, + "list_page_acquire_p95_seconds": 0.0009530080598733449, + "list_search_acquire_p95_seconds": 0.0027888888888888664, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009843915845337379, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.36206970441853625, + "node_cpu_apu6": 0.37032128360636607 + }, + { + "at": "2026-10-28T20:08:00+00:00", + "origin_arrivals_rps": 138.75833333333338, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.3666666666666667, + "list_page_queries_rps": 119.95, + "list_search_queries_rps": 3.5833333333333335, + "list_updated_queries_rps": 0.9166666666666666, + "other_queries_rps": 13.9, + "/v0.1/servers_p95_seconds": 0.09115924473931107, + "/v0/servers_p95_seconds": 0.09221138050860843, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009499999999999999, + "list_page_acquire_p95_seconds": 0.0009528774111270524, + "list_search_acquire_p95_seconds": 0.0009899719495091164, + "list_updated_acquire_p95_seconds": 0.0009499999999999999, + "other_acquire_p95_seconds": 0.0009874140813672671, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.3914110689217826, + "node_cpu_apu6": 0.4244650266092018 + }, + { + "at": "2026-10-28T20:09:00+00:00", + "origin_arrivals_rps": 150.35833333333335, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.425, + "list_page_queries_rps": 133.55, + "list_search_queries_rps": 2.791666666666667, + "list_updated_queries_rps": 1.0333333333333332, + "other_queries_rps": 12.425, + "/v0.1/servers_p95_seconds": 0.09089804084884479, + "/v0/servers_p95_seconds": 0.09190236752263617, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009577235772357723, + "list_page_acquire_p95_seconds": 0.0009525239513726752, + "list_search_acquire_p95_seconds": 0.000986061684460261, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009876822866031502, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.37571846642264667, + "node_cpu_apu6": 0.44709712632417603 + }, + { + "at": "2026-10-28T20:10:00+00:00", + "origin_arrivals_rps": 159.81666666666666, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.4916666666666667, + "list_page_queries_rps": 142.175, + "list_search_queries_rps": 2.4916666666666667, + "list_updated_queries_rps": 1.5833333333333333, + "other_queries_rps": 13.158333333333333, + "/v0.1/servers_p95_seconds": 0.0910552039904705, + "/v0/servers_p95_seconds": 0.09179960769425463, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009576000000000001, + "list_page_acquire_p95_seconds": 0.0009523792576188969, + "list_search_acquire_p95_seconds": 0.0009782608695652173, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.0009890470723306545, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.4811542358237299, + "node_cpu_apu6": 0.4471473048106418 + }, + { + "at": "2026-10-28T20:11:00+00:00", + "origin_arrivals_rps": 159.47500000000002, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.55, + "list_page_queries_rps": 139.825, + "list_search_queries_rps": 1.8833333333333333, + "list_updated_queries_rps": 1.5583333333333333, + "other_queries_rps": 15.616666666666667, + "/v0.1/servers_p95_seconds": 0.09106565207731615, + "/v0/servers_p95_seconds": 0.09177795436485411, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009646153846153845, + "list_page_acquire_p95_seconds": 0.0009527180655779212, + "list_search_acquire_p95_seconds": 0.000977703523693803, + "list_updated_acquire_p95_seconds": 0.0009526170798898073, + "other_acquire_p95_seconds": 0.0009958994053518334, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.4575765385132399, + "node_cpu_apu6": 0.4181248912141503 + }, + { + "at": "2026-10-28T20:12:00+00:00", + "origin_arrivals_rps": 161.08333333333337, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.008333333333333333, + "list_queries_rps": 0.5, + "list_page_queries_rps": 143.51666666666665, + "list_search_queries_rps": 1.6583333333333332, + "list_updated_queries_rps": 0.9833333333333334, + "other_queries_rps": 14.200000000000001, + "/v0.1/servers_p95_seconds": 0.09078803483665797, + "/v0/servers_p95_seconds": 0.09201170674859627, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009636690647482014, + "list_page_acquire_p95_seconds": 0.0009526389520821857, + "list_search_acquire_p95_seconds": 0.000983709677419355, + "list_updated_acquire_p95_seconds": 0.0009552341597796146, + "other_acquire_p95_seconds": 0.0009913151041666664, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.38916595475942206, + "node_cpu_apu6": 0.4472246803777428 + }, + { + "at": "2026-10-28T20:13:00+00:00", + "origin_arrivals_rps": 157.52499999999995, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.4083333333333333, + "list_page_queries_rps": 141.375, + "list_search_queries_rps": 5.958333333333334, + "list_updated_queries_rps": 0.7916666666666666, + "other_queries_rps": 8.941666666666666, + "/v0.1/servers_p95_seconds": 0.0904467904782961, + "/v0/servers_p95_seconds": 0.09215072294063742, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.000964074074074074, + "list_page_acquire_p95_seconds": 0.0009526878269139325, + "list_search_acquire_p95_seconds": 0.0009705776173285198, + "list_updated_acquire_p95_seconds": 0.0009554755043227665, + "other_acquire_p95_seconds": 0.0009899, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.3210972899134504, + "node_cpu_apu6": 0.3610001559080653 + }, + { + "at": "2026-10-28T20:14:00+00:00", + "origin_arrivals_rps": 134.64166666666665, + "origin_499_rps": 0.041666666666666664, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.4166666666666667, + "list_page_queries_rps": 119.54166666666666, + "list_search_queries_rps": 5.958333333333333, + "list_updated_queries_rps": 0.6916666666666667, + "other_queries_rps": 8.116666666666667, + "/v0.1/servers_p95_seconds": 0.08999281842818428, + "/v0/servers_p95_seconds": 0.09221372761491958, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009568345323741007, + "list_page_acquire_p95_seconds": 0.0009527022306502324, + "list_search_acquire_p95_seconds": 0.0009651093439363817, + "list_updated_acquire_p95_seconds": 0.0009559006211180125, + "other_acquire_p95_seconds": 0.0009884442446043168, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.32622390665807566, + "node_cpu_apu6": 0.3389421819583125 + }, + { + "at": "2026-10-28T20:15:00+00:00", + "origin_arrivals_rps": 125.15833333333335, + "origin_499_rps": 0.041666666666666664, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.05, + "list_queries_rps": 0.43333333333333335, + "list_page_queries_rps": 110.53333333333333, + "list_search_queries_rps": 2.4749999999999996, + "list_updated_queries_rps": 0.45833333333333337, + "other_queries_rps": 11.466666666666667, + "/v0.1/servers_p95_seconds": 0.0892524822983641, + "/v0/servers_p95_seconds": 0.09210428728977117, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009647286821705426, + "list_page_acquire_p95_seconds": 0.0009526806709515598, + "list_search_acquire_p95_seconds": 0.0009567736185383244, + "list_updated_acquire_p95_seconds": 0.0009589622641509435, + "other_acquire_p95_seconds": 0.000985632183908046, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.360386742275053, + "node_cpu_apu6": 0.36657987200281544 + }, + { + "at": "2026-10-28T20:16:00+00:00", + "origin_arrivals_rps": 136.44166666666663, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.049999999999999996, + "list_queries_rps": 0.375, + "list_page_queries_rps": 120.43333333333334, + "list_search_queries_rps": 2.875, + "list_updated_queries_rps": 0.8916666666666666, + "other_queries_rps": 11.85, + "/v0.1/servers_p95_seconds": 0.08783388172326055, + "/v0/servers_p95_seconds": 0.09205495057798625, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009581196581196582, + "list_page_acquire_p95_seconds": 0.0009522633418043202, + "list_search_acquire_p95_seconds": 0.0009626554174067494, + "list_updated_acquire_p95_seconds": 0.0009578512396694215, + "other_acquire_p95_seconds": 0.0009726113566814007, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.29201367161779973, + "node_cpu_apu6": 0.3888908733941207 + }, + { + "at": "2026-10-28T20:17:00+00:00", + "origin_arrivals_rps": 143.54166666666666, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.35833333333333334, + "list_page_queries_rps": 131.81666666666666, + "list_search_queries_rps": 1.775, + "list_updated_queries_rps": 1.15, + "other_queries_rps": 8.25, + "/v0.1/servers_p95_seconds": 0.08703549597110312, + "/v0/servers_p95_seconds": 0.09170495629717916, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009586363636363635, + "list_page_acquire_p95_seconds": 0.0009524469931131717, + "list_search_acquire_p95_seconds": 0.000962621789193977, + "list_updated_acquire_p95_seconds": 0.0009540772532188842, + "other_acquire_p95_seconds": 0.0009726190476190477, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.3397397982798187, + "node_cpu_apu6": 0.3771653610596938 + }, + { + "at": "2026-10-28T20:18:00+00:00", + "origin_arrivals_rps": 131.09166666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.03333333333333333, + "list_queries_rps": 0.3666666666666667, + "list_page_queries_rps": 119.29166666666666, + "list_search_queries_rps": 1.4916666666666667, + "list_updated_queries_rps": 1.05, + "other_queries_rps": 8.966666666666667, + "/v0.1/servers_p95_seconds": 0.08689759350674012, + "/v0/servers_p95_seconds": 0.09114361416689024, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009587962962962962, + "list_page_acquire_p95_seconds": 0.0009524069803854584, + "list_search_acquire_p95_seconds": 0.0009708614864864865, + "list_updated_acquire_p95_seconds": 0.0009534671532846716, + "other_acquire_p95_seconds": 0.0009730362021857925, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.30564727782887835, + "node_cpu_apu6": 0.33269270741366197 + }, + { + "at": "2026-10-28T20:19:00+00:00", + "origin_arrivals_rps": 120.75833333333334, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.04166666666666667, + "list_queries_rps": 0.3416666666666667, + "list_page_queries_rps": 105.79166666666666, + "list_search_queries_rps": 1.7083333333333335, + "list_updated_queries_rps": 1.1833333333333333, + "other_queries_rps": 11.833333333333332, + "/v0.1/servers_p95_seconds": 0.08658499095840871, + "/v0/servers_p95_seconds": 0.09100927548933684, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009597938144329895, + "list_page_acquire_p95_seconds": 0.0009521021276595743, + "list_search_acquire_p95_seconds": 0.0009698872785829306, + "list_updated_acquire_p95_seconds": 0.0009565292096219932, + "other_acquire_p95_seconds": 0.0009750896641669383, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.28818883907467047, + "node_cpu_apu6": 0.34304319748456086 + }, + { + "at": "2026-10-28T20:20:00+00:00", + "origin_arrivals_rps": 131.98333333333338, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.32499999999999996, + "list_page_queries_rps": 119.78333333333333, + "list_search_queries_rps": 2.025, + "list_updated_queries_rps": 0.8583333333333333, + "other_queries_rps": 8.966666666666667, + "/v0.1/servers_p95_seconds": 0.08672134441501508, + "/v0/servers_p95_seconds": 0.09118361945636626, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009499999999999999, + "list_page_acquire_p95_seconds": 0.0009520179310344827, + "list_search_acquire_p95_seconds": 0.0009765858208955224, + "list_updated_acquire_p95_seconds": 0.0009559190031152647, + "other_acquire_p95_seconds": 0.0009722012815680363, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.416897066848954, + "node_cpu_apu6": 0.39757469295412495 + }, + { + "at": "2026-10-28T20:21:00+00:00", + "origin_arrivals_rps": 151.39166666666668, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.049999999999999996, + "list_queries_rps": 0.45, + "list_page_queries_rps": 141.50833333333333, + "list_search_queries_rps": 1.85, + "list_updated_queries_rps": 0.5416666666666667, + "other_queries_rps": 6.866666666666666, + "/v0.1/servers_p95_seconds": 0.08781536808821144, + "/v0/servers_p95_seconds": 0.09122686392075079, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009587962962962963, + "list_page_acquire_p95_seconds": 0.0009522890354012761, + "list_search_acquire_p95_seconds": 0.0009650793650793648, + "list_updated_acquire_p95_seconds": 0.0009537999999999999, + "other_acquire_p95_seconds": 0.0009735223642172523, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.42341966646611884, + "node_cpu_apu6": 0.3905308523741648 + }, + { + "at": "2026-10-28T20:22:00+00:00", + "origin_arrivals_rps": 149.75833333333335, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.5, + "list_page_queries_rps": 134.85, + "list_search_queries_rps": 1.475, + "list_updated_queries_rps": 0.6333333333333333, + "other_queries_rps": 12.191666666666666, + "/v0.1/servers_p95_seconds": 0.08842609817101767, + "/v0/servers_p95_seconds": 0.09137831221198156, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009584821428571428, + "list_page_acquire_p95_seconds": 0.0009521799704966399, + "list_search_acquire_p95_seconds": 0.0009726640159045725, + "list_updated_acquire_p95_seconds": 0.000953653846153846, + "other_acquire_p95_seconds": 0.0009760736196319019, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.35563863926702954, + "node_cpu_apu6": 0.3406044961079412 + }, + { + "at": "2026-10-28T20:23:00+00:00", + "origin_arrivals_rps": 124.98333333333333, + "origin_499_rps": 0.008333333333333333, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.4083333333333333, + "list_page_queries_rps": 103.71666666666667, + "list_search_queries_rps": 2.8583333333333334, + "list_updated_queries_rps": 1.0333333333333332, + "other_queries_rps": 17.0, + "/v0.1/servers_p95_seconds": 0.08857860463252387, + "/v0/servers_p95_seconds": 0.09176515151515151, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009583333333333333, + "list_page_acquire_p95_seconds": 0.000952249115672785, + "list_search_acquire_p95_seconds": 0.0009805045871559633, + "list_updated_acquire_p95_seconds": 0.0009576923076923078, + "other_acquire_p95_seconds": 0.0009837195658550894, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.3211285792157772, + "node_cpu_apu6": 0.34617538125385183 + }, + { + "at": "2026-10-28T20:24:00+00:00", + "origin_arrivals_rps": 120.58333333333334, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.075, + "list_queries_rps": 0.38333333333333336, + "list_page_queries_rps": 102.00833333333333, + "list_search_queries_rps": 3.158333333333333, + "list_updated_queries_rps": 1.55, + "other_queries_rps": 13.658333333333335, + "/v0.1/servers_p95_seconds": 0.0889269045842659, + "/v0/servers_p95_seconds": 0.09103634003893578, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009581196581196582, + "list_page_acquire_p95_seconds": 0.00095233905994965, + "list_search_acquire_p95_seconds": 0.0009842857142857142, + "list_updated_acquire_p95_seconds": 0.0009562706270627061, + "other_acquire_p95_seconds": 0.0009818889227952395, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.3032904632770321, + "node_cpu_apu6": 0.3709841731421572 + }, + { + "at": "2026-10-28T20:25:00+00:00", + "origin_arrivals_rps": 131.39999999999998, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.06666666666666667, + "list_queries_rps": 0.425, + "list_page_queries_rps": 119.75, + "list_search_queries_rps": 1.6833333333333333, + "list_updated_queries_rps": 1.3166666666666667, + "other_queries_rps": 8.483333333333333, + "/v0.1/servers_p95_seconds": 0.08853669004984312, + "/v0/servers_p95_seconds": 0.08993540302782321, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009576612903225805, + "list_page_acquire_p95_seconds": 0.0009524350830268582, + "list_search_acquire_p95_seconds": 0.0009857610474631752, + "list_updated_acquire_p95_seconds": 0.0009562913907284767, + "other_acquire_p95_seconds": 0.0009801768055139346, + "sampled_inflight": 2.0, + "node_cpu_gc71": 0.3185789478967005, + "node_cpu_apu6": 0.3733129543637915 + }, + { + "at": "2026-10-28T20:26:00+00:00", + "origin_arrivals_rps": 140.70000000000002, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.3666666666666667, + "list_page_queries_rps": 131.14999999999998, + "list_search_queries_rps": 1.925, + "list_updated_queries_rps": 0.5583333333333333, + "other_queries_rps": 6.566666666666666, + "/v0.1/servers_p95_seconds": 0.08790649272508474, + "/v0/servers_p95_seconds": 0.08935840336134453, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009522860089377792, + "list_search_acquire_p95_seconds": 0.000988283582089552, + "list_updated_acquire_p95_seconds": 0.0009562295081967214, + "other_acquire_p95_seconds": 0.0009796050601666153, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.3667483320059546, + "node_cpu_apu6": 0.38602686816302595 + }, + { + "at": "2026-10-28T20:27:00+00:00", + "origin_arrivals_rps": 151.22500000000002, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.35, + "list_page_queries_rps": 141.425, + "list_search_queries_rps": 2.5166666666666666, + "list_updated_queries_rps": 0.4916666666666667, + "other_queries_rps": 6.208333333333334, + "/v0.1/servers_p95_seconds": 0.08742105110336817, + "/v0/servers_p95_seconds": 0.08923077835785474, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009525125422619697, + "list_search_acquire_p95_seconds": 0.0009836971350613917, + "list_updated_acquire_p95_seconds": 0.0009566666666666668, + "other_acquire_p95_seconds": 0.0009749906050357008, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.36197980532209234, + "node_cpu_apu6": 0.40184516488297956 + }, + { + "at": "2026-10-28T20:28:00+00:00", + "origin_arrivals_rps": 157.21666666666667, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.016666666666666666, + "list_queries_rps": 0.5, + "list_page_queries_rps": 147.89166666666665, + "list_search_queries_rps": 1.9833333333333334, + "list_updated_queries_rps": 0.525, + "other_queries_rps": 6.233333333333333, + "/v0.1/servers_p95_seconds": 0.0867529357586921, + "/v0/servers_p95_seconds": 0.08869666225555066, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009523635368462953, + "list_search_acquire_p95_seconds": 0.0009763431542461007, + "list_updated_acquire_p95_seconds": 0.0009538775510204082, + "other_acquire_p95_seconds": 0.0009602000976085896, + "sampled_inflight": 1.0, + "node_cpu_gc71": 0.30582408750221174, + "node_cpu_apu6": 0.39401532581517434 + }, + { + "at": "2026-10-28T20:29:00+00:00", + "origin_arrivals_rps": 156.775, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.025, + "list_queries_rps": 0.4083333333333333, + "list_page_queries_rps": 147.52499999999998, + "list_search_queries_rps": 1.75, + "list_updated_queries_rps": 0.825, + "other_queries_rps": 6.3, + "/v0.1/servers_p95_seconds": 0.0862369865338916, + "/v0/servers_p95_seconds": 0.08838734251391736, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.0009499999999999999, + "list_page_acquire_p95_seconds": 0.0009524581018848216, + "list_search_acquire_p95_seconds": 0.0009714038128249566, + "list_updated_acquire_p95_seconds": 0.00095, + "other_acquire_p95_seconds": 0.000955077498663816, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.2907251540937674, + "node_cpu_apu6": 0.3931825326869633 + }, + { + "at": "2026-10-28T20:30:00+00:00", + "origin_arrivals_rps": 138.67499999999998, + "origin_499_rps": 0.0, + "origin_5xx_rps": 0.0, + "publish_completions_rps": 0.041666666666666664, + "list_queries_rps": 0.20833333333333331, + "list_page_queries_rps": 129.28333333333333, + "list_search_queries_rps": 1.9666666666666668, + "list_updated_queries_rps": 1.1333333333333333, + "other_queries_rps": 6.183333333333334, + "/v0.1/servers_p95_seconds": 0.08634292725173207, + "/v0/servers_p95_seconds": 0.08833021232674726, + "sampled_pending_acquisitions": 0.0, + "list_acquire_p95_seconds": 0.00095, + "list_page_acquire_p95_seconds": 0.0009525516623917147, + "list_search_acquire_p95_seconds": 0.0009716965742251222, + "list_updated_acquire_p95_seconds": 0.0009500000000000001, + "other_acquire_p95_seconds": 0.0009552573325954621, + "sampled_inflight": 0.0, + "node_cpu_gc71": 0.2485850343691709, + "node_cpu_apu6": 0.346668590981911 + } + ] +} diff --git a/tools/ingressbench/evidence/workload-profile-2026-10-28/profile-summary.json b/tools/ingressbench/evidence/workload-profile-2026-10-28/profile-summary.json new file mode 100644 index 000000000..fc8447598 --- /dev/null +++ b/tools/ingressbench/evidence/workload-profile-2026-10-28/profile-summary.json @@ -0,0 +1,1707 @@ +{ + "date": "2026-10-28", + "production_changed": false, + "staging_configuration_changed": false, + "temporary_generator_created": false, + "purpose": "Calibrate workload fidelity before capacity experiments. Not a production-fix verdict.", + "origin_history": { + "from": "2026-10-28T16:30:00Z", + "to": "2026-10-28T20:30:00Z", + "points": 241 + }, + "ingress_coverage": { + "sampled_minutes": 17, + "span_minutes": 241, + "all_queries_paginated_to_completion": true, + "records": 962375, + "parsed_access_records": 959671, + "unparsed_records": 2704, + "client_records": 870686, + "scope": "One-minute slices every 15 minutes, not continuous ingress history or exact request-arrival replay." + }, + "samples": [ + { + "start": "2026-10-28T16:30:00Z", + "end": "2026-10-28T16:31:00Z", + "query_complete": true, + "entries": 23728, + "parsed": 23727, + "unparsed": 1, + "client_records": 18455, + "client_rps": 307.5833333333333, + "background_proxy_rps": 87.86666666666666, + "cache_hit_percent": 29.99729070712544, + "page_v01_limit100_percent": 68.34512181430335, + "page_top128_percent": 5.995284607611991, + "distinct_page_uris": 9524, + "http2_percent": 8.219994581414252, + "kind_counts": { + "page": 17814, + "search": 243, + "detail": 277, + "updated": 46, + "other": 71, + "publish": 4 + }, + "cache_counts": { + "HIT": 5536, + "STALE": 5272, + "MISS": 7294, + "none": 353 + }, + "response_bytes": { + "mean": 83359.19804930913, + "p50": 84693, + "p95": 159722, + "p99": 277261, + "max": 1360301 + }, + "latency_seconds": { + "mean": 0.012063126523977523, + "p50": 0.007, + "p95": 0.027, + "p99": 0.108, + "max": 4.14 + }, + "page_filter_counts": { + "v0.1|100|latest|active_only|known_params_only": 881, + "v0.1|100|none|active_only|known_params_only": 11279, + "v0.1|default|latest|active_only|known_params_only": 4240, + "v0|100|none|active_only|known_params_only": 640, + "v0|100|latest|active_only|known_params_only": 409, + "v0.1|default|none|active_only|known_params_only": 299, + "v0|60|latest|active_only|known_params_only": 2, + "v0.1|50|none|active_only|known_params_only": 20, + "v0.1|1|none|active_only|known_params_only": 9, + "v0|default|none|active_only|known_params_only": 4, + "v0.1|96|latest|active_only|known_params_only": 3, + "v0.1|100|none|include_deleted|known_params_only": 6, + "v0|1|none|active_only|known_params_only": 1, + "v0|7|none|active_only|known_params_only": 2, + "v0.1|50|latest|include_deleted|known_params_only": 1, + "v0|24|none|active_only|known_params_only": 1, + "v0.1|50|latest|active_only|known_params_only": 3, + "v0|50|none|active_only|known_params_only": 2, + "v0|1|latest|active_only|known_params_only": 1, + "v0|2|none|active_only|known_params_only": 1, + "v0.1|100|latest|include_deleted|known_params_only": 9, + "v0|25|latest|active_only|known_params_only": 1 + }, + "updated_age_counts": { + "<1h": 14, + "<1d": 21, + ">=7d": 9, + "invalid": 2 + }, + "search_length_counts": { + "3": 10, + "6": 35, + "13": 10, + "7": 11, + "15": 14, + "12": 10, + "19": 7, + "8": 8, + "1": 19, + "2": 6, + "10": 10, + "5": 15, + "4": 8, + "9": 14, + "11": 18, + ">20": 21, + "16": 5, + "20": 1, + "17": 17, + "18": 3, + "14": 1 + } + }, + { + "start": "2026-10-28T16:45:00Z", + "end": "2026-10-28T16:46:00Z", + "query_complete": true, + "entries": 18142, + "parsed": 18142, + "unparsed": 0, + "client_records": 13935, + "client_rps": 232.25, + "background_proxy_rps": 70.11666666666666, + "cache_hit_percent": 25.475421600287046, + "page_v01_limit100_percent": 74.19892432391485, + "page_top128_percent": 6.6888872055147335, + "distinct_page_uris": 8089, + "http2_percent": 8.532472192321492, + "kind_counts": { + "page": 13201, + "search": 219, + "detail": 391, + "other": 95, + "updated": 29 + }, + "cache_counts": { + "MISS": 5692, + "STALE": 4207, + "HIT": 3550, + "none": 486 + }, + "response_bytes": { + "mean": 96452.21334768568, + "p50": 91947, + "p95": 172918, + "p99": 306417, + "max": 1356748 + }, + "latency_seconds": { + "mean": 0.014141514172946203, + "p50": 0.008, + "p95": 0.022, + "p99": 0.124, + "max": 10.301 + }, + "page_filter_counts": { + "v0.1|100|none|active_only|known_params_only": 9579, + "v0|30|latest|active_only|known_params_only": 277, + "v0.1|default|latest|active_only|known_params_only": 571, + "v0|100|none|active_only|known_params_only": 1725, + "v0|100|none|include_deleted|known_params_only": 188, + "v0|100|latest|active_only|known_params_only": 554, + "v0.1|100|latest|active_only|known_params_only": 200, + "v0.1|default|none|active_only|known_params_only": 59, + "v0.1|100|none|include_deleted|known_params_only": 15, + "v0|default|none|active_only|known_params_only": 4, + "v0.1|96|latest|active_only|known_params_only": 2, + "v0.1|1|none|active_only|known_params_only": 8, + "v0.1|50|latest|include_deleted|known_params_only": 2, + "v0|7|none|active_only|known_params_only": 5, + "v0|50|none|active_only|known_params_only": 1, + "v0|1|latest|active_only|known_params_only": 1, + "v0|30|none|active_only|other_params": 1, + "v0.1|100|latest|include_deleted|known_params_only": 1, + "v0|1|none|active_only|known_params_only": 1, + "v0|20|none|active_only|other_params": 1, + "v0.1|91|latest|active_only|known_params_only": 2, + "v0.1|25|latest|active_only|known_params_only": 2, + "v0.1|19|latest|active_only|known_params_only": 1, + "v0.1|5|latest|active_only|known_params_only": 1 + }, + "updated_age_counts": { + "<1h": 13, + "<1d": 13, + ">=7d": 2, + "invalid": 1 + }, + "search_length_counts": { + "9": 13, + "14": 4, + "11": 10, + "13": 9, + "5": 12, + "8": 8, + "10": 16, + "6": 23, + ">20": 40, + "15": 5, + "4": 11, + "12": 5, + "20": 6, + "7": 10, + "16": 10, + "1": 17, + "2": 1, + "3": 6, + "19": 8, + "18": 1, + "17": 4 + } + }, + { + "start": "2026-10-28T17:00:00Z", + "end": "2026-10-28T17:01:00Z", + "query_complete": true, + "entries": 108267, + "parsed": 107800, + "unparsed": 467, + "client_records": 106975, + "client_rps": 1782.9166666666667, + "background_proxy_rps": 13.75, + "cache_hit_percent": 93.88922645477915, + "page_v01_limit100_percent": 99.29558128745775, + "page_top128_percent": 98.4699722587792, + "distinct_page_uris": 1547, + "http2_percent": 0.3047440990885721, + "kind_counts": { + "page": 105619, + "search": 94, + "detail": 349, + "updated": 875, + "other": 38 + }, + "cache_counts": { + "STALE": 839, + "HIT": 100438, + "MISS": 1899, + "none": 557, + "UPDATING": 3242 + }, + "response_bytes": { + "mean": 88532.06332320636, + "p50": 81304, + "p95": 126067, + "p99": 126067, + "max": 1384553 + }, + "latency_seconds": { + "mean": 3.5404076653431464, + "p50": 2.52, + "p95": 10.942, + "p99": 18.458, + "max": 51.238 + }, + "page_filter_counts": { + "v0.1|100|none|active_only|known_params_only": 103749, + "v0.1|100|latest|active_only|known_params_only": 1117, + "v0.1|default|latest|active_only|known_params_only": 235, + "v0|100|none|active_only|known_params_only": 253, + "v0|100|latest|active_only|known_params_only": 52, + "v0.1|100|none|include_deleted|known_params_only": 8, + "v0|invalid|none|active_only|known_params_only": 11, + "v0|50|none|active_only|known_params_only": 83, + "v0.1|default|none|active_only|known_params_only": 5, + "v0|default|none|active_only|known_params_only": 90, + "v0|default|none|active_only|other_params": 2, + "v0.1|25|latest|active_only|known_params_only": 1, + "v0.1|50|none|active_only|known_params_only": 1, + "v0|7|none|active_only|known_params_only": 1, + "v0.1|100|latest|include_deleted|known_params_only": 1, + "v0.1|50|latest|include_deleted|known_params_only": 1, + "v0.1|1|none|active_only|known_params_only": 3, + "v0.1|96|latest|active_only|known_params_only": 1, + "v0|60|latest|active_only|known_params_only": 3, + "v0.1|20|latest|active_only|known_params_only": 1, + "v0.1|20|none|active_only|known_params_only": 1 + }, + "updated_age_counts": { + "<7d": 822, + "<1d": 34, + "<1h": 15, + ">=7d": 4 + }, + "search_length_counts": { + "4": 5, + "12": 3, + ">20": 18, + "8": 3, + "6": 11, + "5": 7, + "16": 4, + "13": 4, + "19": 2, + "11": 9, + "7": 5, + "17": 4, + "14": 3, + "15": 3, + "3": 6, + "9": 5, + "10": 1, + "20": 1 + } + }, + { + "start": "2026-10-28T17:15:00Z", + "end": "2026-10-28T17:16:00Z", + "query_complete": true, + "entries": 122439, + "parsed": 121756, + "unparsed": 683, + "client_records": 115601, + "client_rps": 1926.6833333333334, + "background_proxy_rps": 102.58333333333333, + "cache_hit_percent": 86.98367661179401, + "page_v01_limit100_percent": 94.08772822140764, + "page_top128_percent": 56.01838048301384, + "distinct_page_uris": 9947, + "http2_percent": 1.3572546950285898, + "kind_counts": { + "page": 114034, + "detail": 1281, + "updated": 50, + "search": 160, + "other": 74, + "publish": 2 + }, + "cache_counts": { + "HIT": 100554, + "MISS": 6633, + "STALE": 6156, + "UPDATING": 218, + "none": 2040 + }, + "response_bytes": { + "mean": 89963.42442539425, + "p50": 84380, + "p95": 133768, + "p99": 191858, + "max": 1379550 + }, + "latency_seconds": { + "mean": 0.03159180283908351, + "p50": 0.009, + "p95": 0.141, + "p99": 0.305, + "max": 5.537 + }, + "page_filter_counts": { + "v0.1|100|none|active_only|known_params_only": 103992, + "v0|100|none|active_only|known_params_only": 3629, + "v0.1|default|latest|active_only|known_params_only": 1639, + "v0|100|none|include_deleted|known_params_only": 229, + "v0|100|latest|active_only|known_params_only": 730, + "v0.1|100|latest|active_only|known_params_only": 3295, + "v0.1|default|none|active_only|known_params_only": 284, + "v0.1|1|none|active_only|known_params_only": 6, + "v0|default|none|active_only|known_params_only": 6, + "v0|30|latest|active_only|known_params_only": 106, + "v0|60|latest|active_only|known_params_only": 87, + "v0|50|none|active_only|known_params_only": 2, + "v0.1|100|none|include_deleted|known_params_only": 5, + "v0|5|none|active_only|known_params_only": 2, + "v0.1|96|latest|active_only|known_params_only": 1, + "v0.1|24|latest|active_only|known_params_only": 1, + "v0.1|50|latest|include_deleted|known_params_only": 1, + "v0|default|latest|active_only|known_params_only": 15, + "v0|20|none|active_only|known_params_only": 2, + "v0.1|25|latest|active_only|known_params_only": 2 + }, + "updated_age_counts": { + "<1h": 10, + "<1d": 27, + ">=7d": 2, + "<7d": 11 + }, + "search_length_counts": { + "12": 5, + "6": 18, + "4": 10, + ">20": 26, + "20": 2, + "8": 14, + "18": 3, + "1": 17, + "2": 1, + "3": 6, + "7": 13, + "5": 7, + "10": 9, + "16": 4, + "11": 2, + "9": 15, + "19": 4, + "13": 2, + "14": 1, + "15": 1 + } + }, + { + "start": "2026-10-28T17:30:00Z", + "end": "2026-10-28T17:31:00Z", + "query_complete": true, + "entries": 115640, + "parsed": 115611, + "unparsed": 29, + "client_records": 108527, + "client_rps": 1808.7833333333333, + "background_proxy_rps": 118.06666666666666, + "cache_hit_percent": 85.80629705050356, + "page_v01_limit100_percent": 95.15727708248227, + "page_top128_percent": 25.31195454376648, + "distinct_page_uris": 9448, + "http2_percent": 0.9675011748228551, + "kind_counts": { + "page": 107708, + "search": 378, + "detail": 318, + "publish": 2, + "updated": 70, + "other": 51 + }, + "cache_counts": { + "HIT": 93123, + "MISS": 7475, + "STALE": 7083, + "UPDATING": 446, + "none": 400 + }, + "response_bytes": { + "mean": 103584.27665005022, + "p50": 94539, + "p95": 170397, + "p99": 319404, + "max": 1293163 + }, + "latency_seconds": { + "mean": 0.07947548536309758, + "p50": 0.015, + "p95": 0.371, + "p99": 0.697, + "max": 8.94 + }, + "page_filter_counts": { + "v0.1|100|none|active_only|known_params_only": 101975, + "v0.1|default|latest|active_only|known_params_only": 2144, + "v0|100|none|active_only|known_params_only": 2042, + "v0.1|100|latest|active_only|known_params_only": 322, + "v0.1|default|none|active_only|known_params_only": 295, + "v0|100|latest|active_only|known_params_only": 354, + "v0.1|100|none|active_only|other_params": 186, + "v0|100|none|include_deleted|known_params_only": 358, + "v0.1|1|none|active_only|known_params_only": 4, + "v0.1|100|none|include_deleted|known_params_only": 9, + "v0|1|none|active_only|known_params_only": 1, + "v0|default|none|active_only|known_params_only": 3, + "v0.1|30|latest|active_only|known_params_only": 2, + "v0|50|none|active_only|known_params_only": 1, + "v0.1|96|latest|active_only|known_params_only": 2, + "v0.1|50|none|active_only|known_params_only": 1, + "v0|7|none|active_only|known_params_only": 2, + "v0|5|none|active_only|known_params_only": 1, + "v0.1|10|none|active_only|known_params_only": 1, + "v0.1|30|latest|include_deleted|known_params_only": 1, + "v0|25|latest|active_only|known_params_only": 1, + "v0|1|latest|active_only|known_params_only": 1, + "v0|60|latest|active_only|known_params_only": 1, + "v0.1|20|none|active_only|known_params_only": 1 + }, + "updated_age_counts": { + "<1d": 32, + "<1h": 13, + "<7d": 19, + ">=7d": 6 + }, + "search_length_counts": { + ">20": 20, + "16": 12, + "13": 11, + "11": 28, + "4": 19, + "10": 45, + "8": 26, + "7": 24, + "3": 19, + "6": 52, + "17": 1, + "18": 3, + "5": 25, + "9": 22, + "1": 34, + "2": 2, + "14": 9, + "12": 18, + "15": 4, + "19": 3, + "20": 1 + } + }, + { + "start": "2026-10-28T17:45:00Z", + "end": "2026-10-28T17:46:00Z", + "query_complete": true, + "entries": 127345, + "parsed": 127282, + "unparsed": 63, + "client_records": 119014, + "client_rps": 1983.5666666666666, + "background_proxy_rps": 137.8, + "cache_hit_percent": 87.18722167140001, + "page_v01_limit100_percent": 98.19460072249812, + "page_top128_percent": 17.939780543311816, + "distinct_page_uris": 8560, + "http2_percent": 0.9385450451207421, + "kind_counts": { + "page": 118201, + "detail": 416, + "updated": 89, + "search": 228, + "other": 77, + "publish": 3 + }, + "cache_counts": { + "HIT": 103765, + "STALE": 8269, + "MISS": 4822, + "UPDATING": 1597, + "none": 561 + }, + "response_bytes": { + "mean": 110027.92472314181, + "p50": 98555, + "p95": 185557, + "p99": 347915, + "max": 1345425 + }, + "latency_seconds": { + "mean": 0.41493194078007445, + "p50": 0.241, + "p95": 1.339, + "p99": 2.405, + "max": 60.093 + }, + "page_filter_counts": { + "v0.1|100|none|active_only|known_params_only": 115240, + "v0|100|none|active_only|known_params_only": 912, + "v0.1|default|latest|active_only|known_params_only": 740, + "v0|100|latest|active_only|known_params_only": 279, + "v0.1|100|latest|active_only|known_params_only": 818, + "v0.1|100|none|include_deleted|known_params_only": 8, + "v0|100|none|include_deleted|known_params_only": 24, + "v0|default|none|active_only|known_params_only": 62, + "v0.1|default|none|active_only|known_params_only": 71, + "v0.1|50|none|active_only|known_params_only": 23, + "v0.1|100|latest|include_deleted|known_params_only": 1, + "v0.1|1|none|active_only|known_params_only": 11, + "v0|50|none|active_only|known_params_only": 2, + "v0|60|latest|active_only|known_params_only": 2, + "v0.1|30|latest|include_deleted|known_params_only": 1, + "v0|1|latest|active_only|known_params_only": 1, + "v0|1|none|active_only|known_params_only": 2, + "v0|7|none|active_only|known_params_only": 2, + "v0.1|9|none|active_only|known_params_only": 1, + "v0|default|latest|active_only|known_params_only": 1 + }, + "updated_age_counts": { + ">=7d": 41, + "<1h": 10, + "<1d": 24, + "<7d": 14 + }, + "search_length_counts": { + "6": 31, + "7": 13, + "15": 11, + ">20": 35, + "13": 9, + "10": 7, + "1": 17, + "3": 9, + "2": 5, + "11": 18, + "5": 5, + "17": 17, + "4": 11, + "8": 10, + "9": 14, + "18": 3, + "14": 3, + "16": 6, + "19": 3, + "12": 1 + } + }, + { + "start": "2026-10-28T18:00:00Z", + "end": "2026-10-28T18:01:00Z", + "query_complete": true, + "entries": 112640, + "parsed": 112582, + "unparsed": 58, + "client_records": 103322, + "client_rps": 1722.0333333333333, + "background_proxy_rps": 154.33333333333334, + "cache_hit_percent": 82.65422659259403, + "page_v01_limit100_percent": 97.1748874544691, + "page_top128_percent": 18.183061043133502, + "distinct_page_uris": 10483, + "http2_percent": 1.230134918023267, + "kind_counts": { + "page": 102403, + "detail": 399, + "updated": 192, + "search": 235, + "other": 91, + "publish": 2 + }, + "cache_counts": { + "HIT": 85400, + "none": 543, + "MISS": 6244, + "UPDATING": 1871, + "STALE": 9264 + }, + "response_bytes": { + "mean": 109676.03837517663, + "p50": 99774, + "p95": 181629, + "p99": 329439, + "max": 1008538 + }, + "latency_seconds": { + "mean": 0.4991061148640775, + "p50": 0.22, + "p95": 1.846, + "p99": 3.245, + "max": 15.483 + }, + "page_filter_counts": { + "v0.1|100|none|active_only|known_params_only": 95135, + "v0|100|latest|active_only|known_params_only": 475, + "v0.1|100|none|include_deleted|known_params_only": 81, + "v0.1|default|latest|active_only|known_params_only": 1336, + "v0.1|100|latest|active_only|known_params_only": 4253, + "v0|100|none|active_only|known_params_only": 972, + "v0.1|default|none|active_only|known_params_only": 71, + "v0|default|none|active_only|known_params_only": 8, + "v0|50|none|active_only|known_params_only": 4, + "v0.1|100|latest|include_deleted|known_params_only": 41, + "v0|40|latest|active_only|known_params_only": 3, + "v0.1|1|none|active_only|known_params_only": 12, + "v0|1|none|active_only|known_params_only": 1, + "v0|default|none|active_only|other_params": 2, + "v0|7|none|active_only|known_params_only": 2, + "v0.1|25|latest|active_only|known_params_only": 1, + "v0.1|96|latest|active_only|known_params_only": 1, + "v0.1|50|latest|include_deleted|known_params_only": 1, + "v0.1|30|latest|include_deleted|known_params_only": 1, + "v0|1|latest|active_only|known_params_only": 1, + "v0|30|none|active_only|known_params_only": 1, + "v0.1|20|none|active_only|known_params_only": 1 + }, + "updated_age_counts": { + ">=7d": 34, + "<1h": 15, + "<1d": 122, + "<7d": 20, + "invalid": 1 + }, + "search_length_counts": { + "12": 9, + "6": 31, + "13": 6, + "11": 6, + "5": 14, + ">20": 38, + "1": 21, + "2": 9, + "16": 16, + "3": 16, + "7": 13, + "4": 19, + "10": 6, + "8": 12, + "9": 8, + "19": 1, + "17": 3, + "14": 3, + "20": 1, + "18": 1, + "15": 2 + } + }, + { + "start": "2026-10-28T18:15:00Z", + "end": "2026-10-28T18:16:00Z", + "query_complete": true, + "entries": 100722, + "parsed": 100094, + "unparsed": 628, + "client_records": 92229, + "client_rps": 1537.15, + "background_proxy_rps": 131.08333333333334, + "cache_hit_percent": 79.8794305478754, + "page_v01_limit100_percent": 95.57445025254196, + "page_top128_percent": 15.83184454884316, + "distinct_page_uris": 11598, + "http2_percent": 1.0018540806037146, + "kind_counts": { + "page": 90678, + "detail": 1250, + "search": 116, + "updated": 102, + "other": 79, + "publish": 4 + }, + "cache_counts": { + "HIT": 73672, + "MISS": 7657, + "STALE": 7866, + "none": 1962, + "UPDATING": 1072 + }, + "response_bytes": { + "mean": 104015.96119441824, + "p50": 93810, + "p95": 183795, + "p99": 372514, + "max": 1381424 + }, + "latency_seconds": { + "mean": 0.2892259050841164, + "p50": 0.163, + "p95": 1.002, + "p99": 1.778, + "max": 6.582 + }, + "page_filter_counts": { + "v0.1|100|none|active_only|known_params_only": 83772, + "v0.1|100|latest|active_only|known_params_only": 2855, + "v0.1|default|latest|active_only|known_params_only": 2469, + "v0|100|none|active_only|known_params_only": 450, + "v0|default|none|active_only|known_params_only": 167, + "v0.1|100|latest|include_deleted|known_params_only": 30, + "v0|default|none|active_only|other_params": 140, + "v0|100|latest|active_only|known_params_only": 420, + "v0.1|default|none|active_only|known_params_only": 260, + "v0|50|none|active_only|known_params_only": 2, + "v0.1|1|none|active_only|known_params_only": 6, + "v0.1|100|none|include_deleted|known_params_only": 8, + "v0.1|50|latest|include_deleted|known_params_only": 2, + "v0.1|20|none|active_only|known_params_only": 4, + "v0|60|latest|active_only|known_params_only": 80, + "v0|1|latest|active_only|known_params_only": 1, + "v0.1|30|latest|include_deleted|known_params_only": 1, + "v0|100|latest|include_deleted|known_params_only": 6, + "v0|30|none|active_only|known_params_only": 1, + "v0.1|30|none|active_only|known_params_only": 1, + "v0.1|96|latest|active_only|known_params_only": 1, + "v0.1|25|latest|active_only|known_params_only": 2 + }, + "updated_age_counts": { + ">=7d": 49, + "<1h": 13, + "<1d": 39, + "<7d": 1 + }, + "search_length_counts": { + ">20": 13, + "13": 4, + "9": 9, + "8": 5, + "12": 7, + "4": 9, + "6": 13, + "11": 2, + "16": 2, + "1": 17, + "3": 8, + "2": 1, + "7": 10, + "10": 4, + "5": 8, + "15": 2, + "19": 2 + } + }, + { + "start": "2026-10-28T18:30:00Z", + "end": "2026-10-28T18:31:00Z", + "query_complete": true, + "entries": 62714, + "parsed": 62714, + "unparsed": 0, + "client_records": 54662, + "client_rps": 911.0333333333333, + "background_proxy_rps": 134.2, + "cache_hit_percent": 69.27664556730453, + "page_v01_limit100_percent": 92.8813179004843, + "page_top128_percent": 13.614689878269976, + "distinct_page_uris": 11833, + "http2_percent": 2.3014159745344114, + "kind_counts": { + "page": 53479, + "detail": 573, + "search": 504, + "updated": 56, + "other": 49, + "publish": 1 + }, + "cache_counts": { + "HIT": 37868, + "MISS": 8088, + "STALE": 8051, + "none": 623, + "UPDATING": 32 + }, + "response_bytes": { + "mean": 98982.84532216165, + "p50": 93051, + "p95": 160765, + "p99": 278247, + "max": 1367261 + }, + "latency_seconds": { + "mean": 0.015279865354355923, + "p50": 0.005, + "p95": 0.057, + "p99": 0.161, + "max": 2.226 + }, + "page_filter_counts": { + "v0.1|100|none|active_only|known_params_only": 48011, + "v0.1|100|latest|active_only|known_params_only": 1648, + "v0|30|latest|active_only|known_params_only": 77, + "v0|100|none|active_only|known_params_only": 2165, + "v0.1|default|none|active_only|known_params_only": 302, + "v0|default|none|active_only|known_params_only": 356, + "v0|60|latest|active_only|known_params_only": 19, + "v0.1|100|none|include_deleted|known_params_only": 13, + "v0|1|none|active_only|known_params_only": 2, + "v0.1|50|latest|include_deleted|known_params_only": 2, + "v0|100|latest|active_only|known_params_only": 516, + "v0.1|50|latest|active_only|known_params_only": 3, + "v0.1|96|latest|active_only|known_params_only": 2, + "v0|25|latest|active_only|known_params_only": 1, + "v0|50|none|active_only|known_params_only": 1, + "v0|30|none|active_only|known_params_only": 1, + "v0|default|latest|active_only|known_params_only": 25, + "v0.1|1|none|active_only|known_params_only": 3, + "v0.1|30|latest|include_deleted|known_params_only": 1, + "v0.1|default|latest|active_only|known_params_only": 327, + "v0.1|24|latest|active_only|known_params_only": 1, + "v0|7|none|active_only|known_params_only": 1, + "v0|1|latest|active_only|known_params_only": 1, + "v0|2|latest|active_only|known_params_only": 1 + }, + "updated_age_counts": { + "invalid": 4, + "<1d": 32, + "<1h": 12, + ">=7d": 7, + "<7d": 1 + }, + "search_length_counts": { + "19": 6, + "6": 41, + "18": 4, + ">20": 73, + "11": 33, + "17": 70, + "7": 14, + "5": 37, + "13": 3, + "15": 42, + "8": 14, + "3": 17, + "16": 6, + "4": 33, + "10": 71, + "12": 6, + "9": 8, + "14": 7, + "1": 17, + "2": 1, + "20": 1 + } + }, + { + "start": "2026-10-28T18:45:00Z", + "end": "2026-10-28T18:46:00Z", + "query_complete": true, + "entries": 49590, + "parsed": 49590, + "unparsed": 0, + "client_records": 40856, + "client_rps": 680.9333333333333, + "background_proxy_rps": 145.56666666666666, + "cache_hit_percent": 58.85794008224006, + "page_v01_limit100_percent": 89.04290094518916, + "page_top128_percent": 12.842409345462839, + "distinct_page_uris": 11508, + "http2_percent": 3.2259643626395142, + "kind_counts": { + "page": 39463, + "search": 389, + "detail": 871, + "updated": 37, + "other": 44, + "publish": 52 + }, + "cache_counts": { + "HIT": 24047, + "STALE": 8734, + "MISS": 7096, + "none": 967, + "UPDATING": 12 + }, + "response_bytes": { + "mean": 93140.78177011944, + "p50": 92380, + "p95": 149658, + "p99": 256179, + "max": 1385207 + }, + "latency_seconds": { + "mean": 0.011460079302914669, + "p50": 0.004, + "p95": 0.037, + "p99": 0.122, + "max": 4.827 + }, + "page_filter_counts": { + "v0.1|100|none|active_only|known_params_only": 34585, + "v0|100|none|include_deleted|known_params_only": 197, + "v0.1|100|latest|active_only|known_params_only": 545, + "v0|100|none|active_only|known_params_only": 897, + "v0.1|default|latest|active_only|known_params_only": 2682, + "v0|100|latest|active_only|known_params_only": 356, + "v0.1|default|none|active_only|known_params_only": 57, + "v0|60|latest|active_only|known_params_only": 45, + "v0.1|1|none|active_only|known_params_only": 6, + "v0.1|100|none|include_deleted|known_params_only": 9, + "v0|default|none|active_only|known_params_only": 35, + "v0|1|latest|active_only|known_params_only": 1, + "v0|50|none|active_only|known_params_only": 2, + "v0.1|50|none|active_only|known_params_only": 1, + "v0.1|50|latest|include_deleted|known_params_only": 1, + "v0.1|30|latest|include_deleted|known_params_only": 1, + "v0.1|default|none|active_only|other_params": 4, + "v0|default|latest|active_only|known_params_only": 38, + "v0|40|latest|active_only|known_params_only": 1 + }, + "updated_age_counts": { + "<1h": 11, + "<1d": 20, + ">=7d": 5, + "<7d": 1 + }, + "search_length_counts": { + "7": 27, + ">20": 26, + "19": 7, + "8": 29, + "15": 6, + "4": 17, + "6": 53, + "11": 27, + "12": 19, + "10": 52, + "3": 17, + "5": 29, + "9": 32, + "1": 17, + "2": 1, + "20": 1, + "13": 18, + "18": 3, + "14": 8 + } + }, + { + "start": "2026-10-28T19:00:00Z", + "end": "2026-10-28T19:01:00Z", + "query_complete": true, + "entries": 35776, + "parsed": 35753, + "unparsed": 23, + "client_records": 28663, + "client_rps": 477.71666666666664, + "background_proxy_rps": 118.16666666666667, + "cache_hit_percent": 40.58193489864983, + "page_v01_limit100_percent": 77.2590028720819, + "page_top128_percent": 10.89550040503719, + "distinct_page_uris": 12231, + "http2_percent": 5.934479991626836, + "kind_counts": { + "page": 27158, + "detail": 1105, + "search": 160, + "updated": 203, + "other": 37 + }, + "cache_counts": { + "MISS": 8770, + "HIT": 11632, + "none": 1166, + "STALE": 7089, + "UPDATING": 6 + }, + "response_bytes": { + "mean": 87401.30394585353, + "p50": 86892, + "p95": 140187, + "p99": 235562, + "max": 1366049 + }, + "latency_seconds": { + "mean": 0.012462652199698516, + "p50": 0.007, + "p95": 0.031, + "p99": 0.132, + "max": 1.049 + }, + "page_filter_counts": { + "v0.1|100|none|active_only|known_params_only": 13458, + "v0.1|default|latest|active_only|known_params_only": 2057, + "v0|100|latest|active_only|known_params_only": 622, + "v0|100|none|active_only|known_params_only": 2800, + "v0|100|none|include_deleted|known_params_only": 396, + "v0.1|100|latest|active_only|known_params_only": 7487, + "v0.1|100|latest|include_deleted|known_params_only": 19, + "v0.1|default|none|active_only|known_params_only": 64, + "v0|default|none|active_only|known_params_only": 128, + "v0|50|latest|active_only|known_params_only": 1, + "v0.1|100|none|include_deleted|known_params_only": 18, + "v0|50|none|active_only|known_params_only": 3, + "v0|1|none|active_only|known_params_only": 3, + "v0.1|96|latest|active_only|known_params_only": 1, + "v0|default|none|active_only|other_params": 2, + "v0.1|1|none|active_only|known_params_only": 4, + "v0|60|latest|active_only|known_params_only": 42, + "v0.1|25|latest|active_only|known_params_only": 1, + "v0.1|50|none|active_only|known_params_only": 2, + "v0|100|latest|include_deleted|known_params_only": 35, + "v0.1|50|latest|include_deleted|known_params_only": 2, + "v0|7|none|active_only|known_params_only": 2, + "v0.1|30|latest|include_deleted|known_params_only": 1, + "v0.1|5|none|active_only|known_params_only": 1, + "v0|5|none|active_only|known_params_only": 1, + "v0|1|latest|active_only|known_params_only": 1, + "v0|default|latest|active_only|known_params_only": 5, + "v0.1|30|latest|active_only|known_params_only": 1, + "v0.1|20|none|active_only|known_params_only": 1 + }, + "updated_age_counts": { + "<1d": 137, + "<1h": 63, + "invalid": 1, + ">=7d": 2 + }, + "search_length_counts": { + "14": 9, + ">20": 30, + "17": 6, + "16": 6, + "5": 11, + "15": 2, + "4": 9, + "8": 7, + "6": 19, + "11": 3, + "18": 4, + "10": 4, + "7": 6, + "9": 10, + "13": 3, + "12": 4, + "1": 17, + "2": 1, + "3": 7, + "19": 2 + } + }, + { + "start": "2026-10-28T19:15:00Z", + "end": "2026-10-28T19:16:00Z", + "query_complete": true, + "entries": 18910, + "parsed": 18244, + "unparsed": 666, + "client_records": 15145, + "client_rps": 252.41666666666666, + "background_proxy_rps": 51.65, + "cache_hit_percent": 24.602178936942884, + "page_v01_limit100_percent": 67.1909950212858, + "page_top128_percent": 24.561656685186524, + "distinct_page_uris": 8134, + "http2_percent": 8.993067018818092, + "kind_counts": { + "page": 13859, + "detail": 1000, + "other": 56, + "updated": 83, + "search": 144, + "publish": 3 + }, + "cache_counts": { + "MISS": 6592, + "HIT": 3726, + "none": 1726, + "STALE": 3099, + "UPDATING": 2 + }, + "response_bytes": { + "mean": 80305.83856058105, + "p50": 81392, + "p95": 143096, + "p99": 206313, + "max": 1336993 + }, + "latency_seconds": { + "mean": 0.012065170023110265, + "p50": 0.006, + "p95": 0.028, + "p99": 0.173, + "max": 1.755 + }, + "page_filter_counts": { + "v0.1|100|none|active_only|known_params_only": 6482, + "v0|100|latest|active_only|known_params_only": 1423, + "v0.1|default|latest|active_only|known_params_only": 756, + "v0|100|none|active_only|known_params_only": 1966, + "v0.1|100|latest|active_only|known_params_only": 2647, + "v0.1|default|none|active_only|known_params_only": 306, + "v0.1|100|latest|include_deleted|known_params_only": 2, + "v0.1|9|none|active_only|known_params_only": 2, + "v0.1|1|none|active_only|known_params_only": 3, + "v0|60|latest|active_only|known_params_only": 45, + "v0.1|100|none|include_deleted|known_params_only": 181, + "v0.1|25|latest|active_only|known_params_only": 8, + "v0|default|none|active_only|known_params_only": 10, + "v0.1|50|latest|include_deleted|known_params_only": 1, + "v0.1|3|latest|active_only|known_params_only": 1, + "v0|50|none|active_only|known_params_only": 2, + "v0.1|30|latest|include_deleted|known_params_only": 1, + "v0|7|none|active_only|known_params_only": 3, + "v0.1|50|latest|active_only|known_params_only": 1, + "v0|100|latest|include_deleted|known_params_only": 12, + "v0.1|96|latest|active_only|known_params_only": 2, + "v0|default|latest|active_only|known_params_only": 4, + "v0|1|latest|active_only|known_params_only": 1 + }, + "updated_age_counts": { + "<1h": 12, + "<1d": 52, + "<7d": 13, + ">=7d": 5, + "invalid": 1 + }, + "search_length_counts": { + "3": 8, + "12": 4, + ">20": 31, + "9": 9, + "4": 10, + "8": 6, + "6": 18, + "7": 8, + "5": 10, + "14": 2, + "10": 6, + "13": 7, + "16": 2, + "1": 17, + "2": 1, + "15": 2, + "11": 1, + "19": 2 + } + }, + { + "start": "2026-10-28T19:30:00Z", + "end": "2026-10-28T19:31:00Z", + "query_complete": true, + "entries": 14201, + "parsed": 14201, + "unparsed": 0, + "client_records": 11209, + "client_rps": 186.81666666666666, + "background_proxy_rps": 49.86666666666667, + "cache_hit_percent": 16.968507449371042, + "page_v01_limit100_percent": 46.80473372781065, + "page_top128_percent": 12.3767258382643, + "distinct_page_uris": 7310, + "http2_percent": 21.679007940048177, + "kind_counts": { + "page": 10140, + "search": 152, + "detail": 725, + "updated": 128, + "other": 63, + "publish": 1 + }, + "cache_counts": { + "MISS": 5518, + "STALE": 2993, + "HIT": 1902, + "none": 789, + "UPDATING": 7 + }, + "response_bytes": { + "mean": 87152.13310732447, + "p50": 87032, + "p95": 160968, + "p99": 255634, + "max": 1384957 + }, + "latency_seconds": { + "mean": 0.010620572754037346, + "p50": 0.007, + "p95": 0.018, + "p99": 0.116, + "max": 1.734 + }, + "page_filter_counts": { + "v0|100|latest|active_only|known_params_only": 1387, + "v0|100|none|include_deleted|known_params_only": 137, + "v0.1|100|none|active_only|known_params_only": 4442, + "v0.1|100|latest|active_only|known_params_only": 294, + "v0.1|default|latest|active_only|known_params_only": 550, + "v0.1|default|none|active_only|known_params_only": 299, + "v0|100|none|active_only|known_params_only": 2867, + "v0.1|1|none|active_only|known_params_only": 7, + "v0.1|100|none|include_deleted|known_params_only": 10, + "v0|default|none|active_only|known_params_only": 46, + "v0.1|25|latest|active_only|known_params_only": 9, + "v0|1|none|active_only|known_params_only": 1, + "v0|60|latest|active_only|known_params_only": 31, + "v0|7|none|active_only|known_params_only": 3, + "v0|20|none|active_only|known_params_only": 3, + "v0|50|none|active_only|known_params_only": 1, + "v0|1|latest|active_only|known_params_only": 1, + "v0|30|latest|active_only|known_params_only": 5, + "v0|25|latest|active_only|known_params_only": 1, + "v0|100|latest|include_deleted|known_params_only": 45, + "v0.1|30|latest|include_deleted|known_params_only": 1 + }, + "updated_age_counts": { + "invalid": 5, + "<1h": 12, + "<1d": 56, + "<7d": 53, + ">=7d": 2 + }, + "search_length_counts": { + ">20": 29, + "6": 20, + "16": 5, + "15": 4, + "4": 7, + "8": 7, + "11": 3, + "9": 7, + "19": 5, + "12": 6, + "14": 2, + "5": 13, + "10": 6, + "7": 7, + "13": 2, + "18": 3, + "1": 17, + "3": 6, + "2": 1, + "17": 1, + "20": 1 + } + }, + { + "start": "2026-10-28T19:45:00Z", + "end": "2026-10-28T19:46:00Z", + "query_complete": true, + "entries": 11545, + "parsed": 11545, + "unparsed": 0, + "client_records": 8773, + "client_rps": 146.21666666666667, + "background_proxy_rps": 46.2, + "cache_hit_percent": 6.998746152969337, + "page_v01_limit100_percent": 68.77365632096897, + "page_top128_percent": 6.220035326772647, + "distinct_page_uris": 6793, + "http2_percent": 16.550780804741823, + "kind_counts": { + "page": 7926, + "detail": 484, + "publish": 1, + "updated": 76, + "search": 218, + "other": 68 + }, + "cache_counts": { + "STALE": 2772, + "MISS": 4833, + "none": 554, + "HIT": 614 + }, + "response_bytes": { + "mean": 88146.51441924085, + "p50": 87747, + "p95": 163878, + "p99": 269929, + "max": 1385004 + }, + "latency_seconds": { + "mean": 0.010675709563433477, + "p50": 0.008, + "p95": 0.018, + "p99": 0.065, + "max": 0.794 + }, + "page_filter_counts": { + "v0.1|default|latest|active_only|known_params_only": 589, + "v0.1|100|none|active_only|known_params_only": 4814, + "v0|100|none|active_only|known_params_only": 1301, + "v0|100|latest|active_only|known_params_only": 418, + "v0.1|default|none|active_only|known_params_only": 59, + "v0.1|100|latest|active_only|known_params_only": 629, + "v0|default|none|active_only|known_params_only": 61, + "v0.1|100|none|include_deleted|known_params_only": 8, + "v0.1|96|latest|active_only|known_params_only": 1, + "v0.1|1|none|active_only|known_params_only": 7, + "v0|100|latest|include_deleted|known_params_only": 26, + "v0|50|none|active_only|known_params_only": 1, + "v0|5|none|active_only|known_params_only": 1, + "v0.1|30|latest|include_deleted|known_params_only": 1, + "v0|1|latest|active_only|known_params_only": 1, + "v0|60|latest|active_only|known_params_only": 3, + "v0.1|25|latest|active_only|known_params_only": 2, + "v0|40|latest|active_only|known_params_only": 1, + "v0.1|50|latest|active_only|known_params_only": 1, + "v0|1|none|active_only|known_params_only": 1, + "v0|100|none|include_deleted|known_params_only": 1 + }, + "updated_age_counts": { + "<1h": 12, + "<1d": 32, + ">=7d": 17, + "<7d": 15 + }, + "search_length_counts": { + "8": 25, + "15": 4, + "1": 18, + "2": 1, + "3": 16, + "6": 29, + "5": 16, + "10": 9, + "4": 14, + "7": 15, + "17": 2, + ">20": 20, + "13": 3, + "16": 2, + "9": 22, + "19": 4, + "11": 10, + "14": 3, + "18": 1, + "20": 1, + "12": 3 + } + }, + { + "start": "2026-10-28T20:00:00Z", + "end": "2026-10-28T20:01:00Z", + "query_complete": true, + "entries": 16792, + "parsed": 16706, + "unparsed": 86, + "client_records": 14548, + "client_rps": 242.46666666666667, + "background_proxy_rps": 35.96666666666667, + "cache_hit_percent": 37.68215562276602, + "page_v01_limit100_percent": 77.90162912152155, + "page_top128_percent": 34.44539714708863, + "distinct_page_uris": 5806, + "http2_percent": 10.303821831179544, + "kind_counts": { + "page": 12829, + "detail": 731, + "other": 79, + "updated": 708, + "search": 197, + "publish": 4 + }, + "cache_counts": { + "MISS": 6004, + "STALE": 2158, + "none": 902, + "HIT": 5482, + "UPDATING": 2 + }, + "response_bytes": { + "mean": 83831.32815507286, + "p50": 82385, + "p95": 139962, + "p99": 199836, + "max": 1349761 + }, + "latency_seconds": { + "mean": 0.010964187517185094, + "p50": 0.006, + "p95": 0.018, + "p99": 0.207, + "max": 1.018 + }, + "page_filter_counts": { + "v0.1|100|none|active_only|known_params_only": 3851, + "v0|100|none|active_only|known_params_only": 1104, + "v0.1|default|latest|active_only|known_params_only": 860, + "v0|100|latest|active_only|known_params_only": 667, + "v0.1|100|none|include_deleted|known_params_only": 56, + "v0.1|100|latest|active_only|known_params_only": 6085, + "v0|100|none|include_deleted|known_params_only": 160, + "v0.1|100|latest|include_deleted|known_params_only": 2, + "v0.1|default|none|active_only|known_params_only": 8, + "v0.1|96|latest|active_only|known_params_only": 5, + "v0|default|none|active_only|known_params_only": 5, + "v0|30|none|active_only|known_params_only": 1, + "v0|50|none|active_only|known_params_only": 4, + "v0|default|latest|active_only|known_params_only": 5, + "v0|1|latest|active_only|known_params_only": 1, + "v0|1|none|active_only|known_params_only": 1, + "v0|60|latest|active_only|known_params_only": 5, + "v0|default|none|active_only|other_params": 2, + "v0.1|25|latest|active_only|known_params_only": 1, + "v0.1|50|latest|active_only|known_params_only": 1, + "v0.1|30|latest|include_deleted|known_params_only": 1, + "v0.1|1|none|active_only|known_params_only": 3, + "v0.1|20|none|active_only|known_params_only": 1 + }, + "updated_age_counts": { + "<1h": 20, + "invalid": 14, + ">=7d": 571, + "<1d": 103 + }, + "search_length_counts": { + "10": 15, + ">20": 24, + "5": 16, + "6": 29, + "8": 11, + "15": 4, + "7": 14, + "3": 6, + "4": 16, + "17": 3, + "9": 12, + "14": 4, + "12": 7, + "20": 3, + "16": 3, + "13": 3, + "11": 6, + "18": 1, + "1": 17, + "2": 1, + "19": 2 + } + }, + { + "start": "2026-10-28T20:15:00Z", + "end": "2026-10-28T20:16:00Z", + "query_complete": true, + "entries": 13578, + "parsed": 13578, + "unparsed": 0, + "client_records": 9968, + "client_rps": 166.13333333333333, + "background_proxy_rps": 60.166666666666664, + "cache_hit_percent": 12.530096308186195, + "page_v01_limit100_percent": 37.83029001074114, + "page_top128_percent": 6.519871106337272, + "distinct_page_uris": 6450, + "http2_percent": 12.510032102728731, + "kind_counts": { + "page": 9310, + "updated": 103, + "search": 147, + "detail": 364, + "other": 43, + "publish": 1 + }, + "cache_counts": { + "HIT": 1249, + "STALE": 3609, + "MISS": 4701, + "none": 408, + "UPDATING": 1 + }, + "response_bytes": { + "mean": 64733.65198635634, + "p50": 71372, + "p95": 132780, + "p99": 192412, + "max": 936041 + }, + "latency_seconds": { + "mean": 0.010767154895666401, + "p50": 0.006, + "p95": 0.016, + "p99": 0.188, + "max": 0.947 + }, + "page_filter_counts": { + "v0.1|default|latest|active_only|known_params_only": 3723, + "v0.1|100|latest|active_only|known_params_only": 585, + "v0.1|100|none|active_only|known_params_only": 2933, + "v0|100|latest|active_only|known_params_only": 376, + "v0|100|none|active_only|known_params_only": 1152, + "v0.1|default|none|active_only|known_params_only": 294, + "v0|100|none|include_deleted|known_params_only": 169, + "v0.1|1|none|active_only|known_params_only": 7, + "v0|default|none|active_only|known_params_only": 60, + "v0|40|latest|active_only|known_params_only": 1, + "v0|50|none|active_only|known_params_only": 1, + "v0.1|100|none|include_deleted|known_params_only": 4, + "v0.1|10|latest|active_only|known_params_only": 1, + "v0|60|latest|active_only|known_params_only": 1, + "v0.1|25|latest|active_only|known_params_only": 2, + "v0.1|96|latest|active_only|known_params_only": 1 + }, + "updated_age_counts": { + "<1d": 62, + "<1h": 30, + ">=7d": 5, + "<7d": 6 + }, + "search_length_counts": { + ">20": 24, + "12": 7, + "1": 19, + "2": 3, + "6": 18, + "7": 7, + "14": 2, + "13": 4, + "9": 7, + "8": 5, + "3": 7, + "5": 10, + "17": 2, + "10": 6, + "4": 11, + "20": 4, + "16": 4, + "11": 4, + "19": 3 + } + }, + { + "start": "2026-10-28T20:30:00Z", + "end": "2026-10-28T20:31:00Z", + "query_complete": true, + "entries": 10346, + "parsed": 10346, + "unparsed": 0, + "client_records": 8804, + "client_rps": 146.73333333333332, + "background_proxy_rps": 25.7, + "cache_hit_percent": 15.572467060427078, + "page_v01_limit100_percent": 34.16302765647744, + "page_top128_percent": 8.981077147016011, + "distinct_page_uris": 5059, + "http2_percent": 15.311222171740118, + "kind_counts": { + "page": 6870, + "detail": 417, + "other": 622, + "updated": 104, + "search": 789, + "publish": 2 + }, + "cache_counts": { + "MISS": 5395, + "HIT": 1371, + "none": 474, + "STALE": 1541, + "UPDATING": 23 + }, + "response_bytes": { + "mean": 73570.86142662427, + "p50": 81125, + "p95": 142691, + "p99": 189730, + "max": 1270182 + }, + "latency_seconds": { + "mean": 0.015989323034984387, + "p50": 0.006, + "p95": 0.021, + "p99": 0.235, + "max": 3.499 + }, + "page_filter_counts": { + "v0.1|default|none|active_only|known_params_only": 304, + "v0|100|none|active_only|known_params_only": 3255, + "v0.1|100|none|active_only|known_params_only": 2216, + "v0.1|default|latest|active_only|known_params_only": 567, + "v0.1|100|none|include_deleted|known_params_only": 9, + "v0|100|latest|active_only|known_params_only": 309, + "v0.1|100|latest|active_only|known_params_only": 121, + "v0.1|1|none|active_only|known_params_only": 5, + "v0.1|50|none|active_only|known_params_only": 53, + "v0.1|25|latest|active_only|known_params_only": 12, + "v0.1|50|latest|active_only|known_params_only": 2, + "v0|default|none|active_only|known_params_only": 4, + "v0|1|none|active_only|known_params_only": 3, + "v0.1|96|latest|active_only|known_params_only": 2, + "v0|50|none|active_only|known_params_only": 2, + "v0|25|latest|active_only|known_params_only": 1, + "v0.1|30|latest|include_deleted|known_params_only": 1, + "v0.1|100|latest|include_deleted|known_params_only": 1, + "v0|5|none|active_only|known_params_only": 1, + "v0|1|latest|active_only|known_params_only": 1, + "v0.1|30|latest|active_only|known_params_only": 1 + }, + "updated_age_counts": { + "<1h": 26, + "invalid": 20, + "<1d": 38, + ">=7d": 3, + "<7d": 17 + }, + "search_length_counts": { + "12": 24, + "19": 4, + "7": 50, + "6": 127, + "15": 9, + ">20": 29, + "1": 177, + "2": 10, + "3": 59, + "5": 50, + "4": 45, + "10": 63, + "8": 47, + "11": 36, + "20": 3, + "14": 12, + "18": 3, + "9": 21, + "17": 2, + "13": 12, + "16": 6 + } + } + ], + "old_benchmark": { + "offered_rps": 4500, + "duration_seconds": 180, + "scheduled_read_mix": { + "updated_rps": 36, + "search_rps": 3 + }, + "page_mix_percent": { + "v0_limit50": 85.02, + "v0_limit100": 4.99, + "v01_limit50": 4.99, + "v01_limit100": 4.99 + }, + "catalog_keys": 4000, + "hot_selection_percent": 80, + "hot_keys": 128, + "forced_idle_connection_rotation_seconds": 30, + "http1_only": true, + "latest_attempted_cache_hit_percent": 88.84064182849852 + }, + "qualification": { + "status": "NOT_YET_A_VALIDATED_PRODUCTION_REPLAY", + "confirmed": [ + "Production client demand, background refresh records and origin DB query counts are distinct populations.", + "Page API/limit mix and key diversity materially differ from the old stress test.", + "Cache locality changes across the cycle; a fixed hot-key percentage is inadequate." + ], + "remaining": [ + "One-minute log samples do not establish all arrivals between samples or client crawl/connection lifetimes.", + "Measured page frequency model maps ranks to different staging cursor positions; body-size distribution and origin query rate must be calibrated.", + "Protocol mix is measured; TCP connection reuse, compression negotiation and client retry behavior remain unmeasured.", + "Search term selectivity, incremental-sync age/cursor mix, detail calls and concurrent publishes still need matching; old default search/update policies are not qualified.", + "Baseline must use the production release and CPU reservations/placement; the last candidate run was not this control.", + "Natural scale-up/down and sustained-duration acceptance remain untested under a calibrated workload." + ] + }, + "next_gate": "First qualify a production-aligned baseline against measured API/limit mix, key diversity/cache outcomes, wire/decoded body sizes, origin query rates, CPU pressure and acquisition delays. Run the candidate only after baseline fidelity is established. Keep the old 4500-rps workload labeled as stress, not production acceptance." +} diff --git a/tools/ingressbench/evidence/workload-profile-2026-10-28/staging-profile-smoke.json b/tools/ingressbench/evidence/workload-profile-2026-10-28/staging-profile-smoke.json new file mode 100644 index 000000000..a646d250f --- /dev/null +++ b/tools/ingressbench/evidence/workload-profile-2026-10-28/staging-profile-smoke.json @@ -0,0 +1,313 @@ +{ + "Calibration": false, + "RequestPhases": { + "page:HIT": { + "Attempts": 10, + "NewConnections": 0, + "ReusedConnections": 10, + "SchedulerLag": { + "Count": 10, + "MeanMS": 1.7094626000000002, + "P95UpperMS": 5, + "MaximumMS": 2.150458 + }, + "QueueDelay": { + "Count": 10, + "MeanMS": 0.0076333, + "P95UpperMS": 0.1, + "MaximumMS": 0.010792 + }, + "ConnectionWait": { + "Count": 10, + "MeanMS": 0.009829500000000001, + "P95UpperMS": 0.1, + "MaximumMS": 0.015167 + }, + "TLS": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FirstByte": { + "Count": 10, + "MeanMS": 134.11448330000002, + "P95UpperMS": 200, + "MaximumMS": 134.656042 + }, + "Body": { + "Count": 10, + "MeanMS": 2.1762333000000003, + "P95UpperMS": 5, + "MaximumMS": 3.92725 + }, + "HTTP": { + "Count": 10, + "MeanMS": 136.40833750000002, + "P95UpperMS": 200, + "MaximumMS": 137.883417 + } + }, + "page:MISS": { + "Attempts": 189, + "NewConnections": 4, + "ReusedConnections": 185, + "SchedulerLag": { + "Count": 189, + "MeanMS": 1.785414031746031, + "P95UpperMS": 5, + "MaximumMS": 3.644458 + }, + "QueueDelay": { + "Count": 189, + "MeanMS": 16.898746222222222, + "P95UpperMS": 200, + "MaximumMS": 468.908 + }, + "ConnectionWait": { + "Count": 189, + "MeanMS": 6.174938460317448, + "P95UpperMS": 0.1, + "MaximumMS": 329.667666 + }, + "TLS": { + "Count": 4, + "MeanMS": 142.78265599999997, + "P95UpperMS": 200, + "MaximumMS": 145.914458 + }, + "FirstByte": { + "Count": 189, + "MeanMS": 166.8337709682541, + "P95UpperMS": 200, + "MaximumMS": 214.686625 + }, + "Body": { + "Count": 189, + "MeanMS": 35.57214037037036, + "P95UpperMS": 500, + "MaximumMS": 1480.708458 + }, + "HTTP": { + "Count": 189, + "MeanMS": 208.69920370370366, + "P95UpperMS": 500, + "MaximumMS": 1658.189083 + } + } + }, + "StartedAt": "2026-10-28T20:53:49.467804Z", + "FinishedAt": "2026-10-28T20:54:09.513453Z", + "CatalogSHA256": "410b91ddb2a4436ce549834236e282a8501cc8ab9d6748481d32e75dfc80f37b", + "PageWorkloadSHA256": "693e29271b933bd6cad81a4df114d111506b21d1afcb50e457f00ef217a42bbc", + "ConnectionRotation": "0s", + "ReadRate": 10, + "UpdatedRate": 0, + "SearchRate": 0, + "HotPercent": -1, + "Scheduled": 200, + "ScheduledPublishes": 0, + "Completed": 199, + "Dropped": 1, + "DroppedPublishes": 0, + "Bytes": 20382432, + "Status": { + "page:200": 199 + }, + "Cache": { + "HIT": 10, + "MISS": 189 + }, + "Errors": {}, + "Protocols": { + "HTTP/1.1": 199 + }, + "RequestShapes": { + "page|v0.1|100|latest|cursor=true": 10, + "page|v0.1|100|none|cursor=true": 173, + "page|v0.1|30|latest|cursor=true": 1, + "page|v0.1|30|none|cursor=true": 1, + "page|v0|100|none|cursor=true": 9, + "page|v0|30|latest|cursor=true": 1, + "page|v0|30|none|cursor=true": 4 + }, + "ErrorDetails": {}, + "LatencyMS": { + "page_p50": 171.469042, + "page_p95": 694.48675, + "page_p99": 875.734917 + }, + "ElapsedSeconds": 20.045478083, + "Failures": null, + "OmittedFailures": 0, + "Windows": [ + { + "StartedAt": "2026-10-28T20:53:40Z", + "Seconds": 10, + "Reads": { + "Scheduled": 6, + "Dropped": 0, + "Dispatched": 4, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 6, + "MeanMS": 1.7261528333333331, + "P95UpperMS": 5, + "MaximumMS": 2.082708 + }, + "QueueDelay": { + "Count": 4, + "MeanMS": 0.028677, + "P95UpperMS": 0.1, + "MaximumMS": 0.076666 + }, + "HTTPDuration": { + "Count": 4, + "MeanMS": 840.383031, + "P95UpperMS": 1000, + "MaximumMS": 866.8235 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:53:50Z", + "Seconds": 10, + "Reads": { + "Scheduled": 100, + "Dropped": 1, + "Dispatched": 101, + "Completed": 103, + "Failures": 0, + "SchedulerLag": { + "Count": 100, + "MeanMS": 1.6364954299999999, + "P95UpperMS": 5, + "MaximumMS": 3.644458 + }, + "QueueDelay": { + "Count": 101, + "MeanMS": 31.376295772277228, + "P95UpperMS": 500, + "MaximumMS": 468.908 + }, + "HTTPDuration": { + "Count": 101, + "MeanMS": 201.15720831683166, + "P95UpperMS": 500, + "MaximumMS": 873.669209 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + }, + { + "StartedAt": "2026-10-28T20:54:00Z", + "Seconds": 10, + "Reads": { + "Scheduled": 94, + "Dropped": 0, + "Dispatched": 94, + "Completed": 96, + "Failures": 0, + "SchedulerLag": { + "Count": 94, + "MeanMS": 1.9333111808510637, + "P95UpperMS": 5, + "MaximumMS": 2.150458 + }, + "QueueDelay": { + "Count": 94, + "MeanMS": 0.26402965957446806, + "P95UpperMS": 0.1, + "MaximumMS": 23.999 + }, + "HTTPDuration": { + "Count": 94, + "MeanMS": 182.2321565000001, + "P95UpperMS": 200, + "MaximumMS": 1658.189083 + }, + "FailureClasses": null + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "QueueDelay": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "HTTPDuration": { + "Count": 0, + "MeanMS": 0, + "P95UpperMS": 0, + "MaximumMS": 0 + }, + "FailureClasses": null + } + } + ] +} diff --git a/tools/ingressbench/main.go b/tools/ingressbench/main.go new file mode 100644 index 000000000..9a484ea1e --- /dev/null +++ b/tools/ingressbench/main.go @@ -0,0 +1,447 @@ +// ingressbench drives only the explicitly acknowledged staging registry. It runs +// outside the tested node pool, through the public ingress and its real caches. +package main + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "math" + "net" + "net/http" + "net/url" + "os" + "sort" + "strings" + "sync" + "time" +) + +const limitParam = "limit" +const target = "https://staging.registry.modelcontextprotocol.io" + +type result struct { + Calibration bool + RequestPhases map[string]phaseSummary + StartedAt, FinishedAt, CatalogSHA256 string + PageWorkloadSHA256, ConnectionRotation, ReadTraceSHA256 string + ReadRate, UpdatedRate, SearchRate, HotPercent int + Scheduled, ScheduledPublishes, Completed, Dropped, DroppedPublishes int + Bytes int64 + Status, Cache, Errors, Protocols map[string]int + RequestShapes map[string]int + ErrorDetails map[string]string + LatencyMS map[string]float64 + ElapsedSeconds float64 + Failures []requestFailure + OmittedFailures int + Windows []timeWindow +} +type requestFailure struct { + ScheduledAt, FinishedAt time.Time + Kind, Class string + Status int +} +type point struct { + phases phaseObservation + kind, cache, failure, protocol, detail, shape string + status int + elapsed float64 + bytes int64 + scheduledAt, queuedAt, startedAt, finishedAt time.Time +} + +func main() { + ack := flag.Bool("staging", false, "acknowledge load against staging only") + catalogMode := flag.Bool("catalog", false, "collect real pagination URLs before timing") + keysFile := flag.String("keys", "keys.json", "saved real pagination URLs") + output := flag.String("out", "result.json", "output") + rate := flag.Int("rate", 1500, "scheduled reads per second") + updatedRate := flag.Int("updated-rate", 0, "uncached incremental-sync reads per second, within total rate") + searchRate := flag.Int("search-rate", 3, "varied search reads per second, within total rate") + duration := flag.Duration("duration", 10*time.Minute, "offering duration") + workers := flag.Int("workers", 512, "bounded request concurrency and queue") + tokenFile := flag.String("token-file", "", "optional scoped staging publish token file") + hot := flag.Int("hot-percent", 80, "percent of requests using the first 128 keys") + traceFile := flag.String("read-trace", "", "private catalog-indexed read arrival trace; overrides generated rates") + pageWorkloadFile := flag.String("page-workload", "", "measured page-class/key-frequency JSON; overrides hot-percent for pages") + connectionRotation := flag.Duration("connection-rotation", 30*time.Second, "forced idle-connection rotation; 0 disables this synthetic churn") + http2 := flag.Bool("http2", false, "allow HTTP/2 negotiation (HTTP/1.1 only by default)") + calibrate := flag.Bool("calibrate", false, "use a private loopback HTTPS fixture with 64-KiB bodies; never contact staging") + cpuProfile := flag.String("cpu-profile", "", "optional local generator CPU profile file") + flag.Parse() + validateArguments(*ack, *rate, *updatedRate, *searchRate, *workers, *duration, *hot) + validateConnectionRotation(*connectionRotation) + tr := newTransport(*http2) + client := &http.Client{Transport: tr, Timeout: 10 * time.Second, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + baseURL, closeFixture := configureCalibration(tr, *calibrate, *tokenFile, *catalogMode) + defer closeFixture() + defer tr.CloseIdleConnections() + stopProfile := startCPUProfile(*cpuProfile) + defer stopProfile() + if *catalogMode { + collectCatalog(client, *keysFile) + return + } + raw, keys := loadKeys(*keysFile, *traceFile != "") + pageWorkload, err := loadPageWorkload(*pageWorkloadFile, keys) + if err != nil { + panic(err) + } + trace, err := loadReadTrace(*traceFile, keys, *duration, *pageWorkloadFile, *tokenFile) + if err != nil { + panic(err) + } + token := loadToken(*tokenFile) + results := make(chan point, 4096) + var wg sync.WaitGroup + r := result{Status: map[string]int{}, Cache: map[string]int{}, Errors: map[string]int{}, Protocols: map[string]int{}, ErrorDetails: map[string]string{}, LatencyMS: map[string]float64{}} + r.Calibration = *calibrate + samples := map[string][]float64{} + done := make(chan struct{}) + completionTimeline := newTimeline() + readTimeline, publishTimeline := newTimeline(), newTimeline() + go collectResults(results, &r, samples, completionTimeline, done) + request := makeRequesterAt(client, baseURL, token, results) + type job struct { + n int + at, queuedAt time.Time + } + jobs := make(chan job, *workers) + for i := 0; i < *workers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := range jobs { + path, kind := selectedRead(j.n, *hot, *rate, *updatedRate, *searchRate, keys, pageWorkload, trace) + request(path, kind, requestTiming{j.at, j.queuedAt}, nil) + } + }() + } + start := time.Now() + r.StartedAt = start.UTC().Format(time.RFC3339Nano) + r.CatalogSHA256 = fmt.Sprintf("%x", sha256.Sum256(raw)) + r.ReadRate = *rate + r.UpdatedRate = *updatedRate + r.SearchRate = *searchRate + r.HotPercent = *hot + r.ConnectionRotation = connectionRotation.String() + if len(pageWorkload.paths) > 0 { + r.PageWorkloadSHA256 = pageWorkload.SHA256 + r.HotPercent = -1 // The measured distribution replaces this legacy parameter. + } + trace.describe(&r) + runID := fmt.Sprint(start.UnixNano()) + total := trace.count(int(duration.Seconds() * float64(*rate))) + if token != "" { + pubJobs := make(chan job, 16) + for w := 0; w < 16; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := range pubJobs { + i := j.n + body, _ := json.Marshal(map[string]any{"$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", "name": fmt.Sprintf("io.benchmark/autoscale-%s-%d", runID, i), "description": "Autoscaling staging benchmark", versionParam: "1.0.0", "remotes": []map[string]string{{"type": "streamable-http", "url": fmt.Sprintf("https://example.invalid/autoscale-%s-%d/mcp", runID, i)}}}) + request("/v0.1/publish", "publish", requestTiming{j.at, j.queuedAt}, body) + } + }() + } + wg.Add(1) + go func() { + defer wg.Done() + defer close(pubJobs) + for i := 0; i < int(duration.Seconds()); i++ { + at := start.Add(time.Duration(i) * time.Second) + time.Sleep(time.Until(at)) + r.ScheduledPublishes++ + queuedAt := time.Now() + dropped := false + select { + case pubJobs <- job{i, at, queuedAt}: + default: + r.DroppedPublishes++ + dropped = true + } + publishTimeline.scheduled("publish", at, queuedAt, dropped) + } + }() + } + stopRotation := rotateIdleConnections(tr, *connectionRotation) + defer stopRotation() + + for i := 0; i < total; i++ { + at := start.Add(trace.offset(i, *rate)) + time.Sleep(time.Until(at)) + r.Scheduled++ + queuedAt := time.Now() + dropped := false + select { + case jobs <- job{i, at, queuedAt}: + default: + r.Dropped++ + dropped = true + } + readTimeline.scheduled("read", at, queuedAt, dropped) + } + close(jobs) + wg.Wait() + close(results) + <-done + completionTimeline.merge(readTimeline) + completionTimeline.merge(publishTimeline) + r.Windows = completionTimeline.windows() + r.ElapsedSeconds = time.Since(start).Seconds() + r.FinishedAt = time.Now().UTC().Format(time.RFC3339Nano) + for kind, v := range samples { + sort.Float64s(v) + for _, q := range []float64{.5, .95, .99} { + r.LatencyMS[fmt.Sprintf("%s_p%.0f", kind, q*100)] = v[int(math.Ceil(float64(len(v))*q))-1] + } + } + b, _ := json.MarshalIndent(r, "", " ") + if err := os.WriteFile(*output, b, 0600); err != nil { + panic(err) + } + fmt.Fprintln(os.Stdout, string(b)) +} + +// Use the sequence within each class. Indexing by the global request number +// couples modulo 100 to the catalog size and silently visits only a subset. +func pageIndex(n, hotPercent, keyCount int) int { + position := n % 100 + if position < hotPercent { + return ((n/100*hotPercent + position) * 31) % 128 + } + coldSequence := n/100*(100-hotPercent) + position - hotPercent + return (coldSequence * 7919) % keyCount +} + +func errorClass(err error) string { + var n net.Error + switch { + case errors.As(err, &n) && n.Timeout(): + return "timeout" + case errors.Is(err, io.ErrUnexpectedEOF): + return "unexpected_eof" + case errors.Is(err, io.EOF): + return "eof" + case strings.Contains(err.Error(), "connection reset"): + return "connection_reset" + case strings.Contains(err.Error(), "connection refused"): + return "connection_refused" + default: + return fmt.Sprintf("%T", err) + } +} + +func newTransport(allowHTTP2 bool) *http.Transport { + protocols := new(http.Protocols) + protocols.SetHTTP1(true) + protocols.SetHTTP2(allowHTTP2) + return &http.Transport{Protocols: protocols, MaxIdleConns: 2048, MaxIdleConnsPerHost: 1024, MaxConnsPerHost: 2048, IdleConnTimeout: 15 * time.Second} +} +func errorDetail(err error) string { + var u *url.Error + if errors.As(err, &u) { + err = u.Err + } // Exclude request URLs/cursors from diagnostics. + return err.Error() +} + +func collectCatalog(client *http.Client, keysFile string) { + var keys []string + for _, prefix := range []string{v0ListingPath, v01ListingPath} { + for _, limit := range []string{"50", "100"} { + q := url.Values{limitParam: {limit}} + for i := 0; i < 1000; i++ { + path := prefix + "?" + q.Encode() + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, target+path, nil) + if err != nil { + panic(err) + } + resp, err := client.Do(req) + if err != nil { + panic(err) + } + var body struct { + Metadata struct { + Next string `json:"nextCursor"` + } `json:"metadata"` + } + err = json.NewDecoder(resp.Body).Decode(&body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK || err != nil { + panic(fmt.Sprintf("catalog failed status=%d err=%v", resp.StatusCode, err)) + } + keys = append(keys, path) + if body.Metadata.Next == "" { + break + } + q.Set("cursor", body.Metadata.Next) + } + } + } + b, _ := json.Marshal(keys) + if err := os.WriteFile(keysFile, b, 0600); err != nil { + panic(err) + } + fmt.Fprintln(os.Stdout, "catalog keys", len(keys)) +} + +func loadKeys(keysFile string, traced bool) ([]byte, []string) { + raw, err := os.ReadFile(keysFile) + if err != nil { + panic(err) + } + var keys []string + if err = json.Unmarshal(raw, &keys); err != nil { + panic(err) + } + if len(keys) < 128 { + panic("need at least 128 real keys") + } + for _, k := range keys { + u, e := url.Parse(k) + if !validCatalogKey(u, e, traced, k) { + panic("invalid key") + } + } + return raw, keys +} + +type requestTiming struct { + Scheduled, Queued time.Time +} + +func makeRequester(client *http.Client, token string, results chan<- point) func(path, kind string, timing requestTiming, body []byte) { + return makeRequesterAt(client, target, token, results) +} + +func makeRequesterAt(client *http.Client, baseURL, token string, results chan<- point) func(path, kind string, timing requestTiming, body []byte) { + return func(path, kind string, timing requestTiming, body []byte) { + started := time.Now() + method := "GET" + if body != nil { + method = "POST" + } + trace := newRequestTrace() + req, e := http.NewRequestWithContext(trace.context(context.Background()), method, baseURL+path, bytes.NewReader(body)) + p := point{kind: kind, shape: benchmarkRequestShape(path, kind)} + if e == nil { + if body != nil { + req.Header.Set("Authorization", "Bearer "+token) + req.Header.Set("Content-Type", "application/json") + } + resp, e := client.Do(req) + if e == nil { + p.status = resp.StatusCode + p.protocol = resp.Proto + p.cache = resp.Header.Get("X-Registry-Cache") + p.bytes, e = io.Copy(io.Discard, resp.Body) + resp.Body.Close() + if e != nil { + p.status = 0 + p.failure = "body:" + errorClass(e) + p.detail = errorDetail(e) + } + } else { + p.failure = "request:" + errorClass(e) + p.detail = errorDetail(e) + } + } + finished := time.Now() + p.phases = trace.observation(finished) + p.elapsed = float64(finished.Sub(timing.Scheduled)) / float64(time.Millisecond) + p.scheduledAt, p.queuedAt = timing.Scheduled, timing.Queued + p.startedAt, p.finishedAt = started, finished + results <- p + } +} + +func loadToken(tokenFile string) string { + token := "" + if tokenFile != "" { + b, e := os.ReadFile(tokenFile) + if e != nil { + panic(e) + } + token = strings.TrimSpace(string(b)) + } + return token +} + +func readPath(n, hot, rate, updatedRate, searchRate int, keys []string) (string, string) { + idx := pageIndex(n, hot, len(keys)) + path := keys[idx] + kind := pageKind + slot := n % rate + // Spread the low-rate query classes across each second. + specialRate := updatedRate + searchRate + before := slot * specialRate / rate + after := (slot + 1) * specialRate / rate + search := after > before && before < searchRate + updated := after > before && !search + if search { + sequence := n/rate*searchRate + before + q := url.Values{searchKind: {[]string{"github", "mcp", "postgres", "server", "cloud"}[sequence%5]}, limitParam: {fmt.Sprint(50 + sequence%51)}} + path = "/v0.1/servers?" + q.Encode() + kind = searchKind + } else if updated { + age := []time.Duration{7 * 24 * time.Hour, 24 * time.Hour, time.Hour}[n%3] + stamp := time.Now().UTC().Add(-age - time.Duration(n)*time.Microsecond).Format(time.RFC3339Nano) + path = "/v0.1/servers?" + url.Values{"updated_since": {stamp}, limitParam: {"100"}}.Encode() + kind = "updated" + } + return path, kind +} + +func validateArguments(ack bool, rate, updatedRate, searchRate, workers int, duration time.Duration, hot int) { + if updatedRate < 0 || searchRate < 0 || updatedRate+searchRate > rate || !ack || rate < 1 || rate > 10000 || workers < 1 || workers > 2048 || duration <= 0 || duration > 30*time.Minute || hot < 0 || hot > 100 { + panic("explicit staging acknowledgement and bounded arguments required") + } +} + +func collectResults(results <-chan point, r *result, samples map[string][]float64, timeline *timeline, done chan<- struct{}) { + defer close(done) + r.RequestShapes = make(map[string]int) + phases := make(map[string]*phaseAccumulator) + for p := range results { + r.RequestShapes[p.shape]++ + key := phaseKey(p.kind, p.cache) + if phases[key] == nil { + phases[key] = &phaseAccumulator{} + } + phases[key].add(p) + r.Completed++ + timeline.completed(p) + if p.failure != "" || p.status >= http.StatusBadRequest { + // Bounded, credential-free timing evidence for correlating failures + // with ingress/node transitions. Never retain URLs or request bodies. + if len(r.Failures) < 100 { + r.Failures = append(r.Failures, requestFailure{p.scheduledAt.UTC(), p.finishedAt.UTC(), p.kind, p.failure, p.status}) + } else { + r.OmittedFailures++ + } + } + if p.failure != "" { + r.Errors[p.kind+":"+p.failure]++ + if _, ok := r.ErrorDetails[p.failure]; !ok { + r.ErrorDetails[p.failure] = p.detail + } + } + r.Bytes += p.bytes + r.Status[fmt.Sprintf("%s:%d", p.kind, p.status)]++ + r.Cache[p.cache]++ + r.Protocols[p.protocol]++ + samples[p.kind] = append(samples[p.kind], p.elapsed) + } + r.RequestPhases = make(map[string]phaseSummary, len(phases)) + for key, value := range phases { + r.RequestPhases[key] = value.summary() + } +} diff --git a/tools/ingressbench/main_test.go b/tools/ingressbench/main_test.go new file mode 100644 index 000000000..281f1f6ae --- /dev/null +++ b/tools/ingressbench/main_test.go @@ -0,0 +1,82 @@ +package main + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// Calibration must actually exercise the full real-cursor catalog, rather than +// accidentally revisiting a small subset because its length divides the rate. +func TestVariedPagesReachWholeCatalog(t *testing.T) { + for _, hot := range []int{0, 50, 80, 99} { + seen := map[int]bool{} + for n := 0; n < 4000*100; n++ { + i := pageIndex(n, hot, 4000) + if i < 0 || i >= 4000 { + t.Fatalf("invalid index %d", i) + } + seen[i] = true + } + if len(seen) != 4000 { + t.Fatalf("hot=%d visited %d/4000 keys", hot, len(seen)) + } + } +} + +type unavailableTransport struct{} + +func (unavailableTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusServiceUnavailable, Body: io.NopCloser(strings.NewReader("private-body")), Header: make(http.Header), Request: req}, nil +} + +func TestFailureSamplesIncludeTimingWithoutRequestData(t *testing.T) { + points := make(chan point, 1) + scheduled := time.Now().Add(-time.Second) + client := &http.Client{Transport: unavailableTransport{}} + makeRequester(client, "private-token", points)("/v0/servers?search=private-query", "page", requestTiming{scheduled, scheduled}, nil) + close(points) + r := result{Status: map[string]int{}, Cache: map[string]int{}, Errors: map[string]int{}, Protocols: map[string]int{}, ErrorDetails: map[string]string{}} + done := make(chan struct{}) + collectResults(points, &r, map[string][]float64{}, newTimeline(), done) + if len(r.Failures) != 1 || r.Failures[0].Status != http.StatusServiceUnavailable || + !r.Failures[0].ScheduledAt.Equal(scheduled) || !r.Failures[0].FinishedAt.After(scheduled) { + t.Fatalf("missing failure timing: %+v", r.Failures) + } + encoded, err := json.Marshal(r) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(encoded), "private-") { + t.Fatal("request data leaked into results") + } +} + +func TestExplicitHTTPProtocol(t *testing.T) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusNoContent) })) + server.EnableHTTP2 = true + server.StartTLS() + defer server.Close() + for _, h2 := range []bool{false, true} { + tr := newTransport(h2) + tr.TLSClientConfig = server.Client().Transport.(*http.Transport).TLSClientConfig + defer tr.CloseIdleConnections() + client := &http.Client{Transport: tr} + resp, err := client.Get(server.URL) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + want := 1 + if h2 { + want = 2 + } + if resp.ProtoMajor != want { + t.Fatalf("allowHTTP2=%v negotiated %s", h2, resp.Proto) + } + } +} diff --git a/tools/ingressbench/page_workload.go b/tools/ingressbench/page_workload.go new file mode 100644 index 000000000..173c6c416 --- /dev/null +++ b/tools/ingressbench/page_workload.go @@ -0,0 +1,180 @@ +package main + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "math/rand/v2" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "time" +) + +const latestVersion = "latest" + +// PageWorkload models a measured page mix. It is not a replay of client crawl +// sessions: temporal ordering and connection reuse need separate validation. +type PageWorkload struct { + Classes []PageClass `json:"classes"` + SHA256 string `json:"-"` + paths []string +} + +type PageClass struct { + APIVersion string `json:"api_version"` + Limit int `json:"limit"` + VersionFilter string `json:"version_filter"` + // Each count is the observed frequency of one distinct URI in this class. + // These anonymous ranks map to distinct real staging URLs, not client IDs. + KeyCounts []int `json:"key_counts"` +} + +func loadPageWorkload(path string, keys []string) (*PageWorkload, error) { + if path == "" { + return &PageWorkload{}, nil + } + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + var profile PageWorkload + if err := json.Unmarshal(raw, &profile); err != nil { + return nil, err + } + if err := profile.prepare(keys); err != nil { + return nil, err + } + profile.SHA256 = fmt.Sprintf("%x", sha256.Sum256(raw)) + return &profile, nil +} + +func (p *PageWorkload) prepare(keys []string) error { + if len(p.Classes) == 0 || len(p.Classes) > 200 { + return fmt.Errorf("page workload needs 1–200 measured classes") + } + p.paths = nil + rng := rand.New(rand.NewPCG(1, 2)) //nolint:gosec // Reproducible workload permutation, not security-sensitive randomness. + seenClasses := map[string]bool{} + for _, class := range p.Classes { + if (class.APIVersion != "v0" && class.APIVersion != "v0.1") || class.Limit < 1 || class.Limit > 100 || + (class.VersionFilter != "" && class.VersionFilter != latestVersion) || len(class.KeyCounts) == 0 { + return fmt.Errorf("invalid page workload class") + } + identity := fmt.Sprintf("%s/%d/%s", class.APIVersion, class.Limit, class.VersionFilter) + if seenClasses[identity] { + return fmt.Errorf("duplicate page workload class %s", identity) + } + seenClasses[identity] = true + candidates := matchingPageKeys(class, keys) + if len(candidates) < len(class.KeyCounts) { + return fmt.Errorf("class %s needs %d distinct staging keys; catalog has %d", identity, len(class.KeyCounts), len(candidates)) + } + // Catalog ordering must not choose the API version or concentrate hot + // ranks near the beginning of the registry. + rng.Shuffle(len(candidates), func(i, j int) { candidates[i], candidates[j] = candidates[j], candidates[i] }) + for i, count := range class.KeyCounts { + if count <= 0 || count > 1_000_000-len(p.paths) { + return fmt.Errorf("page workload counts must be positive and total at most 1000000") + } + for range count { + p.paths = append(p.paths, candidates[i]) + } + } + } + rng.Shuffle(len(p.paths), func(i, j int) { p.paths[i], p.paths[j] = p.paths[j], p.paths[i] }) + return nil +} + +func (p *PageWorkload) path(sequence int) string { + return p.paths[sequence%len(p.paths)] +} + +// Count page slots preceding n, including pages dropped by the bounded queue. +// Special query slots must not bias which measured page classes are exercised. +func pageSequence(n, rate, specialRate int) int { + return n/rate*(rate-specialRate) + n%rate - n%rate*specialRate/rate +} + +func matchingPageKeys(class PageClass, keys []string) []string { + var candidates []string + seen := map[string]bool{} + for _, key := range keys { + u, err := url.Parse(key) + if err != nil || u.IsAbs() || u.Host != "" || u.Fragment != "" || u.Path != "/"+class.APIVersion+"/servers" { + continue + } + q := u.Query() + valid := q.Get("limit") == fmt.Sprint(class.Limit) && q.Get(versionParam) == class.VersionFilter + for name, values := range q { + if (name != "cursor" && name != "limit" && name != versionParam) || len(values) != 1 { + valid = false + } + } + canonical := u.Path + "?" + q.Encode() + if valid && !seen[canonical] { + seen[canonical] = true + candidates = append(candidates, canonical) + } + } + return candidates +} + +func validateConnectionRotation(interval time.Duration) { + if interval < 0 || interval > 10*time.Minute { + panic("connection rotation must be between 0 and 10 minutes") + } +} + +func rotateIdleConnections(transport *http.Transport, interval time.Duration) func() { + done := make(chan struct{}) + if interval == 0 { + return func() { close(done) } + } + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + transport.CloseIdleConnections() + case <-done: + return + } + } + }() + return func() { close(done) } +} + +// Bounded structural labels let calibration compare delivered mix, without +// retaining search terms, cursor values or server names in results. +func benchmarkRequestShape(path, kind string) string { + u, err := url.Parse(path) + if err != nil { + return kind + "|invalid" + } + api := "other" + if strings.HasPrefix(u.Path, "/v0.1/") { + api = "v0.1" + } else if strings.HasPrefix(u.Path, "/v0/") { + api = "v0" + } + q := u.Query() + limit := "default" + if value := q.Get("limit"); value != "" { + count, parseErr := strconv.Atoi(value) + limit = "invalid" + if parseErr == nil && count > 0 && count <= 100 { + limit = strconv.Itoa(count) + } + } + version := "none" + if value := q.Get(versionParam); value == latestVersion { + version = latestVersion + } else if value != "" { + version = "exact" + } + return fmt.Sprintf("%s|%s|%s|%s|cursor=%t", kind, api, limit, version, q.Get("cursor") != "") +} diff --git a/tools/ingressbench/page_workload_test.go b/tools/ingressbench/page_workload_test.go new file mode 100644 index 000000000..b8ec82e7f --- /dev/null +++ b/tools/ingressbench/page_workload_test.go @@ -0,0 +1,70 @@ +package main + +import ( + "net/url" + "strings" + "testing" +) + +func TestMeasuredPageMixIgnoresCatalogOrdering(t *testing.T) { + keys := []string{"/v0/servers?limit=50&cursor=a", "/v0/servers?limit=50&cursor=b", "/v0.1/servers?limit=100&cursor=a", "/v0.1/servers?limit=100&cursor=b"} + p := PageWorkload{Classes: []PageClass{ + {APIVersion: "v0.1", Limit: 100, KeyCounts: []int{7, 2}}, + {APIVersion: "v0", Limit: 50, KeyCounts: []int{1}}, + }} + if err := p.prepare(keys); err != nil { + t.Fatal(err) + } + counts := map[string]int{} + for n := range 10 { + u, err := url.Parse(p.path(n)) + if err != nil { + t.Fatal(err) + } + counts[u.Path+"|"+u.Query().Get("limit")]++ + } + if counts["/v0.1/servers|100"] != 9 || counts["/v0/servers|50"] != 1 { + t.Fatalf("catalog ordering changed measured mix: %v", counts) + } +} + +func TestMeasuredPageMixRejectsInsufficientDiversity(t *testing.T) { + p := PageWorkload{Classes: []PageClass{{APIVersion: "v0.1", Limit: 100, KeyCounts: []int{2, 1}}}} + err := p.prepare([]string{"/v0.1/servers?limit=100&cursor=a", "/v0.1/servers?cursor=a&limit=100"}) + if err == nil || !strings.Contains(err.Error(), "catalog has 1") { + t.Fatalf("must not silently collapse distinct production keys: %v", err) + } +} + +func TestMeasuredPageMixExcludesOtherFiltersAndExternalURLs(t *testing.T) { + p := PageWorkload{Classes: []PageClass{{APIVersion: "v0.1", Limit: 100, KeyCounts: []int{1}}}} + err := p.prepare([]string{"https://example.invalid/v0.1/servers?limit=100", "/v0.1/servers?limit=100&search=private", "/v0.1/servers?limit=100&version=latest"}) + if err == nil { + t.Fatal("matched an external URL or a different query class") + } +} + +func TestPageSequenceSurvivesSpecialQuerySlots(t *testing.T) { + keys := make([]string, 128) + var want int + for n := range 9000 { + _, kind := readPath(n, 80, 4500, 36, 3, keys) + if kind != "page" { + continue + } + if got := pageSequence(n, 4500, 39); got != want { + t.Fatalf("slot %d: got %d want %d", n, got, want) + } + want++ + } +} + +func TestRequestShapeDoesNotRetainParameterValues(t *testing.T) { + got := benchmarkRequestShape("/v0.1/servers?limit=100&search=private&cursor=private&version=private", "search") + if got != "search|v0.1|100|exact|cursor=true" { + t.Fatal(got) + } + if got := benchmarkRequestShape("/v0.1/servers?limit=private", "page"); strings.Contains(got, "private") { + t.Fatal("unbounded label") + } +} diff --git a/tools/ingressbench/phases.go b/tools/ingressbench/phases.go new file mode 100644 index 000000000..9f3bc9081 --- /dev/null +++ b/tools/ingressbench/phases.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "crypto/tls" + "net/http" + "net/http/httptest" + "net/http/httptrace" + "os" + "runtime/pprof" + "strings" + "sync" + "time" +) + +// The fixture exercises scheduling, HTTPS and response consumption on the load +// host. It includes server overhead, but cannot validate the external network. +func newCalibrationServer() *httptest.Server { + body := []byte(`{"data":"` + strings.Repeat("x", 65536-11) + `"}`) + return httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Registry-Cache", "HIT") + _, _ = w.Write(body) + })) +} + +type phaseObservation struct { + connectionWait, tls, firstByte, body time.Duration + gotConnection, didTLS, gotFirstByte, reused bool +} + +type requestTrace struct { + mu sync.Mutex + getConnection, gotConnection, tlsStart, wroteRequest, firstByte time.Time + tlsDuration time.Duration + didTLS, reused bool +} + +func newRequestTrace() *requestTrace { return &requestTrace{} } + +// Hooks can execute on transport goroutines, including after request failure. +// No addresses, URLs, headers or credentials are retained. +func (t *requestTrace) context(ctx context.Context) context.Context { + return httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{ + GetConn: func(_ string) { t.mu.Lock(); defer t.mu.Unlock(); t.getConnection = time.Now() }, + GotConn: func(info httptrace.GotConnInfo) { + t.mu.Lock() + defer t.mu.Unlock() + t.gotConnection = time.Now() + t.reused = info.Reused + }, + TLSHandshakeStart: func() { t.mu.Lock(); defer t.mu.Unlock(); t.tlsStart = time.Now() }, + TLSHandshakeDone: func(_ tls.ConnectionState, _ error) { + t.mu.Lock() + defer t.mu.Unlock() + if !t.tlsStart.IsZero() { + t.tlsDuration += time.Since(t.tlsStart) + t.didTLS = true + } + }, + WroteRequest: func(_ httptrace.WroteRequestInfo) { t.mu.Lock(); defer t.mu.Unlock(); t.wroteRequest = time.Now() }, + GotFirstResponseByte: func() { t.mu.Lock(); defer t.mu.Unlock(); t.firstByte = time.Now() }, + }) +} + +func (t *requestTrace) observation(finished time.Time) phaseObservation { + t.mu.Lock() + defer t.mu.Unlock() + o := phaseObservation{tls: t.tlsDuration, didTLS: t.didTLS, reused: t.reused} + if !t.getConnection.IsZero() && !t.gotConnection.IsZero() { + o.gotConnection = true + o.connectionWait = t.gotConnection.Sub(t.getConnection) + } + if !t.firstByte.IsZero() { + o.gotFirstByte = true + o.body = finished.Sub(t.firstByte) + if !t.wroteRequest.IsZero() { + o.firstByte = t.firstByte.Sub(t.wroteRequest) + } + } + return o +} + +type phaseSummary struct { + Attempts, NewConnections, ReusedConnections int + SchedulerLag, QueueDelay, ConnectionWait, TLS, FirstByte, Body, HTTP durationSummary +} + +type phaseAccumulator struct { + attempts, newConnections, reusedConnections int + schedulerLag, queueDelay, connectionWait, tls, firstByte, body, http durationHistogram +} + +func phaseKey(kind, cache string) string { + switch cache { + case "HIT", "MISS", "STALE", "UPDATING", "BYPASS", "EXPIRED", "REVALIDATED": + default: + cache = "UNKNOWN" + } + return kind + ":" + cache +} + +func (a *phaseAccumulator) add(p point) { + a.attempts++ + a.schedulerLag.add(p.queuedAt.Sub(p.scheduledAt)) + a.queueDelay.add(p.startedAt.Sub(p.queuedAt)) + a.http.add(p.finishedAt.Sub(p.startedAt)) + o := p.phases + if o.gotConnection { + a.connectionWait.add(o.connectionWait) + if o.reused { + a.reusedConnections++ + } else { + a.newConnections++ + } + } + if o.didTLS { + a.tls.add(o.tls) + } + if o.gotFirstByte { + a.firstByte.add(o.firstByte) + a.body.add(o.body) + } +} + +func (a *phaseAccumulator) summary() phaseSummary { + return phaseSummary{a.attempts, a.newConnections, a.reusedConnections, + a.schedulerLag.summary(), a.queueDelay.summary(), a.connectionWait.summary(), a.tls.summary(), a.firstByte.summary(), a.body.summary(), a.http.summary()} +} + +func configureCalibration(tr *http.Transport, enabled bool, tokenFile string, catalog bool) (string, func()) { + if !enabled { + return target, func() {} + } + if tokenFile != "" || catalog { + panic("calibration cannot publish or collect a staging catalog") + } + fixture := newCalibrationServer() + tr.TLSClientConfig = fixture.Client().Transport.(*http.Transport).TLSClientConfig + return fixture.URL, fixture.Close +} + +func startCPUProfile(path string) func() { + if path == "" { + return func() {} + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + panic(err) + } + if err := pprof.StartCPUProfile(f); err != nil { + _ = f.Close() + panic(err) + } + return func() { pprof.StopCPUProfile(); _ = f.Close() } +} diff --git a/tools/ingressbench/phases_test.go b/tools/ingressbench/phases_test.go new file mode 100644 index 000000000..af5d568bd --- /dev/null +++ b/tools/ingressbench/phases_test.go @@ -0,0 +1,86 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func TestRequestPhasesSeparateServerAndBodyDelayAndReuse(t *testing.T) { + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + time.Sleep(25 * time.Millisecond) + w.Header().Set("X-Registry-Cache", "HIT") + w.WriteHeader(http.StatusOK) + w.(http.Flusher).Flush() + time.Sleep(25 * time.Millisecond) + _, _ = w.Write([]byte("ok")) + })) + defer srv.Close() + client := srv.Client() + defer client.CloseIdleConnections() + ch := make(chan point, 2) + request := makeRequesterAt(client, srv.URL, "", ch) + for range 2 { + now := time.Now() + request("/", "page", requestTiming{now.Add(-50 * time.Millisecond), now}, nil) + } + first, second := <-ch, <-ch + if first.failure != "" || second.failure != "" { + t.Fatal(first.failure, second.failure) + } + if !first.phases.didTLS || first.phases.reused || !second.phases.reused || second.phases.didTLS { + t.Fatalf("unexpected connection reuse: first=%+v second=%+v", first.phases, second.phases) + } + for _, p := range []point{first, second} { + if p.phases.firstByte < 20*time.Millisecond || p.phases.body < 20*time.Millisecond { + t.Fatalf("server and body phases not separated: %+v", p.phases) + } + if p.elapsed < 90 { + t.Fatalf("scheduled latency lost queue time: %f", p.elapsed) + } + } +} + +func TestCalibrationFixtureSizeAndCacheHeader(t *testing.T) { + srv := newCalibrationServer() + defer srv.Close() + ch := make(chan point, 1) + now := time.Now() + makeRequesterAt(srv.Client(), srv.URL, "", ch)("/v0.1/servers", "page", requestTiming{now, now}, nil) + p := <-ch + if p.status != 200 || p.bytes != 65536 || p.cache != "HIT" { + t.Fatalf("invalid fixture result: %+v", p) + } +} + +func TestPhaseAggregationDoesNotInventMissingTransportEvents(t *testing.T) { + a := &phaseAccumulator{} + a.add(point{kind: "page", failure: "timeout"}) + s := a.summary() + if s.Attempts != 1 || s.ConnectionWait.Count != 0 || s.FirstByte.Count != 0 || s.Body.Count != 0 || s.TLS.Count != 0 { + t.Fatalf("missing events were treated as observations: %+v", s) + } + if phaseKey("page", "untrusted-header") != "page:UNKNOWN" { + t.Fatal("unbounded cache label") + } +} + +func TestCalibrationRejectsPublishingAndCatalogCollection(t *testing.T) { + for _, tc := range []struct { + name, token string + catalog bool + }{ + {"publish", "private-token-path", false}, {"catalog", "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + defer func() { + if recover() == nil { + t.Fatal("unsafe calibration mode accepted") + } + }() + _, closeFixture := configureCalibration(newTransport(false), true, tc.token, tc.catalog) + defer closeFixture() + }) + } +} diff --git a/tools/ingressbench/profile_logs.py b/tools/ingressbench/profile_logs.py new file mode 100644 index 000000000..36140bac0 --- /dev/null +++ b/tools/ingressbench/profile_logs.py @@ -0,0 +1,186 @@ +"""Summarize ingress access records without persisting URLs or client identifiers. + +The collector supplies complete, bounded time slices and records pagination +coverage separately. This module does not turn a sample into a traffic census. +""" +import collections +import datetime +import hashlib +import math +import re +import secrets +import urllib.parse + + +ACCESS = re.compile( + r'^\S+ - \S+ \[[^\]]+\] "(?P\S+) (?P\S+) (?PHTTP/[\d.]+)" ' + r'(?P\d{3}) (?P\d+) "[^"\n]*" "[^"\n]*" ' + r'(?P\d+) (?P[\d.]+) ' +) +SAFE_CACHE = {"HIT", "MISS", "STALE", "UPDATING", "EXPIRED", "BYPASS", "REVALIDATED"} + + +def parse_timestamp(value): + # Python 3.9 accepts microseconds; Cloud Logging uses RFC3339 nanoseconds. + value = re.sub(r'(\.\d{6})\d+', r'\1', value) + result = datetime.datetime.fromisoformat(value.replace('Z', '+00:00')) + if result.tzinfo is None: + raise ValueError('timezone missing') + return result + + +def percentile(values, fraction): + return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)] if values else None + + +def size_band(value): + for upper in (1024, 4096, 16384, 65536, 262144, 1048576): + if value <= upper: + return str(upper) + return ">1048576" + + +def request_shape(uri): + url = urllib.parse.urlsplit(uri) + params = urllib.parse.parse_qs(url.query, keep_blank_values=True) + version = "v0.1" if url.path.startswith('/v0.1/') else "v0" if url.path.startswith('/v0/') else "other" + listing = url.path in ('/v0/servers', '/v0.1/servers') + if listing: + kind = "search" if params.get('search', [''])[0] else "updated" if params.get('updated_since', [''])[0] else "page" + elif re.fullmatch(r'/v0(?:\.1)?/servers/.+/versions(?:/.+)?', url.path): + kind = "detail" + elif url.path in ('/v0/publish', '/v0.1/publish'): + kind = "publish" + else: + kind = "other" + limit = params.get('limit', ['default'])[0] + # Bound values; never preserve an arbitrary parameter value as a label. + if limit != 'default': + limit = str(int(limit)) if limit.isdecimal() and 0 < int(limit) <= 100 else 'invalid' + return kind, version, limit, bool(params.get('cursor', [''])[0]), params + + +class Profile: + def __init__(self): + self.salt = secrets.token_bytes(32) + self.parsed = self.unparsed = 0 + self.groups = {} + self.seconds = collections.Counter() + self.shapes = collections.Counter() + self.key_counts = collections.Counter() + self.key_sets_by_second = collections.defaultdict(set) + self.pages_with_cursor = 0 + self.search_lengths = collections.Counter() + self.updated_ages = collections.Counter() + self.page_filters = collections.Counter() + self.page_class_keys = collections.defaultdict(collections.Counter) + self.unsupported_page_requests = 0 + self.first = self.last = None + + def add(self, message, timestamp, pod): + match = ACCESS.match(message) + if not match: + self.unparsed += 1 + return + row = match.groupdict() + if row['method'] not in ('GET', 'HEAD', 'POST', 'PUT', 'PATCH', 'DELETE', 'OPTIONS'): + row['method'] = 'other' + if row['protocol'] not in ('HTTP/1.0', 'HTTP/1.1', 'HTTP/2', 'HTTP/2.0', 'HTTP/3', 'HTTP/3.0'): + row['protocol'] = 'other' + self.parsed += 1 + # NGINX background subrequests have zero request_length in this log + # format. Preserve this population separately rather than inflate RPS. + group = 'client' if int(row['request_bytes']) > 0 else 'zero_request_length' + state = self.groups.setdefault(group, { + 'count': 0, 'cache': collections.Counter(), 'protocol': collections.Counter(), + 'kind': collections.Counter(), 'version': collections.Counter(), + 'status': collections.Counter(), 'pod': collections.Counter(), + 'response_bytes': [], 'duration_seconds': [], 'body_size_buckets': collections.Counter(), + }) + kind, version, limit, cursor, params = request_shape(row['uri']) + cache = re.search(r'cache_status="([^"]*)"', message) + cache = cache.group(1) if cache and cache.group(1) in SAFE_CACHE else 'none' + state['count'] += 1 + for field, value in [('cache', cache), ('protocol', row['protocol']), ('kind', kind), + ('version', version), ('status', row['status']), ('pod', pod)]: + state[field][value] += 1 + state['response_bytes'].append(int(row['bytes'])) + state['duration_seconds'].append(float(row['duration'])) + state['body_size_buckets'][size_band(int(row['bytes']))] += 1 + self.first = min(self.first, timestamp) if self.first is not None else timestamp + self.last = max(self.last, timestamp) if self.last is not None else timestamp + if group != 'client': + return + self.seconds[int(timestamp)] += 1 + self.shapes['|'.join([row['method'], kind, version, limit, row['protocol'], 'cursor' if cursor else 'first'])] += 1 + if kind == 'page': + key = hashlib.blake2b(row['uri'].encode(), key=self.salt).digest() + self.key_counts[key] += 1 + self.key_sets_by_second[int(timestamp)].add(key) + self.pages_with_cursor += int(cursor) + version_filter = params.get('version', [''])[0] + version_filter = 'none' if not version_filter else 'latest' if version_filter == 'latest' else 'exact' + known = {'cursor', 'limit', 'version', 'search', 'updated_since', 'include_deleted'} + extras = 'other_params' if set(params) - known else 'known_params_only' + deleted = 'include_deleted' if params.get('include_deleted', [''])[0].lower() == 'true' else 'active_only' + self.page_filters['|'.join([version, limit, version_filter, deleted, extras])] += 1 + effective_limit = '30' if limit == 'default' else limit + if version in ('v0', 'v0.1') and effective_limit != 'invalid' and version_filter in ('none', 'latest') and deleted == 'active_only' and extras == 'known_params_only': + # Normalize query ordering/default limit for the model; raw + # URI diversity above remains the observed cache-key measure. + canonical = dict(params) + canonical['limit'] = [effective_limit] + normalized = urllib.parse.urlsplit(row['uri']).path + '?' + urllib.parse.urlencode(sorted(canonical.items()), doseq=True) + model_key = hashlib.blake2b(normalized.encode(), key=self.salt).digest() + self.page_class_keys[(version, int(effective_limit), '' if version_filter == 'none' else 'latest')][model_key] += 1 + else: + self.unsupported_page_requests += 1 + if kind == 'search': + n = len(params['search'][0]) + self.search_lengths[str(n) if n <= 20 else '>20'] += 1 + if kind == 'updated': + try: + dt = parse_timestamp(params['updated_since'][0]) + age = timestamp - dt.timestamp() + band = 'future' if age < 0 else '<1h' if age < 3600 else '<1d' if age < 86400 else '<7d' if age < 604800 else '>=7d' + except (ValueError, OverflowError): + band = 'invalid' + self.updated_ages[band] += 1 + + def result(self): + groups = {} + for name, state in self.groups.items(): + groups[name] = {k: dict(v) if isinstance(v, collections.Counter) else v + for k, v in state.items() if k not in ('response_bytes', 'duration_seconds')} + for name2 in ('response_bytes', 'duration_seconds'): + values = state[name2] + groups[name][name2] = {'mean': sum(values) / len(values), 'p50': percentile(values, .5), + 'p95': percentile(values, .95), 'p99': percentile(values, .99), 'max': max(values)} + counts = sorted(self.key_counts.values(), reverse=True) + pages = sum(counts) + return { + 'parsed': self.parsed, 'unparsed': self.unparsed, 'first_log_timestamp': self.first, + 'last_log_timestamp': self.last, 'populations': groups, + 'client_completions_per_second': dict(sorted(self.seconds.items())), + 'client_shapes': dict(self.shapes), + 'page_keys': {'distinct': len(counts), 'requests': pages, 'cursor_requests': self.pages_with_cursor, + 'top_128_share': sum(counts[:128]) / pages if pages else None, + 'singleton_keys': sum(n == 1 for n in counts), + 'distinct_per_second': {str(t): len(s) for t, s in sorted(self.key_sets_by_second.items())}}, + 'search_length_counts': dict(self.search_lengths), 'updated_age_counts': dict(self.updated_ages), + 'page_filter_counts': dict(self.page_filters), + 'page_workload': { + 'classes': [{'api_version': version, 'limit': limit, 'version_filter': filt, + 'key_counts': sorted(counts.values(), reverse=True)} + for (version, limit, filt), counts in sorted(self.page_class_keys.items())], + 'excluded_page_requests': self.unsupported_page_requests, + 'limitations': ['Frequency model only; chronological crawl/client ordering is not replayed.', + 'Query ordering and default limit normalized; original cache-key diversity is reported separately.', + 'Searches, updates, details and publishing need their own measured workloads.'], + }, + 'limitations': ['Time slices and completion-time bins, not a full request arrival trace.', + 'Zero request_length separated as a background-work proxy.', + 'Body byte counts are logged wire sizes; compression can differ from generator decoded bytes.', + 'No TCP connection IDs in this log format: connection reuse/lifetime remains unmeasured.', + 'No URLs, search text, IPs, user agents, request IDs or key hashes are persisted.'], + } diff --git a/tools/ingressbench/read_trace.go b/tools/ingressbench/read_trace.go new file mode 100644 index 000000000..7095a918d --- /dev/null +++ b/tools/ingressbench/read_trace.go @@ -0,0 +1,155 @@ +package main + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "net/url" + "os" + "strings" + "time" +) + +const ( + v0ListingPath = "/v0/servers" + pageKind = "page" + searchKind = "search" + versionParam = "version" + v01ListingPath = "/v0.1/servers" +) + +// ReadTrace contains only catalog references and relative request start times. +// Keep the source catalog private; results expose its hash, never its values. +type ReadTrace struct { + DurationMS int64 `json:"duration_ms"` + Requests []TraceRead `json:"requests"` + SHA256 string `json:"-"` +} + +type TraceRead struct { + AtMS int64 `json:"at_ms"` + Key int `json:"key"` + Kind string `json:"kind"` +} + +func loadReadTrace(filename string, keys []string, duration time.Duration, pageProfile, token string) (*ReadTrace, error) { + t := &ReadTrace{} + if filename == "" { + return t, nil + } + if pageProfile != "" || token != "" { + return nil, fmt.Errorf("read trace cannot combine with a page profile or publishing") + } + raw, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + if err = json.Unmarshal(raw, t); err != nil { + return nil, err + } + if t.DurationMS != duration.Milliseconds() || len(t.Requests) == 0 || len(t.Requests) > 3_000_000 { + return nil, fmt.Errorf("trace duration/count outside bounds") + } + kinds := make([]string, len(keys)) + for i, key := range keys { + kinds[i] = traceReadKind(key) + } + var previous int64 = -1 + perSecond := map[int64]int{} + for _, entry := range t.Requests { + if entry.AtMS < previous || entry.AtMS < 0 || entry.AtMS >= t.DurationMS || entry.Key < 0 || entry.Key >= len(keys) { + return nil, fmt.Errorf("invalid trace time or catalog index") + } + if kinds[entry.Key] == "" || kinds[entry.Key] != entry.Kind { + return nil, fmt.Errorf("trace must reference a supported public GET read") + } + perSecond[entry.AtMS/1000]++ + if perSecond[entry.AtMS/1000] > 10000 { + return nil, fmt.Errorf("trace exceeds 10000 reads per second") + } + previous = entry.AtMS + } + t.SHA256 = fmt.Sprintf("%x", sha256.Sum256(raw)) + return t, nil +} + +func traceReadKind(path string) string { + u, err := url.Parse(path) + if err != nil || u.IsAbs() || u.Host != "" || u.User != nil || u.Fragment != "" { + return "" + } + q, err := url.ParseQuery(u.RawQuery) + if err != nil { + return "" + } + for key, values := range q { + if len(values) != 1 { + return "" + } + switch key { + case "cursor", "limit", versionParam, searchKind, "updated_since", "include_deleted": + default: + return "" + } + } + if u.Path == v0ListingPath || u.Path == v01ListingPath { + if q.Get(searchKind) != "" { + return searchKind + } + if q.Get("updated_since") != "" { + return "updated" + } + return pageKind + } + if (strings.HasPrefix(u.Path, "/v0/servers/") || strings.HasPrefix(u.Path, "/v0.1/servers/")) && strings.Contains(u.Path, "/versions") { + return "detail" + } + return "" +} + +func (t *ReadTrace) count(defaultCount int) int { + if len(t.Requests) > 0 { + return len(t.Requests) + } + return defaultCount +} +func (t *ReadTrace) offset(n, rate int) time.Duration { + if len(t.Requests) > 0 { + return time.Duration(t.Requests[n].AtMS) * time.Millisecond + } + return time.Duration(float64(n) / float64(rate) * float64(time.Second)) +} +func (t *ReadTrace) read(n int, keys []string, path, kind string) (string, string) { + if len(t.Requests) > 0 { + e := t.Requests[n] + return keys[e.Key], e.Kind + } + return path, kind +} +func (t *ReadTrace) describe(r *result) { + if len(t.Requests) > 0 { + r.ReadTraceSHA256 = t.SHA256 + r.ReadRate = -1 + r.SearchRate = -1 + r.UpdatedRate = -1 + r.HotPercent = -1 + } +} + +func selectedRead(n, hot, rate, updated, search int, keys []string, profile *PageWorkload, trace *ReadTrace) (string, string) { + if len(trace.Requests) > 0 { + return trace.read(n, keys, "", "") + } + path, kind := readPath(n, hot, rate, updated, search, keys) + if kind == pageKind && len(profile.paths) > 0 { + path = profile.path(pageSequence(n, rate, updated+search)) + } + return path, kind +} + +func validCatalogKey(u *url.URL, err error, traced bool, key string) bool { + if traced { + return traceReadKind(key) != "" + } + return err == nil && !u.IsAbs() && u.Host == "" && (u.Path == v0ListingPath || u.Path == v01ListingPath) +} diff --git a/tools/ingressbench/read_trace_test.go b/tools/ingressbench/read_trace_test.go new file mode 100644 index 000000000..85f6bc888 --- /dev/null +++ b/tools/ingressbench/read_trace_test.go @@ -0,0 +1,64 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" +) + +func TestTracePreservesBurstSpacingAndKeyReuse(t *testing.T) { + p := filepath.Join(t.TempDir(), "trace.json") + if err := os.WriteFile(p, []byte(`{"duration_ms":1000,"requests":[{"at_ms":10,"key":0,"kind":"page"},{"at_ms":10,"key":0,"kind":"page"},{"at_ms":900,"key":1,"kind":"search"}]}`), 0600); err != nil { + t.Fatal(err) + } + keys := []string{"/v0.1/servers?cursor=a&limit=100", "/v0/servers?search=private&limit=5"} + trace, err := loadReadTrace(p, keys, time.Second, "", "") + if err != nil { + t.Fatal(err) + } + if trace.count(99) != 3 || trace.offset(0, 99) != 10*time.Millisecond || trace.offset(1, 99) != 10*time.Millisecond || trace.offset(2, 99) != 900*time.Millisecond { + t.Fatal("arrival spacing changed") + } + path, kind := trace.read(2, keys, "unused", "unused") + if path != keys[1] || kind != "search" { + t.Fatal("request identity changed") + } +} +func TestTraceRejectsUnsafePathsAndInvalidOrdering(t *testing.T) { + for _, path := range []string{"https://example.com/v0/servers", "//example.com/v0/servers", "/v0.1/publish", "/v0/servers?token=private", "/v0/servers?limit=5&limit=100"} { + if traceReadKind(path) != "" { + t.Fatalf("accepted unsupported path %q", path) + } + } + p := filepath.Join(t.TempDir(), "trace.json") + if err := os.WriteFile(p, []byte(`{"duration_ms":1000,"requests":[{"at_ms":20,"key":0,"kind":"page"},{"at_ms":10,"key":0,"kind":"page"}]}`), 0600); err != nil { + t.Fatal(err) + } + if _, err := loadReadTrace(p, []string{"/v0/servers"}, time.Second, "", ""); err == nil { + t.Fatal("out of order trace accepted") + } + if _, err := loadReadTrace(p, []string{"/v0/servers"}, time.Second, "", "token-file"); err == nil { + t.Fatal("publishing accepted") + } +} + +func TestTraceCatalogAcceptsDetailReads(t *testing.T) { + keys := make([]string, 128) + for i := range keys { + keys[i] = "/v0.1/servers/io.example%2Fserver/versions/latest" + } + raw, err := json.Marshal(keys) + if err != nil { + t.Fatal(err) + } + p := filepath.Join(t.TempDir(), "keys.json") + if err = os.WriteFile(p, raw, 0600); err != nil { + t.Fatal(err) + } + _, loaded := loadKeys(p, true) + if len(loaded) != 128 || traceReadKind(loaded[0]) != "detail" { + t.Fatal("valid detail catalog rejected") + } +} diff --git a/tools/ingressbench/results/2026-10-28-connection-cpu.json b/tools/ingressbench/results/2026-10-28-connection-cpu.json new file mode 100644 index 000000000..5655d9c89 --- /dev/null +++ b/tools/ingressbench/results/2026-10-28-connection-cpu.json @@ -0,0 +1,133 @@ +{ + "schema_version": 1, + "production_changed": false, + "client_results": [ + { + "name": "release-only-250m", + "phase": "burst", + "start": "2026-10-27T23:54:40.144665768Z", + "end": "2026-10-28T00:04:42.398827704Z", + "scheduled_reads": 2700000, + "dropped_reads": 471814, + "dropped_percent": 17.474592592592593, + "read_failures_by_status": 269, + "errors": { + "page:body:connection_reset": 1, + "page:body:timeout": 5, + "page:request:timeout": 155, + "publish:request:timeout": 3, + "search:request:timeout": 4, + "updated:body:timeout": 4, + "updated:request:timeout": 100 + }, + "publishes_acknowledged": 597, + "publishes_scheduled": 600, + "publish_p95_ms": 7588.19546, + "page_p95_ms": 732.436179, + "page_p99_ms": 6102.625358, + "bytes": 128496823837, + "cache": { + "": 859, + "HIT": 1873152, + "MISS": 44340, + "STALE": 170159, + "UPDATING": 140276 + }, + "transport_failures": 272, + "traffic_gate_pass": false + }, + { + "name": "release-only-250m", + "phase": "recovery", + "start": "2026-10-28T00:05:12.530950572Z", + "end": "2026-10-28T00:35:12.529354398Z", + "scheduled_reads": 180000, + "dropped_reads": 0, + "dropped_percent": 0.0, + "read_failures_by_status": 0, + "errors": {}, + "publishes_acknowledged": 1800, + "publishes_scheduled": 1800, + "publish_p95_ms": 13.871881, + "page_p95_ms": 7.751579, + "page_p99_ms": 12.196983, + "bytes": 10394953505, + "cache": { + "": 1800, + "HIT": 126105, + "MISS": 20797, + "STALE": 33098 + }, + "transport_failures": 0, + "traffic_gate_pass": true + }, + { + "name": "release-cpu1", + "phase": "burst", + "start": "2026-10-28T00:42:40.549589261Z", + "end": "2026-10-28T00:52:40.706749832Z", + "scheduled_reads": 2700000, + "dropped_reads": 125976, + "dropped_percent": 4.665777777777778, + "read_failures_by_status": 0, + "errors": {}, + "publishes_acknowledged": 600, + "publishes_scheduled": 600, + "publish_p95_ms": 1349.588691, + "page_p95_ms": 596.716922, + "page_p99_ms": 1129.4293, + "bytes": 148439831516, + "cache": { + "": 600, + "HIT": 2308742, + "MISS": 45761, + "STALE": 210407, + "UPDATING": 9114 + }, + "transport_failures": 0, + "traffic_gate_pass": false + }, + { + "name": "release-cpu1", + "phase": "recovery", + "start": "2026-10-28T00:53:09.569680147Z", + "end": "2026-10-28T01:23:09.573134721Z", + "scheduled_reads": 180000, + "dropped_reads": 0, + "dropped_percent": 0.0, + "read_failures_by_status": 0, + "errors": {}, + "publishes_acknowledged": 1800, + "publishes_scheduled": 1800, + "publish_p95_ms": 14.165564, + "page_p95_ms": 7.111924, + "page_p99_ms": 10.78339, + "bytes": 10395101792, + "cache": { + "": 1800, + "HIT": 126201, + "MISS": 21000, + "STALE": 32799 + }, + "transport_failures": 0, + "traffic_gate_pass": true + } + ], + "limitations": [ + "CPU request changed placement and starting node count; this is not an isolated CPU-priority comparison.", + "Neither candidate passed the no-dropped-work burst criterion.", + "CPU1 candidate retained three idle nodes; autoscaler reported no place to move the app pod.", + "Historical application metrics and original recovery result recovered after reauthentication; missing direct DB cgroup/endpoint snapshots during recovery remain an observation gap." + ], + "cleanup": "Complete: generator VM/disk, scoped publications and whitelist removed; baseline image/250m CPU restored; two Ready nodes verified around02:59UTC.", + "idle_nodes": { + "release-only-250m": 2, + "release-cpu1": 3 + }, + "recovery_backfill": { + "original_client_result_matches_stdout": true, + "all_sampled_app_up_values": 1, + "max_sampled_pool_waiters": 0 + }, + "cleanup_idle_nodes": 2 +} diff --git a/tools/ingressbench/results/2026-10-28-warm-capacity.json b/tools/ingressbench/results/2026-10-28-warm-capacity.json new file mode 100644 index 000000000..8e3280797 --- /dev/null +++ b/tools/ingressbench/results/2026-10-28-warm-capacity.json @@ -0,0 +1,1788 @@ +{ + "experiment": { + "recorded_at": "2026-10-28T08:33:27.785815+00:00", + "question": "Does ready capacity remove the burst loss, or does steady-state capacity remain inadequate?", + "production_changes": false, + "image": "us-central1-docker.pkg.dev/mcp-registry-staging/registry-staging-experiments/registry:staging-connection-release-20261028-v1@sha256:94f184cd38345b059f440756d5eaa22448dbc7d827490220d144bd9356713f94", + "app": { + "replicas": 2, + "cpu_request_m": 250, + "cpu_limit": null, + "pool_max_per_replica": 60, + "pool_min_per_replica": 10, + "placement": "Same pod UIDs and original two nodes verified before each arm" + }, + "database": { + "cpu_request": 2, + "cpu_limit": 3, + "memory_GiB": 4, + "shared_buffers_MiB": 512, + "max_connections": 200 + }, + "driver_sha256": "96693b313ae702e76943ef2242e3de5c471ff3d0f0011b6999a51b88dfeee1a2", + "catalog_sha256": "120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be", + "protocol": "HTTP/1.1", + "hot_percent": 80, + "hot_keys": 128, + "generator": "e2-standard-4 outside cluster", + "arms": { + "warm": { + "start_nodes": 4, + "start_ingress": 6, + "ingress_hpa_min_during_burst": 6, + "release_hpa_min_after_burst": 2 + }, + "autoscale": { + "start_nodes": 2, + "start_ingress": 2, + "ingress_hpa_min": 2, + "ingress_hpa_max": 6 + } + }, + "phases": [ + { + "name": "preparation", + "seconds": 300, + "reads_per_second": 100, + "updated_per_second": 1, + "search_per_second": 1, + "workers": 128, + "publishes_per_second": 0 + }, + { + "name": "burst", + "seconds": 600, + "reads_per_second": 4500, + "updated_per_second": 36, + "search_per_second": 3, + "workers": 1024, + "publishes_per_second": 1 + }, + { + "name": "recovery", + "seconds": 1800, + "reads_per_second": 100, + "updated_per_second": 1, + "search_per_second": 1, + "workers": 128, + "publishes_per_second": 1 + } + ], + "return_to_idle_timeout_after_recovery_seconds": 1500, + "acceptance": "No dropped scheduled requests, controlled end-to-end latency, no errors, complete recovery, natural return to two Ready nodes; a diagnostic A/B alone is not a proven production fix.", + "caveats": [ + "Cache readiness and backend connection distribution are part of warm-versus-scale-up cost.", + "Updated_since is generated relative to wall clock in both arms.", + "One pair is diagnostic; a successful candidate requires repetition.", + "250m held constant because the 1-CPU candidate did not fit the two-node idle requirement." + ], + "driver_source_commit": "4eb0c10810904e47696ea853b09b1348e68f38b5", + "validation": { + "race_tests": "passed", + "lint": "passed", + "github_ci": "https://github.com/modelcontextprotocol/registry/actions/runs/40657658165" + } + }, + "runs": { + "warm": { + "phases": { + "preparation": { + "StartedAt": "2026-10-28T08:33:17.58099227Z", + "FinishedAt": "2026-10-28T08:38:17.576566723Z", + "CatalogSHA256": "120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be", + "ReadRate": 100, + "UpdatedRate": 1, + "SearchRate": 1, + "HotPercent": 80, + "Scheduled": 30000, + "ScheduledPublishes": 0, + "Completed": 30000, + "Dropped": 0, + "DroppedPublishes": 0, + "Status": { + "page:200": 29400, + "search:200": 300, + "updated:200": 300 + }, + "Cache": { + "HIT": 20003, + "MISS": 6417, + "STALE": 3580 + }, + "Errors": {}, + "LatencyMS": { + "page_p50": 2.854372, + "page_p95": 10.496049, + "page_p99": 15.625065, + "search_p50": 12.482938, + "search_p95": 37.62502, + "search_p99": 45.216926, + "updated_p50": 11.099117, + "updated_p95": 15.703356, + "updated_p99": 16.910074 + }, + "ElapsedSeconds": 299.995574337, + "Bytes": 1727061035, + "window_totals": { + "Reads": { + "Scheduled": 30000, + "Dropped": 0, + "Dispatched": 30000, + "Completed": 30000, + "Failures": 0, + "SchedulerLag": { + "count": 30000, + "mean_ms": 0.6086223268000002, + "maximum_ms": 2.061573, + "worst_window_p95_upper_ms": 2 + }, + "QueueDelay": { + "count": 30000, + "mean_ms": 0.009017354166666668, + "maximum_ms": 0.343305, + "worst_window_p95_upper_ms": 0.1 + }, + "HTTPDuration": { + "count": 30000, + "mean_ms": 3.6608936140333332, + "maximum_ms": 141.609588, + "worst_window_p95_upper_ms": 50 + } + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "count": 0, + "mean_ms": null, + "maximum_ms": 0, + "worst_window_p95_upper_ms": 0 + }, + "QueueDelay": { + "count": 0, + "mean_ms": null, + "maximum_ms": 0, + "worst_window_p95_upper_ms": 0 + }, + "HTTPDuration": { + "count": 0, + "mean_ms": null, + "maximum_ms": 0, + "worst_window_p95_upper_ms": 0 + } + } + } + }, + "burst": { + "StartedAt": "2026-10-28T08:38:24.538925958Z", + "FinishedAt": "2026-10-28T08:48:27.012485189Z", + "CatalogSHA256": "120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be", + "ReadRate": 4500, + "UpdatedRate": 36, + "SearchRate": 3, + "HotPercent": 80, + "Scheduled": 2700000, + "ScheduledPublishes": 600, + "Completed": 2338163, + "Dropped": 362437, + "DroppedPublishes": 0, + "Status": { + "page:0": 128, + "page:200": 2317174, + "publish:200": 600, + "search:0": 5, + "search:200": 1556, + "updated:0": 90, + "updated:200": 18610 + }, + "Cache": { + "": 819, + "HIT": 1944887, + "MISS": 38933, + "STALE": 204859, + "UPDATING": 148665 + }, + "Errors": { + "page:body:timeout": 1, + "page:request:timeout": 127, + "search:request:timeout": 5, + "updated:body:timeout": 3, + "updated:request:timeout": 87 + }, + "LatencyMS": { + "page_p50": 242.698843, + "page_p95": 649.870322, + "page_p99": 5565.102796, + "publish_p50": 35.364466, + "publish_p95": 6295.52226, + "publish_p99": 6833.156456, + "search_p50": 287.888307, + "search_p95": 5724.351489, + "search_p99": 6283.778105, + "updated_p50": 4282.275566, + "updated_p95": 5971.486638, + "updated_p99": 9515.563482 + }, + "ElapsedSeconds": 602.473559104, + "Bytes": 134711698432, + "window_totals": { + "Reads": { + "Scheduled": 2700000, + "Dropped": 362437, + "Dispatched": 2337563, + "Completed": 2337563, + "Failures": 223, + "SchedulerLag": { + "count": 2700000, + "mean_ms": 0.3855167980270365, + "maximum_ms": 336.55747, + "worst_window_p95_upper_ms": 200 + }, + "QueueDelay": { + "count": 2337563, + "mean_ms": 223.98283879131463, + "maximum_ms": 1414.987602, + "worst_window_p95_upper_ms": 2000 + }, + "HTTPDuration": { + "count": 2337563, + "mean_ms": 261.15499952233506, + "maximum_ms": 10004.182083, + "worst_window_p95_upper_ms": 5000 + } + }, + "Publishes": { + "Scheduled": 600, + "Dropped": 0, + "Dispatched": 600, + "Completed": 600, + "Failures": 0, + "SchedulerLag": { + "count": 600, + "mean_ms": 0.16387878166666667, + "maximum_ms": 20.948269, + "worst_window_p95_upper_ms": 50 + }, + "QueueDelay": { + "count": 600, + "mean_ms": 0.10357080666666667, + "maximum_ms": 1.639371, + "worst_window_p95_upper_ms": 2 + }, + "HTTPDuration": { + "count": 600, + "mean_ms": 2021.607056951667, + "maximum_ms": 8348.133061, + "worst_window_p95_upper_ms": 10000 + } + } + } + }, + "recovery": { + "StartedAt": "2026-10-28T08:48:31.781882317Z", + "FinishedAt": "2026-10-28T09:18:31.781803019Z", + "CatalogSHA256": "120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be", + "ReadRate": 100, + "UpdatedRate": 1, + "SearchRate": 1, + "HotPercent": 80, + "Scheduled": 180000, + "ScheduledPublishes": 1800, + "Completed": 181800, + "Dropped": 0, + "DroppedPublishes": 0, + "Status": { + "page:200": 176400, + "publish:200": 1800, + "search:200": 1800, + "updated:200": 1800 + }, + "Cache": { + "": 1800, + "HIT": 125939, + "MISS": 19004, + "STALE": 35057 + }, + "Errors": {}, + "LatencyMS": { + "page_p50": 2.799063, + "page_p95": 8.060709, + "page_p99": 13.28227, + "publish_p50": 9.70784, + "publish_p95": 12.978189, + "publish_p99": 15.947302, + "search_p50": 10.425445, + "search_p95": 36.405784, + "search_p99": 41.28453, + "updated_p50": 11.123907, + "updated_p95": 16.047554, + "updated_p99": 21.234086 + }, + "ElapsedSeconds": 1799.999920565, + "Bytes": 10387869614, + "window_totals": { + "Reads": { + "Scheduled": 180000, + "Dropped": 0, + "Dispatched": 180000, + "Completed": 180000, + "Failures": 0, + "SchedulerLag": { + "count": 180000, + "mean_ms": 0.618953111461111, + "maximum_ms": 3.357755, + "worst_window_p95_upper_ms": 2 + }, + "QueueDelay": { + "count": 180000, + "mean_ms": 0.008993479355555552, + "maximum_ms": 0.217032, + "worst_window_p95_upper_ms": 0.1 + }, + "HTTPDuration": { + "count": 180000, + "mean_ms": 3.027214473116667, + "maximum_ms": 378.02543, + "worst_window_p95_upper_ms": 20 + } + }, + "Publishes": { + "Scheduled": 1800, + "Dropped": 0, + "Dispatched": 1800, + "Completed": 1800, + "Failures": 0, + "SchedulerLag": { + "count": 1800, + "mean_ms": 0.5140783033333333, + "maximum_ms": 1.461131, + "worst_window_p95_upper_ms": 2 + }, + "QueueDelay": { + "count": 1800, + "mean_ms": 0.1105722494444445, + "maximum_ms": 0.530764, + "worst_window_p95_upper_ms": 1 + }, + "HTTPDuration": { + "count": 1800, + "mean_ms": 9.354878723888893, + "maximum_ms": 76.663278, + "worst_window_p95_upper_ms": 100 + } + } + } + } + }, + "generator": { + "sample_intervals": 60, + "scope": "Whole four-vCPU generator from /proc/stat and non-loopback network devices; includes monitoring overhead. CPU busy includes steal, reported separately. Does not alone rule out single-thread or network constraints.", + "cpu_busy_percent": { + "mean": 51.34566387688601, + "max": 59.38786008230453 + }, + "cpu_steal_percent": { + "mean": 0.13422269051651112, + "max": 0.3653444676409186 + }, + "cpu_iowait_percent": { + "mean": 0.0034787615935366217, + "max": 0.026567481402763018 + }, + "cpu_softirq_percent": { + "mean": 15.633350912263069, + "max": 17.690732206861238 + }, + "received_gbps": { + "mean": 1.8444311523971006, + "max": 2.1339308808883826 + }, + "transmitted_gbps": { + "mean": 0.021702159757767483, + "max": 0.02599625817023602 + }, + "receive_drops": 0, + "transmit_drops": 0 + }, + "app_pressure": [ + { + "metric": "container_cpu_usage_seconds_total", + "seconds": 587.9370000362396, + "rate": 0.9535294886449475, + "start": 1793176716.769, + "end": 1793177304.706, + "placement": "other_node" + }, + { + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 587.9370000362396, + "rate": 0.13083882115814865, + "start": 1793176716.769, + "end": 1793177304.706, + "placement": "other_node" + }, + { + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 587.9370000362396, + "rate": 0.19931247904584504, + "start": 1793176716.769, + "end": 1793177304.706, + "placement": "other_node" + }, + { + "metric": "container_cpu_usage_seconds_total", + "seconds": 569.0750000476837, + "rate": 0.9428957588279914, + "start": 1793176710.121, + "end": 1793177279.196, + "placement": "with_registry_database" + }, + { + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 569.0750000476837, + "rate": 0.45991289720699324, + "start": 1793176710.121, + "end": 1793177279.196, + "placement": "with_registry_database" + }, + { + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 569.0750000476837, + "rate": 0.5699427438787911, + "start": 1793176710.121, + "end": 1793177279.196, + "placement": "with_registry_database" + } + ], + "integrity": { + "samples": 99, + "commands_failed": [], + "db_throttled_usec_range": [ + 260147, + 475465 + ], + "db_endpoint_samples": 541, + "db_endpoint_errors_or_empty": [], + "db_endpoint_not_ready": [], + "cadvisor_failures_or_empty": [], + "lb_collection_failures": [], + "metric_window": { + "start": 1793176391.513865, + "end": 1793179341.3185751 + }, + "metric_errors": [], + "empty_metrics": [ + "alerts", + "raw_5xx" + ], + "node_count_timeline": [ + { + "at": 1793176391.553984, + "nodes": 4 + }, + { + "at": 1793179032.190119, + "nodes": 3 + }, + { + "at": 1793179332.2599921, + "nodes": 2 + } + ], + "max_app_or_db_restart_count": 0, + "final_nodes": 2 + }, + "peak_sampled_pool_waiters": 927.0, + "client_result_verification": { + "at": 1793179176.142282, + "phases": { + "preparation": { + "identical_json_to_remote_file": true, + "remote_file_sha256": "8a2a1f4f89dddbf0430297dd20e07582b3af01f5fdf30012f4bdc645b9ac2fe0", + "parsed_stdout_file_sha256": "8a2a1f4f89dddbf0430297dd20e07582b3af01f5fdf30012f4bdc645b9ac2fe0" + }, + "burst": { + "identical_json_to_remote_file": true, + "remote_file_sha256": "45111014a9c43d03a967c0e183fd7e4e00066883fca3f183fcac150fdc80147f", + "parsed_stdout_file_sha256": "45111014a9c43d03a967c0e183fd7e4e00066883fca3f183fcac150fdc80147f" + }, + "recovery": { + "identical_json_to_remote_file": true, + "remote_file_sha256": "484884f944468f879d2113f70412738cab2ca47b8b2dceb97e4d0560f13a6d71", + "parsed_stdout_file_sha256": "484884f944468f879d2113f70412738cab2ca47b8b2dceb97e4d0560f13a6d71" + } + } + }, + "database_statement_deltas": { + "before": "diagnostics-1793176386.json", + "after": "diagnostics-1793177394.json", + "window_seconds": 1002.8315010070801, + "reset_unchanged": true, + "scope": "Only statements present in both bounded top-25 snapshots; interval brackets burst with collection padding", + "rows": [ + { + "queryid": 3749554622472550736, + "calls": 234981, + "mean_ms": 3.427575401592019, + "total_exec_ms": 805415.0954414941, + "block_reads": 0, + "temp_blocks_written": 0 + }, + { + "queryid": -4303255870242342024, + "calls": 19069, + "mean_ms": 7.0532287655355175, + "total_exec_ms": 134498.01932999678, + "block_reads": 0, + "temp_blocks_written": 0 + }, + { + "queryid": 903517041871178438, + "calls": 1927, + "mean_ms": 27.081587734305966, + "total_exec_ms": 52186.219564007595, + "block_reads": 0, + "temp_blocks_written": 0 + }, + { + "queryid": 4318848960711986315, + "calls": 670, + "mean_ms": 0.9424345223880013, + "total_exec_ms": 631.4311299999608, + "block_reads": 0, + "temp_blocks_written": 0 + }, + { + "queryid": -5383083741758676298, + "calls": 319, + "mean_ms": 1.6521755548589416, + "total_exec_ms": 527.0440020000024, + "block_reads": 0, + "temp_blocks_written": 0 + }, + { + "queryid": 2079831266944838202, + "calls": 670, + "mean_ms": 0.4706912149255574, + "total_exec_ms": 315.36311400012346, + "block_reads": 0, + "temp_blocks_written": 0 + } + ] + }, + "fixture_cleanup": { + "at": 1793179361.27714, + "ids": [ + "1793176704538925958", + "1793177311781882317" + ], + "code": 0, + "stdout": "BEGIN\nSET\nSET\n2400\nCOMMIT\n", + "stderr": "" + }, + "natural_two_node_idle_at": 1793179338.236856 + }, + "autoscale": { + "phases": { + "preparation": { + "StartedAt": "2026-10-28T09:23:29.842553437Z", + "FinishedAt": "2026-10-28T09:28:29.835821505Z", + "CatalogSHA256": "120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be", + "ReadRate": 100, + "UpdatedRate": 1, + "SearchRate": 1, + "HotPercent": 80, + "Scheduled": 30000, + "ScheduledPublishes": 0, + "Completed": 30000, + "Dropped": 0, + "DroppedPublishes": 0, + "Status": { + "page:200": 29400, + "search:200": 300, + "updated:200": 300 + }, + "Cache": { + "HIT": 21637, + "MISS": 5182, + "STALE": 3181 + }, + "Errors": {}, + "LatencyMS": { + "page_p50": 2.816732, + "page_p95": 9.333098, + "page_p99": 14.922277, + "search_p50": 12.234767, + "search_p95": 36.155952, + "search_p99": 59.734291, + "updated_p50": 10.253646, + "updated_p95": 15.78252, + "updated_p99": 17.29969 + }, + "ElapsedSeconds": 299.993267927, + "Bytes": 1727124918, + "window_totals": { + "Reads": { + "Scheduled": 30000, + "Dropped": 0, + "Dispatched": 30000, + "Completed": 30000, + "Failures": 0, + "SchedulerLag": { + "count": 30000, + "mean_ms": 0.6063905583999999, + "maximum_ms": 3.764489, + "worst_window_p95_upper_ms": 5 + }, + "QueueDelay": { + "count": 30000, + "mean_ms": 0.009647454033333334, + "maximum_ms": 0.178441, + "worst_window_p95_upper_ms": 0.5 + }, + "HTTPDuration": { + "count": 30000, + "mean_ms": 3.380232566833334, + "maximum_ms": 185.536875, + "worst_window_p95_upper_ms": 100 + } + }, + "Publishes": { + "Scheduled": 0, + "Dropped": 0, + "Dispatched": 0, + "Completed": 0, + "Failures": 0, + "SchedulerLag": { + "count": 0, + "mean_ms": null, + "maximum_ms": 0, + "worst_window_p95_upper_ms": 0 + }, + "QueueDelay": { + "count": 0, + "mean_ms": null, + "maximum_ms": 0, + "worst_window_p95_upper_ms": 0 + }, + "HTTPDuration": { + "count": 0, + "mean_ms": null, + "maximum_ms": 0, + "worst_window_p95_upper_ms": 0 + } + } + } + }, + "burst": { + "StartedAt": "2026-10-28T09:28:36.421225785Z", + "FinishedAt": "2026-10-28T09:38:38.824641006Z", + "CatalogSHA256": "120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be", + "ReadRate": 4500, + "UpdatedRate": 36, + "SearchRate": 3, + "HotPercent": 80, + "Scheduled": 2700000, + "ScheduledPublishes": 600, + "Completed": 2202897, + "Dropped": 497703, + "DroppedPublishes": 0, + "Status": { + "page:0": 487, + "page:200": 2182733, + "publish:0": 9, + "publish:200": 591, + "search:0": 8, + "search:200": 1461, + "updated:0": 144, + "updated:200": 17464 + }, + "Cache": { + "": 912, + "HIT": 1837889, + "MISS": 39132, + "STALE": 173643, + "UPDATING": 151321 + }, + "Errors": { + "page:body:connection_reset": 276, + "page:body:timeout": 15, + "page:request:timeout": 196, + "publish:request:connection_reset": 1, + "publish:request:timeout": 8, + "search:body:connection_reset": 1, + "search:request:timeout": 7, + "updated:body:connection_reset": 27, + "updated:body:timeout": 8, + "updated:request:timeout": 109 + }, + "LatencyMS": { + "page_p50": 274.999517, + "page_p95": 660.385472, + "page_p99": 6101.628831, + "publish_p50": 105.753377, + "publish_p95": 7980.214239, + "publish_p99": 10000.385639, + "search_p50": 378.759866, + "search_p95": 7143.601636, + "search_p99": 9022.712035, + "updated_p50": 4024.103398, + "updated_p95": 7726.966087, + "updated_p99": 9746.651614 + }, + "ElapsedSeconds": 602.403415063, + "Bytes": 126860655361, + "window_totals": { + "Reads": { + "Scheduled": 2700000, + "Dropped": 497703, + "Dispatched": 2202297, + "Completed": 2202297, + "Failures": 639, + "SchedulerLag": { + "count": 2700000, + "mean_ms": 0.15650167892407416, + "maximum_ms": 101.855828, + "worst_window_p95_upper_ms": 50 + }, + "QueueDelay": { + "count": 2202297, + "mean_ms": 243.42211870219003, + "maximum_ms": 1229.548229, + "worst_window_p95_upper_ms": 2000 + }, + "HTTPDuration": { + "count": 2202297, + "mean_ms": 278.28733524667865, + "maximum_ms": 10004.880439, + "worst_window_p95_upper_ms": 10000 + } + }, + "Publishes": { + "Scheduled": 600, + "Dropped": 0, + "Dispatched": 600, + "Completed": 600, + "Failures": 9, + "SchedulerLag": { + "count": 600, + "mean_ms": 0.1451826733333333, + "maximum_ms": 4.467296, + "worst_window_p95_upper_ms": 5 + }, + "QueueDelay": { + "count": 600, + "mean_ms": 0.10669146833333333, + "maximum_ms": 0.614831, + "worst_window_p95_upper_ms": 1 + }, + "HTTPDuration": { + "count": 600, + "mean_ms": 2527.673717551668, + "maximum_ms": 10006.079299, + "worst_window_p95_upper_ms": 20000 + } + } + } + }, + "recovery": { + "StartedAt": "2026-10-28T09:38:42.91359417Z", + "FinishedAt": "2026-10-28T10:08:42.911696534Z", + "CatalogSHA256": "120f34ec070fe08acf093e65f13a28631d944cf510915c65233b9eab62c143be", + "ReadRate": 100, + "UpdatedRate": 1, + "SearchRate": 1, + "HotPercent": 80, + "Scheduled": 180000, + "ScheduledPublishes": 1800, + "Completed": 181800, + "Dropped": 0, + "DroppedPublishes": 0, + "Status": { + "page:0": 1, + "page:200": 176399, + "publish:200": 1800, + "search:200": 1800, + "updated:200": 1800 + }, + "Cache": { + "": 1801, + "HIT": 125548, + "MISS": 19351, + "STALE": 35100 + }, + "Errors": { + "page:request:connection_refused": 1 + }, + "LatencyMS": { + "page_p50": 2.779725, + "page_p95": 8.141626, + "page_p99": 13.302961, + "publish_p50": 10.640188, + "publish_p95": 13.951831, + "publish_p99": 16.505905, + "search_p50": 11.450831, + "search_p95": 38.316007, + "search_p99": 43.097837, + "updated_p50": 10.535312, + "updated_p95": 15.449092, + "updated_p99": 22.96925 + }, + "ElapsedSeconds": 1799.99810222, + "Bytes": 10381362354, + "window_totals": { + "Reads": { + "Scheduled": 180000, + "Dropped": 0, + "Dispatched": 180000, + "Completed": 180000, + "Failures": 1, + "SchedulerLag": { + "count": 180000, + "mean_ms": 0.6158125930999999, + "maximum_ms": 9.766306, + "worst_window_p95_upper_ms": 2 + }, + "QueueDelay": { + "count": 180000, + "mean_ms": 0.008858152249999997, + "maximum_ms": 0.249828, + "worst_window_p95_upper_ms": 0.1 + }, + "HTTPDuration": { + "count": 180000, + "mean_ms": 3.0210155262000007, + "maximum_ms": 410.444183, + "worst_window_p95_upper_ms": 20 + } + }, + "Publishes": { + "Scheduled": 1800, + "Dropped": 0, + "Dispatched": 1800, + "Completed": 1800, + "Failures": 0, + "SchedulerLag": { + "count": 1800, + "mean_ms": 0.5303296811111111, + "maximum_ms": 1.455758, + "worst_window_p95_upper_ms": 2 + }, + "QueueDelay": { + "count": 1800, + "mean_ms": 0.1061064388888889, + "maximum_ms": 0.585206, + "worst_window_p95_upper_ms": 1 + }, + "HTTPDuration": { + "count": 1800, + "mean_ms": 10.275798986111107, + "maximum_ms": 94.645054, + "worst_window_p95_upper_ms": 100 + } + } + } + } + }, + "generator": { + "sample_intervals": 60, + "scope": "Whole four-vCPU generator from /proc/stat and non-loopback network devices; includes monitoring overhead. CPU busy includes steal, reported separately. Does not alone rule out single-thread or network constraints.", + "cpu_busy_percent": { + "mean": 51.78341979521962, + "max": 66.04809200209095 + }, + "cpu_steal_percent": { + "mean": 0.0810159241998051, + "max": 0.20860495436766624 + }, + "cpu_iowait_percent": { + "mean": 0.006068008883176232, + "max": 0.07847240387130526 + }, + "cpu_softirq_percent": { + "mean": 14.441653875992783, + "max": 16.95786228160329 + }, + "received_gbps": { + "mean": 1.7375342139601637, + "max": 2.167692710190029 + }, + "transmitted_gbps": { + "mean": 0.020953099346152255, + "max": 0.025794656449407705 + }, + "receive_drops": 0, + "transmit_drops": 0 + }, + "app_pressure": [ + { + "metric": "container_cpu_usage_seconds_total", + "seconds": 562.9060001373291, + "rate": 0.8565447816906756, + "start": 1793179724.264, + "end": 1793180287.17, + "placement": "other_node" + }, + { + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 562.9060001373291, + "rate": 0.1864226566680738, + "start": 1793179724.264, + "end": 1793180287.17, + "placement": "other_node" + }, + { + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 562.9060001373291, + "rate": 0.26435907765007977, + "start": 1793179724.264, + "end": 1793180287.17, + "placement": "other_node" + }, + { + "metric": "container_cpu_usage_seconds_total", + "seconds": 561.9000000953674, + "rate": 0.8413106814731562, + "start": 1793179730.161, + "end": 1793180292.061, + "placement": "with_registry_database" + }, + { + "metric": "container_pressure_cpu_stalled_seconds_total", + "seconds": 561.9000000953674, + "rate": 0.4503017101923044, + "start": 1793179730.161, + "end": 1793180292.061, + "placement": "with_registry_database" + }, + { + "metric": "container_pressure_cpu_waiting_seconds_total", + "seconds": 561.9000000953674, + "rate": 0.5650659102084199, + "start": 1793179730.161, + "end": 1793180292.061, + "placement": "with_registry_database" + } + ], + "integrity": { + "samples": 96, + "commands_failed": [], + "db_throttled_usec_range": [ + 475465, + 815084 + ], + "db_endpoint_samples": 524, + "db_endpoint_errors_or_empty": [], + "db_endpoint_not_ready": [], + "cadvisor_failures_or_empty": [], + "lb_collection_failures": [ + { + "at": 1793182235.030524, + "command": "health", + "detail": { + "code": 1, + "error": "ERROR: (gcloud.compute.target-pools.get-health) Could not get health for some targets:\n - The resource 'projects/mcp-registry-staging/zones/us-central1-b/instances/gke-mcp-registry-sta-mcp-registry-sta-5d130b39-tbww' was not found\n\n" + } + } + ], + "metric_window": { + "start": 1793179403.256125, + "end": 1793182257.731721 + }, + "metric_errors": [], + "empty_metrics": [ + "alerts", + "raw_5xx" + ], + "node_count_timeline": [ + { + "at": 1793179403.2972429, + "nodes": 2 + }, + { + "at": 1793179793.4029498, + "nodes": 3 + }, + { + "at": 1793179883.4181492, + "nodes": 4 + }, + { + "at": 1793181863.921328, + "nodes": 3 + }, + { + "at": 1793182254.023026, + "nodes": 2 + } + ], + "max_app_or_db_restart_count": 0, + "final_nodes": 2 + }, + "peak_sampled_pool_waiters": 895.0, + "client_result_verification": { + "at": 1793182468.525573, + "phases": { + "preparation": { + "identical_json_to_remote_file": true, + "remote_file_sha256": "5c901fb6d3c297819b3db904482a4f3633ae751b3928fdb356ea4c6906f88d39", + "parsed_stdout_file_sha256": "5c901fb6d3c297819b3db904482a4f3633ae751b3928fdb356ea4c6906f88d39" + }, + "burst": { + "identical_json_to_remote_file": true, + "remote_file_sha256": "6868dc2db8c53078354ca76b0b3a56d7212ca36bd0080b63e4c406fad8925cef", + "parsed_stdout_file_sha256": "b8abe2ac5ddadf1d3dff88357493a6b1fc3a90ff93644a81888f98f5cad7feac" + }, + "recovery": { + "identical_json_to_remote_file": true, + "remote_file_sha256": "73c2481468cd8132896cda59f1a4fdaaedc1aad3665ab37267bb0ffbe45ade3f", + "parsed_stdout_file_sha256": "73c2481468cd8132896cda59f1a4fdaaedc1aad3665ab37267bb0ffbe45ade3f" + } + } + }, + "database_statement_deltas": { + "before": "diagnostics-1793179398.json", + "after": "diagnostics-1793180400.json", + "window_seconds": 993.2200920581818, + "reset_unchanged": true, + "scope": "Only statements present in both bounded top-25 snapshots; interval brackets burst with collection padding", + "rows": [ + { + "queryid": 3749554622472550736, + "calls": 203184, + "mean_ms": 3.358564558654681, + "total_exec_ms": 682406.5812856928, + "block_reads": 0, + "temp_blocks_written": 0 + }, + { + "queryid": -4303255870242342024, + "calls": 17970, + "mean_ms": 6.770110423982757, + "total_exec_ms": 121658.88431897014, + "block_reads": 0, + "temp_blocks_written": 0 + }, + { + "queryid": 903517041871178438, + "calls": 1825, + "mean_ms": 28.093580003821685, + "total_exec_ms": 51270.78350697458, + "block_reads": 0, + "temp_blocks_written": 0 + }, + { + "queryid": 2079831266944838202, + "calls": 659, + "mean_ms": 3.270229933232272, + "total_exec_ms": 2155.0815260000672, + "block_reads": 0, + "temp_blocks_written": 0 + }, + { + "queryid": 4318848960711986315, + "calls": 658, + "mean_ms": 0.8853243708206686, + "total_exec_ms": 582.5434359999999, + "block_reads": 0, + "temp_blocks_written": 0 + }, + { + "queryid": -5383083741758676298, + "calls": 271, + "mean_ms": 1.6845018376383203, + "total_exec_ms": 456.49999799998477, + "block_reads": 0, + "temp_blocks_written": 0 + } + ] + }, + "fixture_cleanup": { + "at": 1793182485.414944, + "ids": [ + "1793179716421225785", + "1793180322913594170" + ], + "code": 0, + "stdout": "BEGIN\nSET\nSET\n2395\nCOMMIT\n", + "stderr": "" + }, + "natural_two_node_idle_at": 1793182254.6012611, + "recovery_refusal": { + "failure": { + "ScheduledAt": "2026-10-28T09:52:59.75359417Z", + "FinishedAt": "2026-10-28T09:52:59.756652132Z", + "Kind": "page", + "Class": "request:connection_refused", + "Status": 0 + }, + "nearby_lb_samples": [ + { + "at": "2026-10-28T09:52:04.357756+00:00", + "health": [ + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-wn8j", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-tbww", + "HEALTHY" + ] + ], + "endpoints": [ + [ + "5x9k", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "kg25", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "wn8j", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "tbww", + { + "ready": true, + "serving": true, + "terminating": false + } + ] + ] + }, + { + "at": "2026-10-28T09:52:14.361040+00:00", + "health": [ + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-wn8j", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-tbww", + "HEALTHY" + ] + ], + "endpoints": [ + [ + "5x9k", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "kg25", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "wn8j", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "tbww", + { + "ready": true, + "serving": true, + "terminating": false + } + ] + ] + }, + { + "at": "2026-10-28T09:52:24.365999+00:00", + "health": [ + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-wn8j", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-tbww", + "HEALTHY" + ] + ], + "endpoints": [ + [ + "5x9k", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "kg25", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "wn8j", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "tbww", + { + "ready": true, + "serving": true, + "terminating": false + } + ] + ] + }, + { + "at": "2026-10-28T09:52:34.370444+00:00", + "health": [ + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-wn8j", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-tbww", + "HEALTHY" + ] + ], + "endpoints": [ + [ + "5x9k", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "kg25", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "wn8j", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "tbww", + { + "ready": true, + "serving": true, + "terminating": false + } + ] + ] + }, + { + "at": "2026-10-28T09:52:44.377561+00:00", + "health": [ + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-wn8j", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-tbww", + "HEALTHY" + ] + ], + "endpoints": [ + [ + "5x9k", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "kg25", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "wn8j", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "tbww", + { + "ready": true, + "serving": true, + "terminating": false + } + ] + ] + }, + { + "at": "2026-10-28T09:52:54.385615+00:00", + "health": [ + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-wn8j", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-tbww", + "HEALTHY" + ] + ], + "endpoints": [ + [ + "5x9k", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "kg25", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "wn8j", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "tbww", + { + "ready": true, + "serving": true, + "terminating": false + } + ] + ] + }, + { + "at": "2026-10-28T09:53:04.385983+00:00", + "health": [ + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-wn8j", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-tbww", + "HEALTHY" + ] + ], + "endpoints": [ + [ + "5x9k", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "kg25", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "wn8j", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "tbww", + { + "ready": false, + "serving": true, + "terminating": true + } + ] + ] + }, + { + "at": "2026-10-28T09:53:14.392999+00:00", + "health": [ + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-wn8j", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-tbww", + "HEALTHY" + ] + ], + "endpoints": [ + [ + "5x9k", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "kg25", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "wn8j", + { + "ready": true, + "serving": true, + "terminating": false + } + ] + ] + }, + { + "at": "2026-10-28T09:53:24.394054+00:00", + "health": [ + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-wn8j", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-tbww", + "UNHEALTHY" + ] + ], + "endpoints": [ + [ + "5x9k", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "kg25", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "wn8j", + { + "ready": true, + "serving": true, + "terminating": false + } + ] + ] + }, + { + "at": "2026-10-28T09:53:34.401805+00:00", + "health": [ + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-wn8j", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-tbww", + "UNHEALTHY" + ] + ], + "endpoints": [ + [ + "5x9k", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "kg25", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "wn8j", + { + "ready": true, + "serving": true, + "terminating": false + } + ] + ] + }, + { + "at": "2026-10-28T09:53:44.410284+00:00", + "health": [ + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-wn8j", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-tbww", + "UNHEALTHY" + ] + ], + "endpoints": [ + [ + "5x9k", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "kg25", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "wn8j", + { + "ready": true, + "serving": true, + "terminating": false + } + ] + ] + }, + { + "at": "2026-10-28T09:53:54.418633+00:00", + "health": [ + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-5x9k", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-kg25", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-wn8j", + "HEALTHY" + ], + [ + "gke-mcp-registry-sta-mcp-registry-sta-5d130b39-tbww", + "UNHEALTHY" + ] + ], + "endpoints": [ + [ + "5x9k", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "kg25", + { + "ready": true, + "serving": true, + "terminating": false + } + ], + [ + "wn8j", + { + "ready": true, + "serving": true, + "terminating": false + } + ] + ] + } + ], + "interpretation": "Refusal occurred during ingress downscaling; endpoint on tbww changes to terminating near the failure while LB health still reports healthy. No packet capture identifies the rejecting backend; causation is not established." + } + } + }, + "verdict": "Both burst gates failed; neither arm establishes a production fix.", + "measurement_limitations": [ + "One pair, no passing repeat.", + "Measured v4 driver retained complete failure totals but only 100 individual failure samples; later reset timestamps were omitted.", + "A subsequent driver commit adds complete per-window error-class counts for future runs; it was not used in either measured arm.", + "PostgreSQL statement deltas bracket preparation, burst and early recovery; they are not precise per-burst SQL statistics.", + "Origin gauges are sampled every 60 seconds; brief events may be missed." + ], + "cleanup": { + "at": 1793182484.203678, + "production_changed": false, + "hpa_spec_restored": true, + "restored_app": { + "image": "ghcr.io/modelcontextprotocol/registry:main", + "resources": { + "limits": { + "memory": "1Gi" + }, + "requests": { + "cpu": "250m", + "memory": "512Mi" + } + } + }, + "restored_whitelist": null, + "generator_instances_absent": true, + "generator_disks_absent": true, + "final_verification": { + "at": 1793182542.6208231, + "ready_nodes": 2, + "corpus": { + "at": "2026-10-28T10:15:43.761027+00:00", + "names": 52366, + "versions": 187108, + "benchmark_runs": null, + "database_started": "2026-10-27T21:24:59.430872+00:00" + }, + "matching_response_hashes": 48 + } + } +} diff --git a/tools/ingressbench/test_profile_logs.py b/tools/ingressbench/test_profile_logs.py new file mode 100644 index 000000000..34aa9e467 --- /dev/null +++ b/tools/ingressbench/test_profile_logs.py @@ -0,0 +1,73 @@ +import json +import unittest +from profile_logs import Profile, request_shape, parse_timestamp +from collect_profile_logs import collect + + +def record(uri='/v0/servers?limit=100&cursor=secret', request_bytes=200, protocol='HTTP/2.0'): + return f'192.0.2.1 - - [28/Oct/2026:18:30:00 +0000] "GET {uri} {protocol}" 200 2048 "private-referrer" "private-agent" {request_bytes} 0.100 [upstream] [] - - - - private-request-id cache_status="HIT"' + + +class ProfileTests(unittest.TestCase): + def test_staging_collection_uses_only_selected_project(self): + queries = [] + def fetch(body): + queries.append(body.copy()) + return {'entries': []} + result = collect('2026-10-28T18:30:00Z', 60, 'private-token', fetch, + project='mcp-registry-staging') + self.assertTrue(result['query_complete']) + self.assertEqual(queries[0]['resourceNames'], ['projects/mcp-registry-staging']) + with self.assertRaises(ValueError): + collect('2026-10-28T18:30:00Z', 60, 'private-token', fetch, project='another-project') + + def test_cloud_nanosecond_timestamp(self): + self.assertEqual(parse_timestamp('2026-10-28T18:30:00.123456789Z').microsecond, 123456) + with self.assertRaises(ValueError): + parse_timestamp('2026-10-28T18:30:00') + + def test_background_is_not_client_demand_and_no_identifiers_escape(self): + p = Profile() + p.add(record(), 100, 'ingress-a') + p.add(record(request_bytes=0), 100, 'ingress-a') + p.add('unrecognized', 100, 'ingress-a') + d = p.result() + self.assertEqual(d['populations']['client']['count'], 1) + self.assertEqual(d['populations']['zero_request_length']['count'], 1) + self.assertEqual(d['client_completions_per_second'], {100: 1}) + self.assertEqual(d['unparsed'], 1) + for secret in ['192.0.2.1', 'secret', 'private-agent', 'private-referrer', 'private-request-id']: + self.assertNotIn(secret, json.dumps(d)) + + def test_keys_preserve_reuse_without_exposing_values(self): + p = Profile() + for uri in ['/v0/servers?cursor=a', '/v0/servers?cursor=a', '/v0/servers?cursor=b']: + p.add(record(uri), 100, 'ingress-a') + self.assertEqual(p.result()['page_keys']['distinct'], 2) + self.assertEqual(p.result()['page_keys']['singleton_keys'], 1) + + def test_bounded_shapes_and_search_precedence(self): + self.assertEqual(request_shape('/v0.1/servers?search=private&updated_since=private&limit=secret')[:3], + ('search', 'v0.1', 'invalid')) + p = Profile() + p.add(record('/v0.1/servers?search=private'), 100, 'ingress-a') + self.assertEqual(p.result()['search_length_counts'], {'7': 1}) + self.assertNotIn('private', json.dumps(p.result())) + + def test_pagination_and_caps_are_not_reported_as_complete_traffic(self): + row = {'textPayload': record(), 'timestamp': '2026-10-28T18:30:00.123456789Z'} + def fetch(body): + if body.get('pageToken'): + return {'entries': [row]} + return {'entries': [row], 'nextPageToken': 'next'} + complete = collect('2026-10-28T18:30:00Z', 60, 'private-token', fetch) + self.assertTrue(complete['query_complete']) + self.assertEqual(complete['entries'], 2) + capped = collect('2026-10-28T18:30:00Z', 60, 'private-token', fetch, max_records=1) + self.assertFalse(capped['query_complete']) + self.assertIsNone(capped['populations']['client']['records_per_second']) + self.assertNotIn('private-token', json.dumps(complete)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/ingressbench/timeline.go b/tools/ingressbench/timeline.go new file mode 100644 index 000000000..bf38db83f --- /dev/null +++ b/tools/ingressbench/timeline.go @@ -0,0 +1,172 @@ +package main + +import ( + "sort" + "strconv" + "time" +) + +const windowSeconds = 10 + +var durationBoundsMS = [...]float64{0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000, 20000, 60000, 120000} + +type durationSummary struct { + Count int + MeanMS, P95UpperMS, MaximumMS float64 +} + +type windowTraffic struct { + Scheduled, Dropped, Dispatched, Completed, Failures int + SchedulerLag, QueueDelay, HTTPDuration durationSummary + FailureClasses map[string]int +} + +type timeWindow struct { + StartedAt time.Time + Seconds int + Reads, Publishes windowTraffic +} + +type durationHistogram struct { + count int + sum, maximum float64 + buckets [len(durationBoundsMS) + 1]int +} + +func (h *durationHistogram) add(d time.Duration) { + ms := max(0, float64(d)/float64(time.Millisecond)) + h.count++ + h.sum += ms + h.maximum = max(h.maximum, ms) + h.buckets[sort.SearchFloat64s(durationBoundsMS[:], ms)]++ +} + +func (h *durationHistogram) merge(other durationHistogram) { + h.count += other.count + h.sum += other.sum + h.maximum = max(h.maximum, other.maximum) + for i, n := range other.buckets { + h.buckets[i] += n + } +} + +func (h durationHistogram) summary() durationSummary { + if h.count == 0 { + return durationSummary{} + } + out := durationSummary{Count: h.count, MeanMS: h.sum / float64(h.count), MaximumMS: h.maximum} + remaining := (95*h.count + 99) / 100 + for i, n := range h.buckets { + remaining -= n + if remaining <= 0 { + out.P95UpperMS = h.maximum + if i < len(durationBoundsMS) { + out.P95UpperMS = durationBoundsMS[i] + } + break + } + } + return out +} + +type trafficAccumulator struct { + scheduled, dropped, dispatched, completed, failures int + schedulerLag, queueDelay, httpDuration durationHistogram + failureClasses map[string]int +} + +func (a *trafficAccumulator) merge(b trafficAccumulator) { + a.scheduled += b.scheduled + a.dropped += b.dropped + a.dispatched += b.dispatched + a.completed += b.completed + a.failures += b.failures + a.schedulerLag.merge(b.schedulerLag) + a.queueDelay.merge(b.queueDelay) + a.httpDuration.merge(b.httpDuration) + for class, count := range b.failureClasses { + if a.failureClasses == nil { + a.failureClasses = make(map[string]int) + } + a.failureClasses[class] += count + } +} + +func (a trafficAccumulator) summary() windowTraffic { + return windowTraffic{a.scheduled, a.dropped, a.dispatched, a.completed, a.failures, + a.schedulerLag.summary(), a.queueDelay.summary(), a.httpDuration.summary(), a.failureClasses} +} + +// Each timeline has one writer. Read scheduling, publish scheduling and result +// collection merge only after all goroutines finish, avoiding per-request locks. +type timeline struct { + buckets map[int64]*[2]trafficAccumulator +} + +func newTimeline() *timeline { return &timeline{buckets: make(map[int64]*[2]trafficAccumulator)} } + +func (t *timeline) at(at time.Time, kind string) *trafficAccumulator { + key := at.Unix() / windowSeconds + if t.buckets[key] == nil { + t.buckets[key] = &[2]trafficAccumulator{} + } + index := 0 + if kind == "publish" { + index = 1 + } + return &t.buckets[key][index] +} + +func (t *timeline) scheduled(kind string, intended, observed time.Time, dropped bool) { + t.at(intended, kind).scheduled++ + a := t.at(observed, kind) + a.schedulerLag.add(observed.Sub(intended)) + if dropped { + a.dropped++ + } +} + +func (t *timeline) completed(p point) { + a := t.at(p.startedAt, p.kind) + a.dispatched++ + a.queueDelay.add(p.startedAt.Sub(p.queuedAt)) + a.httpDuration.add(p.finishedAt.Sub(p.startedAt)) + a = t.at(p.finishedAt, p.kind) + a.completed++ + if p.failure != "" || p.status == 0 || p.status >= 400 { + a.failures++ + class := p.failure + if class == "" { + class = "status:" + strconv.Itoa(p.status) + } + if a.failureClasses == nil { + a.failureClasses = make(map[string]int) + } + a.failureClasses[class]++ + } +} + +func (t *timeline) merge(other *timeline) { + for key, groups := range other.buckets { + if t.buckets[key] == nil { + t.buckets[key] = &[2]trafficAccumulator{} + } + for i, group := range groups { + t.buckets[key][i].merge(group) + } + } +} + +func (t *timeline) windows() []timeWindow { + keys := make([]int64, 0, len(t.buckets)) + for key := range t.buckets { + keys = append(keys, key) + } + sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) + out := make([]timeWindow, 0, len(keys)) + for _, key := range keys { + groups := t.buckets[key] + out = append(out, timeWindow{time.Unix(key*windowSeconds, 0).UTC(), windowSeconds, groups[0].summary(), groups[1].summary()}) + } + return out +} diff --git a/tools/ingressbench/timeline_test.go b/tools/ingressbench/timeline_test.go new file mode 100644 index 000000000..467f6066e --- /dev/null +++ b/tools/ingressbench/timeline_test.go @@ -0,0 +1,78 @@ +package main + +import ( + "testing" + "time" +) + +func TestTimelineSeparatesSchedulingQueueAndHTTPAcrossWindows(t *testing.T) { + base := time.Unix(0, 0) + planned := base.Add(9900 * time.Millisecond) + queued := base.Add(10200 * time.Millisecond) + started := base.Add(12200 * time.Millisecond) + finished := base.Add(21200 * time.Millisecond) + scheduling, completion := newTimeline(), newTimeline() + scheduling.scheduled("read", planned, queued, false) + scheduling.scheduled("read", base.Add(11*time.Second), base.Add(11*time.Second), true) + completion.completed(point{kind: "page", status: 200, scheduledAt: planned, queuedAt: queued, startedAt: started, finishedAt: finished}) + scheduling.scheduled("publish", planned, queued, false) + completion.completed(point{kind: "publish", status: 503, scheduledAt: planned, queuedAt: queued, startedAt: started, finishedAt: finished}) + completion.merge(scheduling) + w := completion.windows() + if len(w) != 3 || w[0].Reads.Scheduled != 1 || w[0].Publishes.Scheduled != 1 || + w[1].Reads.Scheduled != 1 || w[1].Reads.Dropped != 1 || w[1].Reads.Dispatched != 1 || + w[2].Reads.Completed != 1 || w[2].Publishes.Completed != 1 || w[2].Publishes.Failures != 1 { + t.Fatalf("wrong window attribution: %+v", w) + } + if w[1].Reads.SchedulerLag.MeanMS != 150 || w[1].Reads.QueueDelay.MeanMS != 2000 || + w[1].Reads.HTTPDuration.MeanMS != 9000 || w[1].Reads.HTTPDuration.P95UpperMS != 10000 { + t.Fatalf("timing components were conflated: %+v", w[1].Reads) + } + if w[0].Reads.Dispatched != 0 || w[1].Reads.Completed != 0 || w[2].Reads.Dispatched != 0 { + t.Fatal("late completion was attributed to the wrong interval") + } +} + +func TestTimelineHistogramMergeAndOverflow(t *testing.T) { + var a, b durationHistogram + for range 19 { + a.add(time.Millisecond) + } + b.add(3 * time.Minute) + a.merge(b) + s := a.summary() + if s.Count != 20 || s.P95UpperMS != 1 || s.MaximumMS != 180000 { + t.Fatalf("wrong percentile after merge: %+v", s) + } + if b.summary().P95UpperMS != 180000 || (durationHistogram{}).summary().Count != 0 { + t.Fatal("overflow or empty histogram produced a misleading result") + } +} + +func BenchmarkTimeline(b *testing.B) { + t := newTimeline() + at := time.Unix(100, 0) + p := point{kind: "page", status: 200, scheduledAt: at, queuedAt: at.Add(time.Millisecond), + startedAt: at.Add(2 * time.Millisecond), finishedAt: at.Add(10 * time.Millisecond)} + b.ReportAllocs() + for b.Loop() { + t.scheduled("read", p.scheduledAt, p.queuedAt, false) + t.completed(p) + } +} + +func TestTimelineRetainsFailureClassesAfterMerging(t *testing.T) { + at := time.Unix(100, 0) + a, b := newTimeline(), newTimeline() + for range 150 { + a.completed(point{kind: "page", failure: "body:connection_reset", scheduledAt: at, queuedAt: at, startedAt: at, finishedAt: at}) + } + b.completed(point{kind: "page", failure: "body:connection_reset", scheduledAt: at, queuedAt: at, startedAt: at, finishedAt: at}) + b.completed(point{kind: "page", status: 503, scheduledAt: at, queuedAt: at, startedAt: at, finishedAt: at}) + b.completed(point{kind: "publish", failure: "request:timeout", scheduledAt: at, queuedAt: at, startedAt: at, finishedAt: at}) + a.merge(b) + w := a.windows()[0] + if w.Reads.Failures != 152 || w.Reads.FailureClasses["body:connection_reset"] != 151 || w.Reads.FailureClasses["status:503"] != 1 || w.Publishes.FailureClasses["request:timeout"] != 1 { + t.Fatalf("failure classes lost, capped or mixed across request kinds: %+v", w) + } +} diff --git a/tools/prepare-listing-json/main.go b/tools/prepare-listing-json/main.go new file mode 100644 index 000000000..eff7b1b7e --- /dev/null +++ b/tools/prepare-listing-json/main.go @@ -0,0 +1,74 @@ +// prepare-listing-json backfills derived HTTP response bytes after migration 018. +// It never changes authoritative server JSON or official metadata. +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + "github.com/jackc/pgx/v5" + "github.com/modelcontextprotocol/registry/internal/database" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run() error { + batch := flag.Int("batch", 500, "rows per transaction (1..1000)") + pause := flag.Duration("pause", 100*time.Millisecond, "delay between batches") + flag.Parse() + uri := os.Getenv("DATABASE_URL") + if uri == "" { + return fmt.Errorf("DATABASE_URL is required") + } + if *pause < 0 { + return fmt.Errorf("pause must be nonnegative") + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + config, err := pgx.ParseConfig(uri) + if err != nil { + return fmt.Errorf("invalid DATABASE_URL") + } + config.ConnectTimeout = 10 * time.Second + config.RuntimeParams["statement_timeout"] = "5s" + config.RuntimeParams["lock_timeout"] = "1s" + config.RuntimeParams["application_name"] = "registry-listing-backfill" + conn, err := pgx.ConnectConfig(ctx, config) + if err != nil { + return fmt.Errorf("database connection failed") + } + defer conn.Close(context.Background()) + name, version := "", "" + encoder := json.NewEncoder(os.Stdout) + for { + batchCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + result, err := database.BackfillListingJSONBatch(batchCtx, conn, name, version, *batch) + cancel() + if err != nil { + return fmt.Errorf("backfill batch failed: %w", err) + } + if err := encoder.Encode(result); err != nil { + return err + } + if result.Scanned == 0 { + return nil + } + name, version = result.LastName, result.LastVersion + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(*pause): + } + } +}