feat(server): keep an uninstalling release listed until its pods are gone

docs/openapi.yaml

@@ -825,14 +825,24 @@ paths:
     delete:
       tags: [releases]
       summary: Uninstall
-      description: Removes the release first, then its plan record, so a failure never leaves a live release untracked.
+      description: >-
+        Removes the release. Helm uninstalls and keeps the plan until pods
+        finish terminating. An llmsvc release returns 200 as soon as the
+        LLMService delete is accepted; the operator removes the route,
+        uninstalls the chart, and waits for pods. The release stays listed
+        with phase uninstalling, and deletion passes through the operator's
+        stage, until the LLMService is gone.
       operationId: uninstall
       parameters:
         - $ref: "#/components/parameters/namespace"
         - $ref: "#/components/parameters/release"
       responses:
         "200":
-          description: Uninstalled.
+          description: >-
+            Uninstalled, or output is "uninstall already in progress" when the
+            release is already uninstalling. An llmsvc release returns that
+            in-progress body as soon as the delete is accepted; the operator
+            finishes the teardown.
           content:
             application/json:
               schema:
@@ -846,6 +856,11 @@ paths:
         "401": { $ref: "#/components/responses/Unauthorized" }
         "403": { $ref: "#/components/responses/Forbidden" }
         "404": { $ref: "#/components/responses/NotFound" }
+        "409":
+          description: An apply on this release is in progress.
+          content:
+            application/json:
+              schema: { $ref: "#/components/schemas/Error" }
         "500": { $ref: "#/components/responses/ServerError" }
         "502": { $ref: "#/components/responses/BadGateway" }
 
@@ -1877,26 +1892,55 @@ components:
 
     Deployment:
       type: object
-      required: [release, namespace, revision, backend]
+      required: [release, namespace, backend]
       properties:
         release: { type: string }
         namespace: { type: string }
         chart: { type: string, description: "`name-version`." }
         status: { type: string, description: "helm's status." }
-        revision: { type: integer }
+        revision: { type: integer, description: "helm's revision. Omitted when there is no revision." }
         updated: { type: string, format: date-time }
         model: { type: string }
         variant: { type: string }
         catalogRef: { type: string }
         catalog: { type: string, description: "The configured catalog it came from; absent when the site no longer lists it." }
         version: { type: string }
-        phase: { type: string, description: "swiss's apply phase: applying, applied, failed (or the operator's)." }
-        drift: { type: string, description: "What a person should look at." }
+        phase: { type: string, description: "swiss's apply phase: applying, applied, failed, uninstalling (or the operator's). uninstalling is also set on an LLMService while metadata.deletionTimestamp is set or status.phase is Deleting." }
+        drift: { type: string, description: "What a person should look at. An uninstall in progress names the operator's stage when there is one." }
+        deletion:
+          $ref: "#/components/schemas/LLMDeletion"
         route: { type: string }
         backend: { $ref: "#/components/schemas/ApplyWith" }
         external: { type: boolean, description: "An LLMService swiss did not write." }
         cleanupPending: { type: boolean, description: "Both a plan ConfigMap and an LLMService exist." }
 
+    LLMDeletion:
+      type: object
+      description: Operator staged teardown of an LLMService. Swiss passes the fields through.
+      properties:
+        stage: { type: string, description: "RouteRemoving, RouteSettling, Uninstalling, PodsTerminating, ForceDeleting, or Released." }
+        outcome: { type: string, description: "Clean, NoRoute, RouteStuck, PodsForceDeleted, or TimedOut." }
+        podsRemaining: { type: integer }
+        deadline: { type: string, format: date-time }
+        startedAt: { type: string, format: date-time }
+        message: { type: string }
+        pods:
+          type: array
+          items: { $ref: "#/components/schemas/LLMDeletionPod" }
+        forceDeleted:
+          type: array
+          items: { type: string }
+
+    LLMDeletionPod:
+      type: object
+      properties:
+        name: { type: string }
+        phase: { type: string }
+        node: { type: string }
+        deletionTimestamp: { type: string, format: date-time }
+        grace: { type: integer }
+        stuck: { type: boolean }
+
     ReleaseStatus:
       type: object
       required: [release, namespace, exists, revision, pods, ready, total, authPrefix]
@@ -1933,7 +1977,7 @@ components:
           description: Helm releases. The last apply as swiss recorded it.
           required: [phase]
           properties:
-            phase: { type: string, enum: [applying, applied, failed] }
+            phase: { type: string, enum: [applying, applied, failed, uninstalling] }
             action: { type: string }
             revision: { type: integer }
             startedAt: { type: string, format: date-time }
@@ -1947,9 +1991,12 @@ components:
           type: object
           description: LLMService releases. The operator's status.
           properties:
-            phase: { type: string, examples: [Applying, Applied, Failed] }
+            phase: { type: string, description: "uninstalling while metadata.deletionTimestamp is set or status.phase is Deleting; otherwise the operator's phase.", examples: [Applying, Applied, Failed, uninstalling] }
+            operatorPhase: { type: string, description: "status.phase as the operator wrote it, including Deleting.", examples: [Applying, Applied, Failed, Deleting] }
             message: { type: string }
             revision: { type: integer }
+            deletion:
+              $ref: "#/components/schemas/LLMDeletion"
             conditions:
               type: array
               items:

docs/swiss-design.md

@@ -52,13 +52,54 @@ form / flags  ─┘             (hash)     ↓
 | `POST /api/diff` | diff a stored plan against the live release |
 | `POST /api/apply` | upgrade, asserting the diffed revision when one is supplied |
 | `POST /api/install` | create |
-| `DELETE /api/releases/{ns}/{release}` | uninstall |
+| `DELETE /api/releases/{ns}/{release}` | uninstall; a helm release stays listed as `uninstalling` until helm has removed it and its pods are gone |
 | `POST /api/releases/{ns}/{release}/chat` | health check: one real inference request |
 | `GET /api/runs`, `GET /api/runs/{id}` | the operation log; output is per row, not in the list |
 | `GET /api/nodes` | the GPU inventory: type, allocatable, in use, and what holds it |
 | `GET /api/profile` | the site profile as parsed, after defaults |
 
 
+## Uninstalling
+
+A helm release stays on `GET /api/deployments` with `phase: uninstalling` until helm has removed the release and its pods are gone. `DELETE` returns when helm does; it does not wait for those pods to finish terminating.
+
+| | |
+| --- | --- |
+| marker | plan ConfigMap `status.yaml` `phase: uninstalling`, plus label `swiss.modelsphere.dev/phase=uninstalling` so the metadata-only list can see it |
+| memory | `namespace/release` on the process; `inflight` while that helm call has not returned, so a list cannot delete the plan underneath it |
+| rebuild | the background loop at startup, and the first deployments list, fill that set from ConfigMaps carrying the label |
+| drop | helm no longer has the release, and no pod matches `app=<release>-<chart>` (when the plan names a chart) or `app.kubernetes.io/instance=<release>` |
+| check | every 5s from `Server.Run`, and again when the deployments list or that release's status is read |
+
+```
+DELETE
+  ├─ no plan ConfigMap ─► helm uninstall ─► delete the ConfigMap
+  └─ plan ConfigMap
+        write phase + label
+        helm uninstall
+          ├─ error ─► phase failed, clear the label, keep the ConfigMap
+          └─ ok ─► delete archive Secrets
+                helm release still recorded, or pods match either selector
+                  ├─ yes ─► stay listed (phase uninstalling)
+                  └─ no  ─► delete the ConfigMap and drop the row
+```
+
+| situation | result |
+| --- | --- |
+| helm uninstall fails | `phase: failed` and the error; the plan stays; the row does not stay `uninstalling` |
+| DELETE again, helm release already gone | 200, `output` is `uninstall already in progress` |
+| DELETE again, helm release still recorded | helm uninstall runs again |
+| install, upgrade, or rollback of that name | 409 `release <ns>/<name> is uninstalling` |
+| ConfigMap delete fails | the row stays; the next check retries |
+| pod list fails, or reading the helm release fails | the row stays |
+| helm release still recorded and no pods (restart mid-uninstall) | the row stays `uninstalling`; the next DELETE runs helm again |
+| swissd restarts | the label is read back; the row stays until the helm release and its pods are both gone |
+
+An LLMService with `metadata.deletionTimestamp` set reports `phase: uninstalling` on the deployments row and `llmservice.phase` on status, whatever `status.phase` says. Install, upgrade, and rollback of that name are 409. `DELETE` of an LLMService still waits until the object is gone.
+
+`phase` gains the value `uninstalling` on a deployment row, on helm `planStatus.phase`, and on `llmservice.phase`. No other field on those objects changes.
+
+
 ## Catalog
 
 See [model-catalog](https://github.com/modelsphere/model-catalog).

internal/autoscale/decide.go

@@ -26,6 +26,9 @@ const (
 	// HoldReleaseFailed is a helm or plan failure this loop did not record.
 	// The tick holds instead of applying again.
 	HoldReleaseFailed = "release failed"
+	// HoldUninstalling is a release on its way out: the plan is marked
+	// uninstalling, or the LLMService is being deleted. Nothing is applied.
+	HoldUninstalling = "uninstalling"
 
 	backoffBase = time.Minute
 	backoffCap  = 30 * time.Minute
@@ -73,6 +76,8 @@ type Live struct {
 	Roles                 []Role   `json:"roles,omitempty" yaml:"roles,omitempty"`
 	Routers               []Router `json:"routers,omitempty" yaml:"routers,omitempty"`
 	LockBusy              bool     `json:"lockBusy,omitempty" yaml:"lockBusy,omitempty"`
+	// LLMDeleting is an LLMService with a deletionTimestamp or status.phase Deleting.
+	LLMDeleting bool `json:"llmDeleting,omitempty" yaml:"llmDeleting,omitempty"`
 }
 
 // Role is one DisaggregatedSet roleStatuses entry.
@@ -265,6 +270,9 @@ func current(live Live) int {
 
 // rolloutHold is a change still landing. It waits in either direction.
 func rolloutHold(live Live) string {
+	if live.PlanPhase == "uninstalling" || live.LLMDeleting || live.LLMPhase == "Deleting" {
+		return HoldUninstalling
+	}
 	if live.LockBusy {
 		return HoldApplyInProgress
 	}

internal/autoscale/testdata/decide/guard-llm-deleting.yaml

@@ -0,0 +1,23 @@
+name: guard-llm-deleting
+description: an LLMService being deleted holds, even on a scale-down
+config:
+  enabled: true
+  minSlices: 1
+  maxSlices: 4
+live:
+  backend: llmsvc
+  llmPhase: Applied
+  llmGeneration: 1
+  llmObservedGeneration: 1
+  llmDeleting: true
+  pd: true
+  planReplicaCount: 3
+decision:
+  ok: true
+  replicas: 1
+wantHold: "uninstalling"
+wantScale: false
+wantCurrent: 3
+want: 0
+wantFailed: false
+hasDecision: true

internal/autoscale/testdata/decide/guard-llm-phase-deleting.yaml

@@ -0,0 +1,22 @@
+name: guard-llm-phase-deleting
+description: status.phase Deleting holds, even when deletionTimestamp is not set
+config:
+  enabled: true
+  minSlices: 1
+  maxSlices: 4
+live:
+  backend: llmsvc
+  llmPhase: Deleting
+  llmGeneration: 1
+  llmObservedGeneration: 1
+  pd: true
+  planReplicaCount: 3
+decision:
+  ok: true
+  replicas: 1
+wantHold: "uninstalling"
+wantScale: false
+wantCurrent: 3
+want: 0
+wantFailed: false
+hasDecision: true

internal/autoscale/testdata/decide/guard-uninstalling.yaml

@@ -0,0 +1,22 @@
+name: guard-uninstalling
+description: a plan marked uninstalling holds before any other guard
+config:
+  enabled: true
+  minSlices: 1
+  maxSlices: 4
+live:
+  backend: helm
+  helmStatus: deployed
+  planPhase: uninstalling
+  pd: true
+  planReplicaCount: 1
+  lockBusy: true
+decision:
+  ok: true
+  replicas: 2
+wantHold: "uninstalling"
+wantScale: false
+wantCurrent: 1
+want: 0
+wantFailed: false
+hasDecision: true

internal/cluster/kube.go

@@ -62,6 +62,10 @@ type Kube struct {
 // PlanConfigMapPrefix names the ConfigMap holding a release's live plan.
 const PlanConfigMapPrefix = "swiss-plan-"
 
+// LabelPhase is the plan ConfigMap label whose value is the release phase when
+// that phase has to be visible without reading the ConfigMap body.
+const LabelPhase = "swiss.modelsphere.dev/phase"
+
 // PlanSecretPrefix names the Secrets holding previous plans, one per revision:
 // swiss.plan.v1.<release>.v<revision>, the shape helm uses for its own.
 const PlanSecretPrefix = "swiss.plan.v1."
@@ -264,7 +268,7 @@ func (k *Kube) ManagedRefs(ctx context.Context) ([]ManagedRef, error) {
 			if !ok || name == "" {
 				continue
 			}
-			out = append(out, ManagedRef{Namespace: item.Namespace, Name: name})
+			out = append(out, ManagedRef{Namespace: item.Namespace, Name: name, Labels: maps.Clone(item.GetLabels())})
 		}
 	}
 	sort.Slice(out, func(i, j int) bool {
@@ -615,16 +619,22 @@ var (
 )
 
 // PutConfigMap creates or replaces a ConfigMap given as "namespace/name".
-func (k *Kube) PutConfigMap(ctx context.Context, ref string, data map[string]string) error {
+func (k *Kube) PutConfigMap(ctx context.Context, ref string, data, labels map[string]string) error {
 	ns, name, err := SplitRef(ref)
 	if err != nil {
 		return err
 	}
+	labs := map[string]string{"app.kubernetes.io/managed-by": "swiss"}
+	for k, v := range labels {
+		if v != "" {
+			labs[k] = v
+		}
+	}
 	cm := &corev1.ConfigMap{
 		ObjectMeta: metav1.ObjectMeta{
 			Namespace: ns,
 			Name:      name,
-			Labels:    map[string]string{"app.kubernetes.io/managed-by": "swiss"},
+			Labels:    labs,
 		},
 		Data: data,
 	}
@@ -731,6 +741,18 @@ func (k *Kube) Pods(ctx context.Context, namespace, selector string) ([]Pod, err
 			Phase: string(p.Status.Phase),
 			Node:  p.Spec.NodeName,
 		}
+		if p.DeletionTimestamp != nil && !p.DeletionTimestamp.IsZero() {
+			t := p.DeletionTimestamp.Time
+			pod.DeletionTimestamp = &t
+		}
+		if p.DeletionGracePeriodSeconds != nil {
+			v := *p.DeletionGracePeriodSeconds
+			pod.DeletionGracePeriodSeconds = &v
+		}
+		if p.Spec.TerminationGracePeriodSeconds != nil {
+			v := *p.Spec.TerminationGracePeriodSeconds
+			pod.TerminationGracePeriodSeconds = &v
+		}
 		if !p.CreationTimestamp.IsZero() {
 			pod.AgeSecond = int64(time.Since(p.CreationTimestamp.Time).Seconds())
 		}