ci: capture bounded pod failure diagnostics

.github/utils/collect_pod_diagnostics.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+
+set -u
+
+namespace="${1:-}"
+pod_name="${2:-}"
+log_tail_lines="${LOG_TAIL_LINES:-100}"
+log_limit_bytes="${LOG_LIMIT_BYTES:-32768}"
+event_limit="${EVENT_LIMIT:-20}"
+container_limit="${CONTAINER_LIMIT:-10}"
+
+if [[ -z "${namespace}" || -z "${pod_name}" ]]; then
+    echo "usage: $0 <namespace> <pod-name>" >&2
+    exit 2
+fi
+
+for value in "${log_tail_lines}" "${log_limit_bytes}" "${event_limit}" "${container_limit}"; do
+    if [[ ! "${value}" =~ ^[1-9][0-9]*$ ]]; then
+        echo "diagnostic limits must be positive integers" >&2
+        exit 2
+    fi
+done
+
+echo "==================== pod ${namespace}/${pod_name} diagnostics ===================="
+
+pod_json="$(kubectl get pod -n "${namespace}" "${pod_name}" -o json 2>&1)"
+if [[ $? -ne 0 ]]; then
+    echo "unable to read pod: ${pod_json}"
+    exit 0
+fi
+
+# Select only runtime fields needed for debugging. Environment variables,
+# volumes, annotations, and Secret data are deliberately excluded.
+printf '%s\n' "${pod_json}" | jq '
+    def safe_arg:
+        if test("(?i)(password|passwd|token|secret|authorization|credential|api.?key|license)") then
+            "<redacted>"
+        elif length > 256 then
+            .[0:256] + "...<truncated>"
+        else
+            .
+        end;
+    def state_summary:
+        if .waiting then
+            {waiting: {reason: (.waiting.reason // ""), message: (.waiting.message // "")}}
+        elif .terminated then
+            {terminated: {
+                reason: (.terminated.reason // ""),
+                message: (.terminated.message // ""),
+                exitCode: (.terminated.exitCode // null),
+                signal: (.terminated.signal // null),
+                startedAt: (.terminated.startedAt // null),
+                finishedAt: (.terminated.finishedAt // null)
+            }}
+        elif .running then
+            {running: {startedAt: (.running.startedAt // null)}}
+        else
+            {}
+        end;
+    def status_for($name):
+        first(.status.containerStatuses[]? | select(.name == $name)) // {};
+    {
+        pod: .metadata.name,
+        phase: (.status.phase // ""),
+        reason: (.status.reason // ""),
+        message: (.status.message // ""),
+        containers: [
+            .spec.containers[] as $spec
+            | status_for($spec.name) as $status
+            | {
+                name: $spec.name,
+                image: $spec.image,
+                imageID: ($status.imageID // ""),
+                command: (($spec.command // []) | map(safe_arg)),
+                args: (($spec.args // []) | map(safe_arg)),
+                restartCount: ($status.restartCount // 0),
+                currentState: (($status.state // {}) | state_summary),
+                lastState: (($status.lastState // {}) | state_summary)
+            }
+        ]
+    }
+'
+
+echo "==================== pod ${namespace}/${pod_name} recent events ===================="
+events_json="$(kubectl get events -n "${namespace}" \
+    --field-selector "involvedObject.kind=Pod,involvedObject.name=${pod_name}" \
+    -o json 2>&1)"
+if [[ $? -eq 0 ]]; then
+    printf '%s\n' "${events_json}" | jq -r --argjson limit "${event_limit}" '
+        .items
+        | sort_by(.lastTimestamp // .eventTime // .metadata.creationTimestamp // "")
+        | .[-$limit:]
+        | .[]
+        | [
+            (.lastTimestamp // .eventTime // .metadata.creationTimestamp // ""),
+            (.type // ""),
+            (.reason // ""),
+            ((.message // "") | gsub("[\\r\\n]+"; " ") | .[0:1000])
+        ]
+        | @tsv
+    '
+else
+    echo "unable to read pod events: ${events_json}"
+fi
+
+printf '%s\n' "${pod_json}" | jq -r '
+    .spec.containers[] as $spec
+    | (first(.status.containerStatuses[]? | select(.name == $spec.name)) // {}) as $status
+    | [$spec.name, ($status.restartCount // 0)]
+    | @tsv
+' | head -n "${container_limit}" | while IFS=$'\t' read -r container_name restart_count; do
+    echo "==================== pod ${namespace}/${pod_name} container ${container_name} current log ===================="
+    kubectl logs -n "${namespace}" "${pod_name}" -c "${container_name}" \
+        --tail="${log_tail_lines}" --limit-bytes="${log_limit_bytes}" 2>&1 || true
+
+    if [[ "${restart_count}" -gt 0 ]]; then
+        echo "==================== pod ${namespace}/${pod_name} container ${container_name} previous log ===================="
+        kubectl logs -n "${namespace}" "${pod_name}" -c "${container_name}" --previous \
+            --tail="${log_tail_lines}" --limit-bytes="${log_limit_bytes}" 2>&1 || true
+    fi
+done
+
+exit 0

.github/utils/tests/test_collect_pod_diagnostics.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+diagnostic_script="${script_dir}/../collect_pod_diagnostics.sh"
+
+kubectl() {
+    if [[ "$1" == "get" && "$2" == "pod" ]]; then
+        printf '%s\n' '{
+            "metadata": {"name": "hermes-agent-abc"},
+            "spec": {
+                "containers": [{
+                    "name": "hermes-agent",
+                    "image": "apecloud/hermes-agent:v1",
+                    "command": ["/app/hermes-agent"],
+                    "args": ["--token=top-secret", "--mode=worker"],
+                    "env": [{"name": "PRIVATE_VALUE", "value": "must-not-leak"}]
+                }]
+            },
+            "status": {
+                "phase": "Pending",
+                "containerStatuses": [{
+                    "name": "hermes-agent",
+                    "imageID": "docker-pullable://apecloud/hermes-agent@sha256:1234",
+                    "restartCount": 2,
+                    "state": {"waiting": {"reason": "RunContainerError", "message": "runtime failed"}},
+                    "lastState": {"terminated": {"reason": "Error", "exitCode": 126}}
+                }]
+            }
+        }'
+        return 0
+    fi
+
+    if [[ "$1" == "get" && "$2" == "events" ]]; then
+        printf '%s\n' '{
+            "items": [{
+                "lastTimestamp": "2026-08-05T00:00:00Z",
+                "type": "Warning",
+                "reason": "Failed",
+                "message": "container start failed"
+            }]
+        }'
+        return 0
+    fi
+
+    if [[ "$1" == "logs" ]]; then
+        printf 'mock-log args: %s\n' "$*"
+        return 0
+    fi
+
+    printf 'unexpected kubectl call: %s\n' "$*" >&2
+    return 1
+}
+export -f kubectl
+
+output="$(bash "${diagnostic_script}" kb-cloud hermes-agent-abc)"
+
+for expected in \
+    'RunContainerError' \
+    '"exitCode": 126' \
+    'docker-pullable://apecloud/hermes-agent@sha256:1234' \
+    $'Warning\tFailed\tcontainer start failed' \
+    '--tail=100 --limit-bytes=32768' \
+    '--previous --tail=100 --limit-bytes=32768'; do
+    if ! grep -Fq -- "${expected}" <<<"${output}"; then
+        printf 'missing expected diagnostic: %s\n%s\n' "${expected}" "${output}" >&2
+        exit 1
+    fi
+done
+
+for forbidden in 'top-secret' 'must-not-leak'; do
+    if grep -Fq -- "${forbidden}" <<<"${output}"; then
+        printf 'sensitive value leaked: %s\n%s\n' "${forbidden}" "${output}" >&2
+        exit 1
+    fi
+done
+
+echo "collect_pod_diagnostics contract: PASS"

.github/workflows/cloud-e2e-installer.yml

@@ -245,22 +245,13 @@ jobs:
                   kubectl logs --tail 100 -n kb-cloud ${installer_pod_name}
               done 
 
-              echo "describe image pull error installer pod"
-              installer_error_pods=$(kubectl get pods -n kb-cloud | (grep "kb-cloud-installer" | grep -v "Completed" | grep -v "Running" | egrep "ImagePull|Pending|Evicted" || true) | awk '{print $1}')
-              echo "installer image pull error pod: ${installer_error_pods}"
-              for installer_error_pod in $(echo "${installer_error_pods}"); do
-                  echo "==================== pod ${installer_error_pod} describe ===================="
-                  kubectl describe pod -n kb-cloud ${installer_error_pod}
-                  echo ""
-              done
-
-              # logs kb-cloud error pod 
-              kb_cloud_error_pods=$(kubectl get pods -n kb-cloud | (grep -v "kb-cloud-installer" | grep -v "Completed" | grep -v "Running" | grep -v "NAME" | grep -v "ImagePull" || true) | awk '{print $1}')
-              echo "kb-cloud error pod: $kb_cloud_error_pods"
+              kb_cloud_error_pods=$(kubectl get pods -n kb-cloud --no-headers \
+                  | awk '$3 != "Running" && $3 != "Completed" {print $1}' \
+                  | head -n 10)
+              echo "kb-cloud error pod: ${kb_cloud_error_pods}"
               for kb_cloud_error_pod in $(echo "${kb_cloud_error_pods}"); do
-                  echo "==================== pod ${kb_cloud_error_pod} logs ===================="
-                  kubectl logs --tail 100 -n kb-cloud ${kb_cloud_error_pod}
-                  echo ""
+                  bash ${{ github.workspace }}/apecloud-cd/.github/utils/collect_pod_diagnostics.sh \
+                      kb-cloud "${kb_cloud_error_pod}" || true
               done
 
               set -e
@@ -585,22 +576,13 @@ jobs:
                   kubectl logs --tail 100 -n kb-cloud ${installer_pod_name}
               done 
 
-              # describe image pull error installer pod
-              installer_error_pods=$(kubectl get pods -n kb-cloud | (grep "kb-cloud-installer" | grep -v "Completed" | grep -v "Running" | egrep "ImagePull|Pending|Evicted" || true) | awk '{print $1}')
-              echo "installer image pull error pod: ${installer_error_pods}"
-              for installer_error_pod in $(echo "${installer_error_pods}"); do
-                  echo "==================== pod ${installer_error_pod} describe ===================="
-                  kubectl describe pod -n kb-cloud ${installer_error_pod}
-                  echo ""
-              done
-
-              # logs kb-cloud error pod 
-              kb_cloud_error_pods=$(kubectl get pods -n kb-cloud | (grep -v "kb-cloud-installer" | grep -v "Completed" | grep -v "Running" | grep -v "NAME" | grep -v "ImagePull" || true) | awk '{print $1}')
-              echo "kb-cloud error pod: $kb_cloud_error_pods"
+              kb_cloud_error_pods=$(kubectl get pods -n kb-cloud --no-headers \
+                  | awk '$3 != "Running" && $3 != "Completed" {print $1}' \
+                  | head -n 10)
+              echo "kb-cloud error pod: ${kb_cloud_error_pods}"
               for kb_cloud_error_pod in $(echo "${kb_cloud_error_pods}"); do
-                  echo "==================== pod ${kb_cloud_error_pod} logs ===================="
-                  kubectl logs --tail 100 -n kb-cloud ${kb_cloud_error_pod}
-                  echo ""
+                  bash ${{ github.workspace }}/apecloud-cd/.github/utils/collect_pod_diagnostics.sh \
+                      kb-cloud "${kb_cloud_error_pod}" || true
               done
 
               set -e
@@ -684,22 +666,13 @@ jobs:
               upgrade_result="[FAILED]"
               echo upgrade-result="${upgrade_result}" >> $GITHUB_OUTPUT
 
-              # logs kb-cloud error pod 
-              kb_cloud_error_pods=$(kubectl get pods -n kb-cloud --no-headers | grep -v "Completed" | grep -v "Running" | awk '{print $1}')
-              echo "kb-cloud error pod: $kb_cloud_error_pods"
+              kb_cloud_error_pods=$(kubectl get pods -n kb-cloud --no-headers \
+                  | awk '$3 != "Running" && $3 != "Completed" {print $1}' \
+                  | head -n 10)
+              echo "kb-cloud error pod: ${kb_cloud_error_pods}"
               for kb_cloud_error_pod in $(echo "${kb_cloud_error_pods}"); do
-                  echo "==================== pod ${kb_cloud_error_pod} logs ===================="
-                  kubectl logs --tail 100 -n kb-cloud ${kb_cloud_error_pod}
-                  echo ""
-              done
-
-              # describe image pull error kb-cloud pod
-              kb_cloud_image_pull_error_pods=$(kubectl get pods -n kb-cloud --no-headers | grep -v "Completed" | grep -v "Running" | egrep "ImagePull|Pending|Evicted" | awk '{print $1}')
-              echo "kb-cloud image pull error pod: $kb_cloud_image_pull_error_pods"
-              for kb_cloud_image_pull_error_pod in $(echo "${kb_cloud_image_pull_error_pods}"); do
-                  echo "==================== pod ${kb_cloud_image_pull_error_pod} describe ===================="
-                  kubectl describe pod -n kb-cloud ${kb_cloud_image_pull_error_pod}
-                  echo ""
+                  bash ${{ github.workspace }}/apecloud-cd/.github/utils/collect_pod_diagnostics.sh \
+                      kb-cloud "${kb_cloud_error_pod}" || true
               done
               exit 1
           else