From 65fa74f5cc7330c56fa04ac6c53a5df27770a37d Mon Sep 17 00:00:00 2001 From: daiqingyuan Date: Wed, 5 Aug 2026 16:22:34 +0800 Subject: [PATCH] ci: capture bounded pod failure diagnostics --- .github/utils/collect_pod_diagnostics.sh | 123 ++++++++++++++++++ .../tests/test_collect_pod_diagnostics.sh | 79 +++++++++++ .github/workflows/cloud-e2e-installer.yml | 63 +++------ 3 files changed, 220 insertions(+), 45 deletions(-) create mode 100755 .github/utils/collect_pod_diagnostics.sh create mode 100755 .github/utils/tests/test_collect_pod_diagnostics.sh diff --git a/.github/utils/collect_pod_diagnostics.sh b/.github/utils/collect_pod_diagnostics.sh new file mode 100755 index 00000000..6c7eda3b --- /dev/null +++ b/.github/utils/collect_pod_diagnostics.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash + +set -u + +namespace="${1:-}" +pod_name="${2:-}" +log_tail_lines="${LOG_TAIL_LINES:-100}" +log_limit_bytes="${LOG_LIMIT_BYTES:-32768}" +event_limit="${EVENT_LIMIT:-20}" +container_limit="${CONTAINER_LIMIT:-10}" + +if [[ -z "${namespace}" || -z "${pod_name}" ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +for value in "${log_tail_lines}" "${log_limit_bytes}" "${event_limit}" "${container_limit}"; do + if [[ ! "${value}" =~ ^[1-9][0-9]*$ ]]; then + echo "diagnostic limits must be positive integers" >&2 + exit 2 + fi +done + +echo "==================== pod ${namespace}/${pod_name} diagnostics ====================" + +pod_json="$(kubectl get pod -n "${namespace}" "${pod_name}" -o json 2>&1)" +if [[ $? -ne 0 ]]; then + echo "unable to read pod: ${pod_json}" + exit 0 +fi + +# Select only runtime fields needed for debugging. Environment variables, +# volumes, annotations, and Secret data are deliberately excluded. +printf '%s\n' "${pod_json}" | jq ' + def safe_arg: + if test("(?i)(password|passwd|token|secret|authorization|credential|api.?key|license)") then + "" + elif length > 256 then + .[0:256] + "..." + else + . + end; + def state_summary: + if .waiting then + {waiting: {reason: (.waiting.reason // ""), message: (.waiting.message // "")}} + elif .terminated then + {terminated: { + reason: (.terminated.reason // ""), + message: (.terminated.message // ""), + exitCode: (.terminated.exitCode // null), + signal: (.terminated.signal // null), + startedAt: (.terminated.startedAt // null), + finishedAt: (.terminated.finishedAt // null) + }} + elif .running then + {running: {startedAt: (.running.startedAt // null)}} + else + {} + end; + def status_for($name): + first(.status.containerStatuses[]? | select(.name == $name)) // {}; + { + pod: .metadata.name, + phase: (.status.phase // ""), + reason: (.status.reason // ""), + message: (.status.message // ""), + containers: [ + .spec.containers[] as $spec + | status_for($spec.name) as $status + | { + name: $spec.name, + image: $spec.image, + imageID: ($status.imageID // ""), + command: (($spec.command // []) | map(safe_arg)), + args: (($spec.args // []) | map(safe_arg)), + restartCount: ($status.restartCount // 0), + currentState: (($status.state // {}) | state_summary), + lastState: (($status.lastState // {}) | state_summary) + } + ] + } +' + +echo "==================== pod ${namespace}/${pod_name} recent events ====================" +events_json="$(kubectl get events -n "${namespace}" \ + --field-selector "involvedObject.kind=Pod,involvedObject.name=${pod_name}" \ + -o json 2>&1)" +if [[ $? -eq 0 ]]; then + printf '%s\n' "${events_json}" | jq -r --argjson limit "${event_limit}" ' + .items + | sort_by(.lastTimestamp // .eventTime // .metadata.creationTimestamp // "") + | .[-$limit:] + | .[] + | [ + (.lastTimestamp // .eventTime // .metadata.creationTimestamp // ""), + (.type // ""), + (.reason // ""), + ((.message // "") | gsub("[\\r\\n]+"; " ") | .[0:1000]) + ] + | @tsv + ' +else + echo "unable to read pod events: ${events_json}" +fi + +printf '%s\n' "${pod_json}" | jq -r ' + .spec.containers[] as $spec + | (first(.status.containerStatuses[]? | select(.name == $spec.name)) // {}) as $status + | [$spec.name, ($status.restartCount // 0)] + | @tsv +' | head -n "${container_limit}" | while IFS=$'\t' read -r container_name restart_count; do + echo "==================== pod ${namespace}/${pod_name} container ${container_name} current log ====================" + kubectl logs -n "${namespace}" "${pod_name}" -c "${container_name}" \ + --tail="${log_tail_lines}" --limit-bytes="${log_limit_bytes}" 2>&1 || true + + if [[ "${restart_count}" -gt 0 ]]; then + echo "==================== pod ${namespace}/${pod_name} container ${container_name} previous log ====================" + kubectl logs -n "${namespace}" "${pod_name}" -c "${container_name}" --previous \ + --tail="${log_tail_lines}" --limit-bytes="${log_limit_bytes}" 2>&1 || true + fi +done + +exit 0 diff --git a/.github/utils/tests/test_collect_pod_diagnostics.sh b/.github/utils/tests/test_collect_pod_diagnostics.sh new file mode 100755 index 00000000..d876542f --- /dev/null +++ b/.github/utils/tests/test_collect_pod_diagnostics.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +diagnostic_script="${script_dir}/../collect_pod_diagnostics.sh" + +kubectl() { + if [[ "$1" == "get" && "$2" == "pod" ]]; then + printf '%s\n' '{ + "metadata": {"name": "hermes-agent-abc"}, + "spec": { + "containers": [{ + "name": "hermes-agent", + "image": "apecloud/hermes-agent:v1", + "command": ["/app/hermes-agent"], + "args": ["--token=top-secret", "--mode=worker"], + "env": [{"name": "PRIVATE_VALUE", "value": "must-not-leak"}] + }] + }, + "status": { + "phase": "Pending", + "containerStatuses": [{ + "name": "hermes-agent", + "imageID": "docker-pullable://apecloud/hermes-agent@sha256:1234", + "restartCount": 2, + "state": {"waiting": {"reason": "RunContainerError", "message": "runtime failed"}}, + "lastState": {"terminated": {"reason": "Error", "exitCode": 126}} + }] + } + }' + return 0 + fi + + if [[ "$1" == "get" && "$2" == "events" ]]; then + printf '%s\n' '{ + "items": [{ + "lastTimestamp": "2026-08-05T00:00:00Z", + "type": "Warning", + "reason": "Failed", + "message": "container start failed" + }] + }' + return 0 + fi + + if [[ "$1" == "logs" ]]; then + printf 'mock-log args: %s\n' "$*" + return 0 + fi + + printf 'unexpected kubectl call: %s\n' "$*" >&2 + return 1 +} +export -f kubectl + +output="$(bash "${diagnostic_script}" kb-cloud hermes-agent-abc)" + +for expected in \ + 'RunContainerError' \ + '"exitCode": 126' \ + 'docker-pullable://apecloud/hermes-agent@sha256:1234' \ + $'Warning\tFailed\tcontainer start failed' \ + '--tail=100 --limit-bytes=32768' \ + '--previous --tail=100 --limit-bytes=32768'; do + if ! grep -Fq -- "${expected}" <<<"${output}"; then + printf 'missing expected diagnostic: %s\n%s\n' "${expected}" "${output}" >&2 + exit 1 + fi +done + +for forbidden in 'top-secret' 'must-not-leak'; do + if grep -Fq -- "${forbidden}" <<<"${output}"; then + printf 'sensitive value leaked: %s\n%s\n' "${forbidden}" "${output}" >&2 + exit 1 + fi +done + +echo "collect_pod_diagnostics contract: PASS" diff --git a/.github/workflows/cloud-e2e-installer.yml b/.github/workflows/cloud-e2e-installer.yml index 27726fd4..01889289 100644 --- a/.github/workflows/cloud-e2e-installer.yml +++ b/.github/workflows/cloud-e2e-installer.yml @@ -245,22 +245,13 @@ jobs: kubectl logs --tail 100 -n kb-cloud ${installer_pod_name} done - echo "describe image pull error installer pod" - installer_error_pods=$(kubectl get pods -n kb-cloud | (grep "kb-cloud-installer" | grep -v "Completed" | grep -v "Running" | egrep "ImagePull|Pending|Evicted" || true) | awk '{print $1}') - echo "installer image pull error pod: ${installer_error_pods}" - for installer_error_pod in $(echo "${installer_error_pods}"); do - echo "==================== pod ${installer_error_pod} describe ====================" - kubectl describe pod -n kb-cloud ${installer_error_pod} - echo "" - done - - # logs kb-cloud error pod - kb_cloud_error_pods=$(kubectl get pods -n kb-cloud | (grep -v "kb-cloud-installer" | grep -v "Completed" | grep -v "Running" | grep -v "NAME" | grep -v "ImagePull" || true) | awk '{print $1}') - echo "kb-cloud error pod: $kb_cloud_error_pods" + kb_cloud_error_pods=$(kubectl get pods -n kb-cloud --no-headers \ + | awk '$3 != "Running" && $3 != "Completed" {print $1}' \ + | head -n 10) + echo "kb-cloud error pod: ${kb_cloud_error_pods}" for kb_cloud_error_pod in $(echo "${kb_cloud_error_pods}"); do - echo "==================== pod ${kb_cloud_error_pod} logs ====================" - kubectl logs --tail 100 -n kb-cloud ${kb_cloud_error_pod} - echo "" + bash ${{ github.workspace }}/apecloud-cd/.github/utils/collect_pod_diagnostics.sh \ + kb-cloud "${kb_cloud_error_pod}" || true done set -e @@ -585,22 +576,13 @@ jobs: kubectl logs --tail 100 -n kb-cloud ${installer_pod_name} done - # describe image pull error installer pod - installer_error_pods=$(kubectl get pods -n kb-cloud | (grep "kb-cloud-installer" | grep -v "Completed" | grep -v "Running" | egrep "ImagePull|Pending|Evicted" || true) | awk '{print $1}') - echo "installer image pull error pod: ${installer_error_pods}" - for installer_error_pod in $(echo "${installer_error_pods}"); do - echo "==================== pod ${installer_error_pod} describe ====================" - kubectl describe pod -n kb-cloud ${installer_error_pod} - echo "" - done - - # logs kb-cloud error pod - kb_cloud_error_pods=$(kubectl get pods -n kb-cloud | (grep -v "kb-cloud-installer" | grep -v "Completed" | grep -v "Running" | grep -v "NAME" | grep -v "ImagePull" || true) | awk '{print $1}') - echo "kb-cloud error pod: $kb_cloud_error_pods" + kb_cloud_error_pods=$(kubectl get pods -n kb-cloud --no-headers \ + | awk '$3 != "Running" && $3 != "Completed" {print $1}' \ + | head -n 10) + echo "kb-cloud error pod: ${kb_cloud_error_pods}" for kb_cloud_error_pod in $(echo "${kb_cloud_error_pods}"); do - echo "==================== pod ${kb_cloud_error_pod} logs ====================" - kubectl logs --tail 100 -n kb-cloud ${kb_cloud_error_pod} - echo "" + bash ${{ github.workspace }}/apecloud-cd/.github/utils/collect_pod_diagnostics.sh \ + kb-cloud "${kb_cloud_error_pod}" || true done set -e @@ -684,22 +666,13 @@ jobs: upgrade_result="[FAILED]" echo upgrade-result="${upgrade_result}" >> $GITHUB_OUTPUT - # logs kb-cloud error pod - kb_cloud_error_pods=$(kubectl get pods -n kb-cloud --no-headers | grep -v "Completed" | grep -v "Running" | awk '{print $1}') - echo "kb-cloud error pod: $kb_cloud_error_pods" + kb_cloud_error_pods=$(kubectl get pods -n kb-cloud --no-headers \ + | awk '$3 != "Running" && $3 != "Completed" {print $1}' \ + | head -n 10) + echo "kb-cloud error pod: ${kb_cloud_error_pods}" for kb_cloud_error_pod in $(echo "${kb_cloud_error_pods}"); do - echo "==================== pod ${kb_cloud_error_pod} logs ====================" - kubectl logs --tail 100 -n kb-cloud ${kb_cloud_error_pod} - echo "" - done - - # describe image pull error kb-cloud pod - kb_cloud_image_pull_error_pods=$(kubectl get pods -n kb-cloud --no-headers | grep -v "Completed" | grep -v "Running" | egrep "ImagePull|Pending|Evicted" | awk '{print $1}') - echo "kb-cloud image pull error pod: $kb_cloud_image_pull_error_pods" - for kb_cloud_image_pull_error_pod in $(echo "${kb_cloud_image_pull_error_pods}"); do - echo "==================== pod ${kb_cloud_image_pull_error_pod} describe ====================" - kubectl describe pod -n kb-cloud ${kb_cloud_image_pull_error_pod} - echo "" + bash ${{ github.workspace }}/apecloud-cd/.github/utils/collect_pod_diagnostics.sh \ + kb-cloud "${kb_cloud_error_pod}" || true done exit 1 else