From d577b168bdd1e5b15b6e5f444d9cc13a6c229a2c Mon Sep 17 00:00:00 2001 From: Leon Date: Tue, 11 Aug 2026 16:59:32 +0800 Subject: [PATCH 01/10] fix: run kbagent under tini --- .github/workflows/cicd-pull-request.yml | 12 +++ Makefile | 4 + cmd/kbagent/main.go | 26 ++++++ cmd/kbagent/main_test.go | 94 +++++++++++++++++++ docker/Dockerfile-tools | 3 +- hack/test-kbagent-init.sh | 109 +++++++++++++++++++++++ pkg/controller/component/kbagent.go | 3 +- pkg/controller/component/kbagent_test.go | 1 + 8 files changed, 250 insertions(+), 2 deletions(-) create mode 100644 cmd/kbagent/main_test.go create mode 100755 hack/test-kbagent-init.sh diff --git a/.github/workflows/cicd-pull-request.yml b/.github/workflows/cicd-pull-request.yml index fe1b05814b5..f81dc03232a 100644 --- a/.github/workflows/cicd-pull-request.yml +++ b/.github/workflows/cicd-pull-request.yml @@ -158,6 +158,18 @@ jobs: DOCKERFILE_PATH: "./docker/Dockerfile-tools" secrets: inherit + test-kbagent-init: + needs: trigger-mode + if: contains(needs.trigger-mode.outputs.trigger-mode, '[docker]') + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: docker/setup-buildx-action@v3 + - name: verify kbagent PID 1 behavior + run: make test-kbagent-init + check-dataprotection-image: needs: trigger-mode if: contains(needs.trigger-mode.outputs.trigger-mode, '[docker]') diff --git a/Makefile b/Makefile index bea4ec0d5a8..ae7fa18b10b 100644 --- a/Makefile +++ b/Makefile @@ -238,6 +238,10 @@ dataprotection: generate build-checks ## Build dataprotection binary. kbagent: generate build-checks $(GO) build -ldflags=${LD_FLAGS} -o bin/kbagent ./cmd/kbagent/main.go +.PHONY: test-kbagent-init +test-kbagent-init: ## Build the tools image and verify PID 1 orphan reaping. + hack/test-kbagent-init.sh + .PHONY: helmhook helmhook: $(GO) build -o bin/helmhook ./cmd/helmhook/main.go diff --git a/cmd/kbagent/main.go b/cmd/kbagent/main.go index 3eb0f43fe48..4b50039e52c 100644 --- a/cmd/kbagent/main.go +++ b/cmd/kbagent/main.go @@ -24,6 +24,7 @@ import ( "fmt" "os" "os/signal" + "path/filepath" "strings" "syscall" @@ -42,10 +43,30 @@ import ( const ( defaultMaxConcurrency = 32 + tiniBinaryName = "tini-static" ) var serverConfig server.Config +type execProcessFunc func(argv0 string, argv []string, envv []string) error +type executableFunc func() (string, error) + +func reexecUnderInit(pid int, args, env []string, executable executableFunc, execProcess execProcessFunc) error { + if pid != 1 { + return nil + } + exe, err := executable() + if err != nil { + return errors.Wrap(err, "resolve kbagent executable") + } + tini := filepath.Join(filepath.Dir(exe), tiniBinaryName) + argv := []string{tini, "--", exe} + if len(args) > 1 { + argv = append(argv, args[1:]...) + } + return errors.Wrap(execProcess(tini, argv, env), "exec tini-static") +} + func init() { viper.AutomaticEnv() @@ -60,6 +81,11 @@ func init() { } func main() { + if err := reexecUnderInit(os.Getpid(), os.Args, os.Environ(), os.Executable, syscall.Exec); err != nil { + _, _ = fmt.Fprintf(os.Stderr, "failed to launch kbagent under tini: %v\n", err) + os.Exit(1) + } + // Set GOMAXPROCS _, _ = maxprocs.Set() diff --git a/cmd/kbagent/main_test.go b/cmd/kbagent/main_test.go new file mode 100644 index 00000000000..79495c10db7 --- /dev/null +++ b/cmd/kbagent/main_test.go @@ -0,0 +1,94 @@ +/* +Copyright (C) 2022-2026 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package main + +import ( + "errors" + "reflect" + "testing" +) + +func TestReexecUnderInitSkipsNonPID1(t *testing.T) { + called := false + err := reexecUnderInit(2, []string{"/bin/kbagent"}, nil, + func() (string, error) { + called = true + return "", nil + }, + func(string, []string, []string) error { + called = true + return nil + }) + if err != nil || called { + t.Fatalf("reexecUnderInit() = %v, called = %v", err, called) + } +} + +func TestReexecUnderInitPreservesArgumentsAndEnvironment(t *testing.T) { + tests := []struct { + name string + exe string + }{ + {name: "tools image", exe: "/bin/kbagent"}, + {name: "custom action image", exe: "/kubeblocks/kbagent"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + args := []string{tt.exe, "--port", "3501", "--server=false"} + env := []string{"A=B", "C=D"} + var gotPath string + var gotArgs, gotEnv []string + err := reexecUnderInit(1, args, env, + func() (string, error) { return tt.exe, nil }, + func(path string, argv, envv []string) error { + gotPath = path + gotArgs = append([]string(nil), argv...) + gotEnv = append([]string(nil), envv...) + return nil + }) + if err != nil { + t.Fatalf("reexecUnderInit() error = %v", err) + } + wantPath := tt.exe[:len(tt.exe)-len("kbagent")] + tiniBinaryName + wantArgs := []string{wantPath, "--", tt.exe, "--port", "3501", "--server=false"} + if gotPath != wantPath || !reflect.DeepEqual(gotArgs, wantArgs) || !reflect.DeepEqual(gotEnv, env) { + t.Fatalf("exec = (%q, %#v, %#v), want (%q, %#v, %#v)", gotPath, gotArgs, gotEnv, wantPath, wantArgs, env) + } + }) + } +} + +func TestReexecUnderInitReturnsStartupErrors(t *testing.T) { + resolveErr := errors.New("resolve") + err := reexecUnderInit(1, []string{"kbagent"}, nil, + func() (string, error) { return "", resolveErr }, + func(string, []string, []string) error { return nil }) + if !errors.Is(err, resolveErr) { + t.Fatalf("expected executable error, got %v", err) + } + + execErr := errors.New("exec") + err = reexecUnderInit(1, []string{"/bin/kbagent"}, nil, + func() (string, error) { return "/bin/kbagent", nil }, + func(string, []string, []string) error { return execErr }) + if !errors.Is(err, execErr) { + t.Fatalf("expected exec error, got %v", err) + } +} diff --git a/docker/Dockerfile-tools b/docker/Dockerfile-tools index 0dce566f222..7c4773ed01f 100644 --- a/docker/Dockerfile-tools +++ b/docker/Dockerfile-tools @@ -66,7 +66,8 @@ ENV APK_MIRROR=${APK_MIRROR} RUN if [ -n "${APK_MIRROR}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${APK_MIRROR}/g" /etc/apk/repositories; fi RUN apk upgrade --no-cache # curl is used to send certain requests, such as reload in config-manager. -RUN apk add --no-cache curl kubectl helm jq --allow-untrusted \ +RUN apk add --no-cache curl kubectl helm jq tini-static --allow-untrusted \ + && cp /sbin/tini-static /bin/tini-static \ && rm -rf /var/cache/apk/* # copy kubeblocks tools diff --git a/hack/test-kbagent-init.sh b/hack/test-kbagent-init.sh new file mode 100755 index 00000000000..25f013ed6bb --- /dev/null +++ b/hack/test-kbagent-init.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +IMAGE="${1:-}" + +if [[ -z "${IMAGE}" ]]; then + case "$(docker info --format '{{.Architecture}}')" in + amd64|x86_64) PLATFORM="linux/amd64" ;; + arm64|aarch64) PLATFORM="linux/arm64" ;; + *) + echo "unsupported Docker architecture; set KBAGENT_TEST_PLATFORM explicitly" >&2 + exit 1 + ;; + esac + PLATFORM="${KBAGENT_TEST_PLATFORM:-${PLATFORM}}" + IMAGE="test.local/kubeblocks/kubeblocks-tools:kbagent-init-test" + docker buildx build "${ROOT_DIR}" \ + --file "${ROOT_DIR}/docker/Dockerfile-tools" \ + --platform "${PLATFORM}" \ + --build-arg "GOPROXY=${GOPROXY:-https://proxy.golang.org}" \ + --tag "${IMAGE}" \ + --load +fi + +CONTAINER_NAME="kbagent-init-test-$$" +cleanup() { + docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +ACTIONS='[{"name":"orphan","exec":{"command":["/bin/sh","-c","timeout 1 sleep 0.2 /dev/null 2>&1 &"]}},{"name":"timeout-tree","exec":{"command":["/bin/sh","-c","(sleep 2; touch /tmp/descendant-survived) & wait"]},"timeoutSeconds":1}]' + +docker run -d \ + --name "${CONTAINER_NAME}" \ + -e "KB_AGENT_ACTION=${ACTIONS}" \ + "${IMAGE}" \ + /bin/kbagent >/dev/null + +ready=false +for _ in $(seq 1 60); do + if docker exec "${CONTAINER_NAME}" curl -fsS \ + -H 'Content-Type: application/json' \ + -d '{"action":"orphan","rerun":true}' \ + http://127.0.0.1:3501/v1.0/action >/dev/null 2>&1; then + ready=true + break + fi + sleep 0.1 +done +if [[ "${ready}" != "true" ]]; then + echo "kbagent HTTP endpoint did not become ready" >&2 + docker logs "${CONTAINER_NAME}" >&2 + exit 1 +fi + +docker exec "${CONTAINER_NAME}" sh -ec ' + test "$(cat /proc/1/comm)" = "tini-static" + found_agent=false + for status in /proc/[0-9]*/status; do + name=$(awk "\$1 == \"Name:\" { print \$2 }" "$status") + ppid=$(awk "\$1 == \"PPid:\" { print \$2 }" "$status") + if [ "$name" = "kbagent" ] && [ "$ppid" = "1" ]; then + found_agent=true + break + fi + done + test "$found_agent" = true + + i=0 + while [ "$i" -lt 2000 ]; do + curl -fsS -H "Content-Type: application/json" \ + -d "{\"action\":\"orphan\",\"rerun\":true}" \ + http://127.0.0.1:3501/v1.0/action >/dev/null + i=$((i + 1)) + done +' + +sleep 1 +docker exec "${CONTAINER_NAME}" sh -ec ' + zombies=0 + for status in /proc/[0-9]*/status; do + state=$(awk "\$1 == \"State:\" { print \$2 }" "$status") + ppid=$(awk "\$1 == \"PPid:\" { print \$2 }" "$status") + if [ "$state" = "Z" ] && [ "$ppid" = "1" ]; then + zombies=$((zombies + 1)) + fi + done + test "$zombies" -eq 0 +' + +docker exec "${CONTAINER_NAME}" curl -sS \ + -H 'Content-Type: application/json' \ + -d '{"action":"timeout-tree","rerun":true}' \ + http://127.0.0.1:3501/v1.0/action >/dev/null +sleep 2.5 +if docker exec "${CONTAINER_NAME}" test -e /tmp/descendant-survived; then + echo "action timeout did not kill the descendant process" >&2 + exit 1 +fi + +docker stop --time 10 "${CONTAINER_NAME}" >/dev/null +if [[ "$(docker inspect --format '{{.State.ExitCode}}' "${CONTAINER_NAME}")" != "0" ]]; then + echo "kbagent did not exit cleanly after SIGTERM" >&2 + exit 1 +fi + +echo "kbagent PID 1 regression test passed" diff --git a/pkg/controller/component/kbagent.go b/pkg/controller/component/kbagent.go index c6f8c51010d..8a80d772e5d 100644 --- a/pkg/controller/component/kbagent.go +++ b/pkg/controller/component/kbagent.go @@ -41,6 +41,7 @@ import ( const ( kbAgentCommand = "/bin/kbagent" + kbAgentInitCommand = "/bin/tini-static" kbAgentSharedMountPath = "/kubeblocks" kbAgentCommandOnSharedMount = "/kubeblocks/kbagent" @@ -460,7 +461,7 @@ func handleCustomImageNContainerDefined(synthesizedComp *SynthesizedComponent, c initContainer := builder.NewContainerBuilder(kbagent.InitContainerName). SetImage(viper.GetString(constant.KBToolsImage)). SetImagePullPolicy(corev1.PullIfNotPresent). - AddCommands([]string{"cp", "-r", kbAgentCommand, kbAgentSharedMountPath + "/"}...). + AddCommands([]string{"cp", "-r", kbAgentCommand, kbAgentInitCommand, kbAgentSharedMountPath + "/"}...). AddVolumeMounts(sharedVolumeMount). GetObject() synthesizedComp.PodSpec.InitContainers = append(synthesizedComp.PodSpec.InitContainers, *initContainer) diff --git a/pkg/controller/component/kbagent_test.go b/pkg/controller/component/kbagent_test.go index e7b91016c4e..1f3058a4c5f 100644 --- a/pkg/controller/component/kbagent_test.go +++ b/pkg/controller/component/kbagent_test.go @@ -302,6 +302,7 @@ var _ = Describe("kb-agent", func() { ic := kbAgentInitContainer() Expect(ic).ShouldNot(BeNil()) + Expect(ic.Command).Should(Equal([]string{"cp", "-r", kbAgentCommand, kbAgentInitCommand, kbAgentSharedMountPath + "/"})) c := kbAgentContainer() Expect(c).ShouldNot(BeNil()) From 5a5379cf833b2bfdf8d654bd657bd7fd46c2f393 Mon Sep 17 00:00:00 2001 From: Leon Date: Tue, 11 Aug 2026 17:03:42 +0800 Subject: [PATCH 02/10] chore: keep kbagent test out of CI workflow --- .github/workflows/cicd-pull-request.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/cicd-pull-request.yml b/.github/workflows/cicd-pull-request.yml index f81dc03232a..fe1b05814b5 100644 --- a/.github/workflows/cicd-pull-request.yml +++ b/.github/workflows/cicd-pull-request.yml @@ -158,18 +158,6 @@ jobs: DOCKERFILE_PATH: "./docker/Dockerfile-tools" secrets: inherit - test-kbagent-init: - needs: trigger-mode - if: contains(needs.trigger-mode.outputs.trigger-mode, '[docker]') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - uses: docker/setup-buildx-action@v3 - - name: verify kbagent PID 1 behavior - run: make test-kbagent-init - check-dataprotection-image: needs: trigger-mode if: contains(needs.trigger-mode.outputs.trigger-mode, '[docker]') From 900693758ca3755f43110751dff1fd3f7c7bb18d Mon Sep 17 00:00:00 2001 From: Leon Date: Tue, 11 Aug 2026 17:07:19 +0800 Subject: [PATCH 03/10] chore: remove standalone kbagent image check --- Makefile | 4 -- hack/test-kbagent-init.sh | 109 -------------------------------------- 2 files changed, 113 deletions(-) delete mode 100755 hack/test-kbagent-init.sh diff --git a/Makefile b/Makefile index ae7fa18b10b..bea4ec0d5a8 100644 --- a/Makefile +++ b/Makefile @@ -238,10 +238,6 @@ dataprotection: generate build-checks ## Build dataprotection binary. kbagent: generate build-checks $(GO) build -ldflags=${LD_FLAGS} -o bin/kbagent ./cmd/kbagent/main.go -.PHONY: test-kbagent-init -test-kbagent-init: ## Build the tools image and verify PID 1 orphan reaping. - hack/test-kbagent-init.sh - .PHONY: helmhook helmhook: $(GO) build -o bin/helmhook ./cmd/helmhook/main.go diff --git a/hack/test-kbagent-init.sh b/hack/test-kbagent-init.sh deleted file mode 100755 index 25f013ed6bb..00000000000 --- a/hack/test-kbagent-init.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -IMAGE="${1:-}" - -if [[ -z "${IMAGE}" ]]; then - case "$(docker info --format '{{.Architecture}}')" in - amd64|x86_64) PLATFORM="linux/amd64" ;; - arm64|aarch64) PLATFORM="linux/arm64" ;; - *) - echo "unsupported Docker architecture; set KBAGENT_TEST_PLATFORM explicitly" >&2 - exit 1 - ;; - esac - PLATFORM="${KBAGENT_TEST_PLATFORM:-${PLATFORM}}" - IMAGE="test.local/kubeblocks/kubeblocks-tools:kbagent-init-test" - docker buildx build "${ROOT_DIR}" \ - --file "${ROOT_DIR}/docker/Dockerfile-tools" \ - --platform "${PLATFORM}" \ - --build-arg "GOPROXY=${GOPROXY:-https://proxy.golang.org}" \ - --tag "${IMAGE}" \ - --load -fi - -CONTAINER_NAME="kbagent-init-test-$$" -cleanup() { - docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true -} -trap cleanup EXIT - -ACTIONS='[{"name":"orphan","exec":{"command":["/bin/sh","-c","timeout 1 sleep 0.2 /dev/null 2>&1 &"]}},{"name":"timeout-tree","exec":{"command":["/bin/sh","-c","(sleep 2; touch /tmp/descendant-survived) & wait"]},"timeoutSeconds":1}]' - -docker run -d \ - --name "${CONTAINER_NAME}" \ - -e "KB_AGENT_ACTION=${ACTIONS}" \ - "${IMAGE}" \ - /bin/kbagent >/dev/null - -ready=false -for _ in $(seq 1 60); do - if docker exec "${CONTAINER_NAME}" curl -fsS \ - -H 'Content-Type: application/json' \ - -d '{"action":"orphan","rerun":true}' \ - http://127.0.0.1:3501/v1.0/action >/dev/null 2>&1; then - ready=true - break - fi - sleep 0.1 -done -if [[ "${ready}" != "true" ]]; then - echo "kbagent HTTP endpoint did not become ready" >&2 - docker logs "${CONTAINER_NAME}" >&2 - exit 1 -fi - -docker exec "${CONTAINER_NAME}" sh -ec ' - test "$(cat /proc/1/comm)" = "tini-static" - found_agent=false - for status in /proc/[0-9]*/status; do - name=$(awk "\$1 == \"Name:\" { print \$2 }" "$status") - ppid=$(awk "\$1 == \"PPid:\" { print \$2 }" "$status") - if [ "$name" = "kbagent" ] && [ "$ppid" = "1" ]; then - found_agent=true - break - fi - done - test "$found_agent" = true - - i=0 - while [ "$i" -lt 2000 ]; do - curl -fsS -H "Content-Type: application/json" \ - -d "{\"action\":\"orphan\",\"rerun\":true}" \ - http://127.0.0.1:3501/v1.0/action >/dev/null - i=$((i + 1)) - done -' - -sleep 1 -docker exec "${CONTAINER_NAME}" sh -ec ' - zombies=0 - for status in /proc/[0-9]*/status; do - state=$(awk "\$1 == \"State:\" { print \$2 }" "$status") - ppid=$(awk "\$1 == \"PPid:\" { print \$2 }" "$status") - if [ "$state" = "Z" ] && [ "$ppid" = "1" ]; then - zombies=$((zombies + 1)) - fi - done - test "$zombies" -eq 0 -' - -docker exec "${CONTAINER_NAME}" curl -sS \ - -H 'Content-Type: application/json' \ - -d '{"action":"timeout-tree","rerun":true}' \ - http://127.0.0.1:3501/v1.0/action >/dev/null -sleep 2.5 -if docker exec "${CONTAINER_NAME}" test -e /tmp/descendant-survived; then - echo "action timeout did not kill the descendant process" >&2 - exit 1 -fi - -docker stop --time 10 "${CONTAINER_NAME}" >/dev/null -if [[ "$(docker inspect --format '{{.State.ExitCode}}' "${CONTAINER_NAME}")" != "0" ]]; then - echo "kbagent did not exit cleanly after SIGTERM" >&2 - exit 1 -fi - -echo "kbagent PID 1 regression test passed" From 37335a6bb457f7af25da67fbcc74f20ad0595c5c Mon Sep 17 00:00:00 2001 From: Leon Date: Tue, 11 Aug 2026 17:43:38 +0800 Subject: [PATCH 04/10] fix: defer custom kbagent init migration --- .../instanceset/in_place_update_util.go | 73 +++++++++++++++++++ .../instanceset/in_place_update_util_test.go | 68 +++++++++++++++++ .../instanceset/reconciler_update.go | 1 + .../instanceset/reconciler_update_test.go | 73 +++++++++++++++++++ 4 files changed, 215 insertions(+) diff --git a/pkg/controller/instanceset/in_place_update_util.go b/pkg/controller/instanceset/in_place_update_util.go index d37f27393b7..6c8b26a7f50 100644 --- a/pkg/controller/instanceset/in_place_update_util.go +++ b/pkg/controller/instanceset/in_place_update_util.go @@ -35,6 +35,7 @@ import ( "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/controller/instancetemplate" intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" + "github.com/apecloud/kubeblocks/pkg/kbagent" viper "github.com/apecloud/kubeblocks/pkg/viperx" ) @@ -48,8 +49,13 @@ const ( var ( errTemplateNotFound = fmt.Errorf("no template found for pod") + + legacyKBAgentInitCopyCommand = []string{"cp", "-r", "/bin/kbagent", "/kubeblocks/"} + kbAgentInitCopyCommand = []string{"cp", "-r", "/bin/kbagent", "/bin/tini-static", "/kubeblocks/"} ) +const kbAgentCommandOnSharedMount = "/kubeblocks/kbagent" + func supportPodVerticalScaling() bool { return viper.GetBool(constant.FeatureGateInPlacePodVerticalScaling) } @@ -84,6 +90,7 @@ func filterInPlaceFields(src *corev1.PodTemplateSpec) *corev1.PodTemplateSpec { for i := range template.Spec.InitContainers { template.Spec.InitContainers[i].Image = "" } + normalizeKBAgentInitCommandForRevision(template) // filter spec.activeDeadlineSeconds template.Spec.ActiveDeadlineSeconds = nil // filter spec.tolerations @@ -99,6 +106,71 @@ func filterInPlaceFields(src *corev1.PodTemplateSpec) *corev1.PodTemplateSpec { return template } +// normalizeKBAgentInitCommandForRevision keeps the one-time addition of tini-static +// from changing existing custom-image Pod revisions. The actual InstanceSet template +// remains untouched, so every newly created Pod uses the new copy command. +func normalizeKBAgentInitCommandForRevision(template *corev1.PodTemplateSpec) { + if template == nil || !hasSharedKBAgentCommand(template.Spec.Containers) { + return + } + for i := range template.Spec.InitContainers { + initContainer := &template.Spec.InitContainers[i] + if initContainer.Name == kbagent.InitContainerName && reflect.DeepEqual(initContainer.Command, kbAgentInitCopyCommand) { + initContainer.Command = append([]string(nil), legacyKBAgentInitCopyCommand...) + return + } + } +} + +func hasSharedKBAgentCommand(containers []corev1.Container) bool { + index := slices.IndexFunc(containers, func(container corev1.Container) bool { + return container.Name == kbagent.ContainerName + }) + return index >= 0 && reflect.DeepEqual(containers[index].Command, []string{kbAgentCommandOnSharedMount}) +} + +// podForDeferredKBAgentInitMigration returns a desired Pod view that retains the +// live init-kbagent image and command. This lets existing custom-image Pods keep +// running without applying an immutable init command change, while the real +// InstanceSet template stays on the new image and two-file copy command. +func podForDeferredKBAgentInitMigration(old, desired *corev1.Pod) (*corev1.Pod, bool) { + if old == nil || desired == nil || + !hasSharedKBAgentCommand(old.Spec.Containers) || !hasSharedKBAgentCommand(desired.Spec.Containers) { + return desired, false + } + oldIndex := slices.IndexFunc(old.Spec.InitContainers, func(container corev1.Container) bool { + return container.Name == kbagent.InitContainerName + }) + desiredIndex := slices.IndexFunc(desired.Spec.InitContainers, func(container corev1.Container) bool { + return container.Name == kbagent.InitContainerName + }) + if oldIndex < 0 || desiredIndex < 0 { + return desired, false + } + oldInit := &old.Spec.InitContainers[oldIndex] + desiredInit := &desired.Spec.InitContainers[desiredIndex] + if !reflect.DeepEqual(oldInit.Command, legacyKBAgentInitCopyCommand) || + !reflect.DeepEqual(desiredInit.Command, kbAgentInitCopyCommand) { + return desired, false + } + + // If the image also changed, it must be a KB-managed tools image update. + // Normalize the command first so the existing image-only classifier can + // validate that no application image is being deferred. + if !intctrlutil.EqualContainerImageInSpec(oldInit.Image, desiredInit.Image) { + commandNormalized := desired.DeepCopy() + commandNormalized.Spec.InitContainers[desiredIndex].Command = append([]string(nil), oldInit.Command...) + if !intctrlutil.OnlyKBManagedPodImagesChanged(old, commandNormalized) { + return desired, false + } + } + + normalized := desired.DeepCopy() + normalized.Spec.InitContainers[desiredIndex].Image = oldInit.Image + normalized.Spec.InitContainers[desiredIndex].Command = append([]string(nil), oldInit.Command...) + return normalized, true +} + func copyRequestsNLimitsFields(container *corev1.Container) (corev1.ResourceList, corev1.ResourceList) { requests := make(corev1.ResourceList) limits := make(corev1.ResourceList) @@ -421,6 +493,7 @@ func getPodUpdatePolicy(its *workloads.InstanceSet, pod *corev1.Pod) (podUpdateP if err != nil { return noOpsPolicy, "", err } + newPod, _ = podForDeferredKBAgentInitMigration(pod, newPod) specUpdatePolicy := getPodUpdatePolicyInSpec(its, pod, newPod) if getPodRevision(pod) != updateRevisions[pod.Name] && getPodRevision(pod) != proposedRevisions[pod.Name] { diff --git a/pkg/controller/instanceset/in_place_update_util_test.go b/pkg/controller/instanceset/in_place_update_util_test.go index d4a4b6f6103..8eaee0f74c1 100644 --- a/pkg/controller/instanceset/in_place_update_util_test.go +++ b/pkg/controller/instanceset/in_place_update_util_test.go @@ -70,6 +70,36 @@ var _ = Describe("instance util test", func() { Expect(result.Spec.Containers[0].Resources.Limits).ShouldNot(HaveKey(corev1.ResourceCPU)) Expect(result.Spec.Containers[0].Resources.Limits).ShouldNot(HaveKey(corev1.ResourceMemory)) }) + + It("keeps the custom-image tini copy migration revision-compatible", func() { + oldTemplate := &corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{ + Name: "init-kbagent", + Image: "docker.io/apecloud/kubeblocks-tools:1.0.0", + Command: append([]string(nil), legacyKBAgentInitCopyCommand...), + }}, + Containers: []corev1.Container{ + {Name: "app", Image: "mysql:8.0"}, + {Name: "kbagent", Image: "custom-action:1.0", Command: []string{"/kubeblocks/kbagent"}}, + }, + }} + newTemplate := oldTemplate.DeepCopy() + newTemplate.Spec.InitContainers[0].Image = "mirror.local/apecloud/kubeblocks-tools:1.1.0" + newTemplate.Spec.InitContainers[0].Command = append([]string(nil), kbAgentInitCopyCommand...) + + oldFiltered := filterInPlaceFields(oldTemplate) + newFiltered := filterInPlaceFields(newTemplate) + Expect(newFiltered).Should(Equal(oldFiltered)) + Expect(newTemplate.Spec.InitContainers[0].Command).Should(Equal(kbAgentInitCopyCommand), + "revision normalization must not modify the desired template") + + parent := builder.NewInstanceSetBuilder(namespace, name).SetUID(uid).GetObject() + oldRevision, err := buildInstanceTemplateRevision(oldTemplate, parent, nil) + Expect(err).ShouldNot(HaveOccurred()) + newRevision, err := buildInstanceTemplateRevision(newTemplate, parent, nil) + Expect(err).ShouldNot(HaveOccurred()) + Expect(newRevision).Should(Equal(oldRevision)) + }) }) Context("mergeInPlaceFields & equalXFields", func() { @@ -96,6 +126,44 @@ var _ = Describe("instance util test", func() { }) Context("container image comparison", func() { + It("defers the custom-image init command and tools image as one pair", func() { + oldToolsImage := viper.GetString(constant.KBToolsImage) + defer viper.Set(constant.KBToolsImage, oldToolsImage) + viper.Set(constant.KBToolsImage, "docker.io/apecloud/kubeblocks-tools:1.0.0") + + oldPod := &corev1.Pod{Spec: corev1.PodSpec{ + InitContainers: []corev1.Container{{ + Name: "init-kbagent", + Image: "docker.io/apecloud/kubeblocks-tools:1.0.0", + Command: append([]string(nil), legacyKBAgentInitCopyCommand...), + }}, + Containers: []corev1.Container{ + {Name: "app", Image: "mysql:8.0"}, + {Name: "kbagent", Image: "custom-action:1.0", Command: []string{"/kubeblocks/kbagent"}}, + }, + }} + desiredPod := oldPod.DeepCopy() + desiredPod.Spec.InitContainers[0].Image = "mirror.local/apecloud/kubeblocks-tools:1.1.0" + desiredPod.Spec.InitContainers[0].Command = append([]string(nil), kbAgentInitCopyCommand...) + + normalized, ok := podForDeferredKBAgentInitMigration(oldPod, desiredPod) + Expect(ok).Should(BeTrue()) + Expect(normalized.Spec.InitContainers[0].Image).Should(Equal(oldPod.Spec.InitContainers[0].Image)) + Expect(normalized.Spec.InitContainers[0].Command).Should(Equal(legacyKBAgentInitCopyCommand)) + Expect(desiredPod.Spec.InitContainers[0].Image).Should(Equal("mirror.local/apecloud/kubeblocks-tools:1.1.0")) + Expect(desiredPod.Spec.InitContainers[0].Command).Should(Equal(kbAgentInitCopyCommand)) + + appUpgrade := desiredPod.DeepCopy() + appUpgrade.Spec.Containers[0].Image = "mysql:8.4" + _, ok = podForDeferredKBAgentInitMigration(oldPod, appUpgrade) + Expect(ok).Should(BeFalse(), "application image changes must not be deferred") + + defaultImagePod := desiredPod.DeepCopy() + defaultImagePod.Spec.Containers[1].Command = []string{"/bin/kbagent"} + _, ok = podForDeferredKBAgentInitMigration(oldPod, defaultImagePod) + Expect(ok).Should(BeFalse(), "the compatibility path is only for custom-image workloads") + }) + It("ignores registry rewrites but keeps tag, digest, and basename strict", func() { oldPod := buildRandomPod() newPod := oldPod.DeepCopy() diff --git a/pkg/controller/instanceset/reconciler_update.go b/pkg/controller/instanceset/reconciler_update.go index 008261d4dc5..8135b82c177 100644 --- a/pkg/controller/instanceset/reconciler_update.go +++ b/pkg/controller/instanceset/reconciler_update.go @@ -179,6 +179,7 @@ func (r *updateReconciler) Reconcile(tree *kubebuilderx.ObjectTree) (kubebuilder if err != nil { return kubebuilderx.Continue, err } + newPod, _ = podForDeferredKBAgentInitMigration(pod, newPod) newMergedPod := copyAndMerge(pod, newPod) supportResizeSubResource, err := intctrlutil.SupportResizeSubResource() if err != nil { diff --git a/pkg/controller/instanceset/reconciler_update_test.go b/pkg/controller/instanceset/reconciler_update_test.go index 4f63e75e099..85ef466f6be 100644 --- a/pkg/controller/instanceset/reconciler_update_test.go +++ b/pkg/controller/instanceset/reconciler_update_test.go @@ -39,6 +39,7 @@ import ( workloads "github.com/apecloud/kubeblocks/apis/workloads/v1" "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/controller/builder" + "github.com/apecloud/kubeblocks/pkg/controller/instancetemplate" "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" "github.com/apecloud/kubeblocks/pkg/controller/lifecycle" intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" @@ -653,6 +654,78 @@ var _ = Describe("update reconciler test", func() { "switchover must not be invoked when only KB-managed tools images differ") }) + It("defers the custom-image init migration only for existing pods", func() { + oldToolsImage := viper.GetString(constant.KBToolsImage) + defer viper.Set(constant.KBToolsImage, oldToolsImage) + viper.Set(constant.KBToolsImage, "mirror.local/apecloud/kubeblocks-tools:1.1.0") + + its.Spec.PodManagementPolicy = appsv1.ParallelPodManagement + its.Spec.Replicas = ptr.To[int32](1) + its.Spec.PodUpdatePolicy = kbappsv1.ReCreatePodUpdatePolicyType + its.Spec.PodUpgradePolicy = kbappsv1.ReCreatePodUpdatePolicyType + its.Spec.Template.Spec.InitContainers = []corev1.Container{{ + Name: "init-kbagent", + Image: "docker.io/apecloud/kubeblocks-tools:1.0.0", + Command: append([]string(nil), legacyKBAgentInitCopyCommand...), + }} + its.Spec.Template.Spec.Containers = []corev1.Container{ + {Name: "app", Image: "mysql:8.0"}, + {Name: "kbagent", Image: "custom-action:1.0", Command: []string{"/kubeblocks/kbagent"}}, + } + + tree := kubebuilderx.NewObjectTree() + tree.SetRoot(its) + prepareForUpdate(tree) + pods := tree.List(&corev1.Pod{}) + Expect(pods).Should(HaveLen(1)) + pod := pods[0].(*corev1.Pod) + pod.UID = "custom-image-pod-uid" + pod.Status.Phase = corev1.PodRunning + pod.Status.Conditions = append(pod.Status.Conditions, corev1.PodCondition{ + Type: corev1.PodReady, + Status: corev1.ConditionTrue, + LastTransitionTime: metav1.NewTime(time.Now().Add(-1 * minReadySeconds * time.Second)), + }) + + its.Spec.Template.Spec.InitContainers[0].Image = "mirror.local/apecloud/kubeblocks-tools:1.1.0" + its.Spec.Template.Spec.InitContainers[0].Command = append([]string(nil), kbAgentInitCopyCommand...) + Expect(its.Spec.Template.Spec.InitContainers[0].Command).Should(Equal(kbAgentInitCopyCommand), + "the desired template must retain the new command") + + reconciler := NewUpdateReconciler() + res, err := reconciler.Reconcile(tree) + Expect(err).Should(BeNil()) + Expect(res).Should(Equal(kubebuilderx.Continue)) + + postPods := tree.List(&corev1.Pod{}) + Expect(postPods).Should(HaveLen(1)) + livePod := postPods[0].(*corev1.Pod) + Expect(livePod.UID).Should(Equal(pod.UID), "the existing Pod must not be recreated") + Expect(livePod.Spec.InitContainers[0].Image).Should(Equal("docker.io/apecloud/kubeblocks-tools:1.0.0")) + Expect(livePod.Spec.InitContainers[0].Command).Should(Equal(legacyKBAgentInitCopyCommand), + "the existing Pod must keep the old image and command as one pair") + + itsExt, err := instancetemplate.BuildInstanceSetExt(its, nil) + Expect(err).ShouldNot(HaveOccurred()) + templates := instancetemplate.BuildInstanceTemplateExt(itsExt) + Expect(templates).Should(HaveLen(1)) + replacement, err := buildInstancePodByTemplate(pod.Name, templates[0], its, "") + Expect(err).ShouldNot(HaveOccurred()) + Expect(replacement.Spec.InitContainers[0].Image).Should(Equal("mirror.local/apecloud/kubeblocks-tools:1.1.0")) + Expect(replacement.Spec.InitContainers[0].Command).Should(Equal(kbAgentInitCopyCommand), + "a recreated Pod must copy both kbagent and tini-static") + + if its.Spec.Template.Annotations == nil { + its.Spec.Template.Annotations = map[string]string{} + } + its.Spec.Template.Annotations[constant.RestartAnnotationKey] = "2026-08-11T12:00:00Z" + _, err = NewRevisionUpdateReconciler().Reconcile(tree) + Expect(err).ShouldNot(HaveOccurred()) + policy, _, _, err := getPodUpdatePolicy(its, livePod) + Expect(err).ShouldNot(HaveOccurred()) + Expect(policy).Should(Equal(recreatePolicy), "an explicit restart must still recreate the Pod") + }) + DescribeTable("keeps recreation policy for revision and application image changes", func(mutateTemplate func(*workloads.InstanceSet)) { oldToolsImage := viper.GetString(constant.KBToolsImage) From 03b3eee5ede41ea2d93c7d0bf4978d049b506cc0 Mon Sep 17 00:00:00 2001 From: Leon Date: Wed, 12 Aug 2026 10:39:11 +0800 Subject: [PATCH 05/10] fix: keep kbagent init pair during app upgrade --- .../instanceset/in_place_update_util.go | 15 +++++++++------ .../instanceset/in_place_update_util_test.go | 8 ++++++-- .../instanceset/reconciler_update_test.go | 10 ++++++++-- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/pkg/controller/instanceset/in_place_update_util.go b/pkg/controller/instanceset/in_place_update_util.go index 6c8b26a7f50..1cd23569f81 100644 --- a/pkg/controller/instanceset/in_place_update_util.go +++ b/pkg/controller/instanceset/in_place_update_util.go @@ -154,13 +154,16 @@ func podForDeferredKBAgentInitMigration(old, desired *corev1.Pod) (*corev1.Pod, return desired, false } - // If the image also changed, it must be a KB-managed tools image update. - // Normalize the command first so the existing image-only classifier can - // validate that no application image is being deferred. + // If the image also changed, validate the init-kbagent image transition in + // isolation. Other containers may have legitimate image updates of their own; + // they must not prevent this init image and command from being retained as a + // pair on the live Pod. if !intctrlutil.EqualContainerImageInSpec(oldInit.Image, desiredInit.Image) { - commandNormalized := desired.DeepCopy() - commandNormalized.Spec.InitContainers[desiredIndex].Command = append([]string(nil), oldInit.Command...) - if !intctrlutil.OnlyKBManagedPodImagesChanged(old, commandNormalized) { + oldInitPod := &corev1.Pod{Spec: corev1.PodSpec{InitContainers: []corev1.Container{*oldInit.DeepCopy()}}} + desiredInitCopy := desiredInit.DeepCopy() + desiredInitCopy.Command = append([]string(nil), oldInit.Command...) + desiredInitPod := &corev1.Pod{Spec: corev1.PodSpec{InitContainers: []corev1.Container{*desiredInitCopy}}} + if !intctrlutil.OnlyKBManagedPodImagesChanged(oldInitPod, desiredInitPod) { return desired, false } } diff --git a/pkg/controller/instanceset/in_place_update_util_test.go b/pkg/controller/instanceset/in_place_update_util_test.go index 8eaee0f74c1..b4388dba9a0 100644 --- a/pkg/controller/instanceset/in_place_update_util_test.go +++ b/pkg/controller/instanceset/in_place_update_util_test.go @@ -155,8 +155,12 @@ var _ = Describe("instance util test", func() { appUpgrade := desiredPod.DeepCopy() appUpgrade.Spec.Containers[0].Image = "mysql:8.4" - _, ok = podForDeferredKBAgentInitMigration(oldPod, appUpgrade) - Expect(ok).Should(BeFalse(), "application image changes must not be deferred") + normalized, ok = podForDeferredKBAgentInitMigration(oldPod, appUpgrade) + Expect(ok).Should(BeTrue(), "application image changes must not prevent deferring the init pair") + Expect(normalized.Spec.InitContainers[0].Image).Should(Equal(oldPod.Spec.InitContainers[0].Image)) + Expect(normalized.Spec.InitContainers[0].Command).Should(Equal(legacyKBAgentInitCopyCommand)) + Expect(normalized.Spec.Containers[0].Image).Should(Equal("mysql:8.4"), + "the application image must remain at its desired value") defaultImagePod := desiredPod.DeepCopy() defaultImagePod.Spec.Containers[1].Command = []string{"/bin/kbagent"} diff --git a/pkg/controller/instanceset/reconciler_update_test.go b/pkg/controller/instanceset/reconciler_update_test.go index 85ef466f6be..2f423596eb0 100644 --- a/pkg/controller/instanceset/reconciler_update_test.go +++ b/pkg/controller/instanceset/reconciler_update_test.go @@ -654,15 +654,18 @@ var _ = Describe("update reconciler test", func() { "switchover must not be invoked when only KB-managed tools images differ") }) - It("defers the custom-image init migration only for existing pods", func() { + It("defers the custom-image init pair while updating the application image in place", func() { oldToolsImage := viper.GetString(constant.KBToolsImage) defer viper.Set(constant.KBToolsImage, oldToolsImage) viper.Set(constant.KBToolsImage, "mirror.local/apecloud/kubeblocks-tools:1.1.0") + origSupportResize := intctrlutil.SupportResizeSubResource + intctrlutil.SupportResizeSubResource = func() (bool, error) { return false, nil } + defer func() { intctrlutil.SupportResizeSubResource = origSupportResize }() its.Spec.PodManagementPolicy = appsv1.ParallelPodManagement its.Spec.Replicas = ptr.To[int32](1) its.Spec.PodUpdatePolicy = kbappsv1.ReCreatePodUpdatePolicyType - its.Spec.PodUpgradePolicy = kbappsv1.ReCreatePodUpdatePolicyType + its.Spec.PodUpgradePolicy = kbappsv1.PreferInPlacePodUpdatePolicyType its.Spec.Template.Spec.InitContainers = []corev1.Container{{ Name: "init-kbagent", Image: "docker.io/apecloud/kubeblocks-tools:1.0.0", @@ -689,6 +692,7 @@ var _ = Describe("update reconciler test", func() { its.Spec.Template.Spec.InitContainers[0].Image = "mirror.local/apecloud/kubeblocks-tools:1.1.0" its.Spec.Template.Spec.InitContainers[0].Command = append([]string(nil), kbAgentInitCopyCommand...) + its.Spec.Template.Spec.Containers[0].Image = "mysql:8.4" Expect(its.Spec.Template.Spec.InitContainers[0].Command).Should(Equal(kbAgentInitCopyCommand), "the desired template must retain the new command") @@ -704,6 +708,8 @@ var _ = Describe("update reconciler test", func() { Expect(livePod.Spec.InitContainers[0].Image).Should(Equal("docker.io/apecloud/kubeblocks-tools:1.0.0")) Expect(livePod.Spec.InitContainers[0].Command).Should(Equal(legacyKBAgentInitCopyCommand), "the existing Pod must keep the old image and command as one pair") + Expect(livePod.Spec.Containers[0].Image).Should(Equal("mysql:8.4"), + "the application image must still be updated in place") itsExt, err := instancetemplate.BuildInstanceSetExt(its, nil) Expect(err).ShouldNot(HaveOccurred()) From 9415d90296a7307674c5e58bc393f6c71375e1cd Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 13 Aug 2026 11:20:28 +0800 Subject: [PATCH 06/10] refactor: centralize kbagent init migration --- pkg/controller/instanceset/in_place_update_util.go | 3 +-- pkg/controller/instanceset/instance_util.go | 13 +++++++++++++ pkg/controller/instanceset/reconciler_update.go | 3 +-- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/pkg/controller/instanceset/in_place_update_util.go b/pkg/controller/instanceset/in_place_update_util.go index 1cd23569f81..957df5e19d7 100644 --- a/pkg/controller/instanceset/in_place_update_util.go +++ b/pkg/controller/instanceset/in_place_update_util.go @@ -492,11 +492,10 @@ func getPodUpdatePolicy(its *workloads.InstanceSet, pod *corev1.Pod) (podUpdateP if index < 0 { return noOpsPolicy, "", errors.Wrapf(errTemplateNotFound, "pod: %s/%s", pod.Namespace, pod.Name) } - newPod, err := buildInstancePodByTemplate(pod.Name, templateList[index], its, getPodRevision(pod)) + newPod, err := buildInstancePodByTemplateForUpdate(pod, templateList[index], its) if err != nil { return noOpsPolicy, "", err } - newPod, _ = podForDeferredKBAgentInitMigration(pod, newPod) specUpdatePolicy := getPodUpdatePolicyInSpec(its, pod, newPod) if getPodRevision(pod) != updateRevisions[pod.Name] && getPodRevision(pod) != proposedRevisions[pod.Name] { diff --git a/pkg/controller/instanceset/instance_util.go b/pkg/controller/instanceset/instance_util.go index c7b4d0e8188..ea167e0885a 100644 --- a/pkg/controller/instanceset/instance_util.go +++ b/pkg/controller/instanceset/instance_util.go @@ -419,6 +419,19 @@ func buildInstancePodByTemplate(name string, template *instancetemplate.Instance return pod, nil } +// buildInstancePodByTemplateForUpdate builds the desired Pod view used to +// evaluate and apply updates to an existing Pod. Compatibility adjustments are +// intentionally kept out of buildInstancePodByTemplate so newly created and +// replacement Pods always use the latest template. +func buildInstancePodByTemplateForUpdate(oldPod *corev1.Pod, template *instancetemplate.InstanceTemplateExt, parent *workloads.InstanceSet) (*corev1.Pod, error) { + desiredPod, err := buildInstancePodByTemplate(oldPod.Name, template, parent, getPodRevision(oldPod)) + if err != nil { + return nil, err + } + desiredPod, _ = podForDeferredKBAgentInitMigration(oldPod, desiredPod) + return desiredPod, nil +} + func buildInstancePVCByTemplate(name string, template *instancetemplate.InstanceTemplateExt, parent *workloads.InstanceSet) ([]*corev1.PersistentVolumeClaim, error) { var pvcs []*corev1.PersistentVolumeClaim labels := getMatchLabels(parent.Name) diff --git a/pkg/controller/instanceset/reconciler_update.go b/pkg/controller/instanceset/reconciler_update.go index 8135b82c177..bb983deb247 100644 --- a/pkg/controller/instanceset/reconciler_update.go +++ b/pkg/controller/instanceset/reconciler_update.go @@ -175,11 +175,10 @@ func (r *updateReconciler) Reconcile(tree *kubebuilderx.ObjectTree) (kubebuilder } switch updatePolicy { case inPlaceUpdatePolicy: - newPod, err := buildInstancePodByTemplate(pod.Name, nameToTemplateMap[pod.Name], its, getPodRevision(pod)) + newPod, err := buildInstancePodByTemplateForUpdate(pod, nameToTemplateMap[pod.Name], its) if err != nil { return kubebuilderx.Continue, err } - newPod, _ = podForDeferredKBAgentInitMigration(pod, newPod) newMergedPod := copyAndMerge(pod, newPod) supportResizeSubResource, err := intctrlutil.SupportResizeSubResource() if err != nil { From ed6a140886bfdb21259b7c8f9bd65cc80f056e34 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 13 Aug 2026 11:33:07 +0800 Subject: [PATCH 07/10] refactor: apply kbagent revision normalization last --- pkg/controller/instanceset/in_place_update_util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/instanceset/in_place_update_util.go b/pkg/controller/instanceset/in_place_update_util.go index 957df5e19d7..31297955945 100644 --- a/pkg/controller/instanceset/in_place_update_util.go +++ b/pkg/controller/instanceset/in_place_update_util.go @@ -90,7 +90,6 @@ func filterInPlaceFields(src *corev1.PodTemplateSpec) *corev1.PodTemplateSpec { for i := range template.Spec.InitContainers { template.Spec.InitContainers[i].Image = "" } - normalizeKBAgentInitCommandForRevision(template) // filter spec.activeDeadlineSeconds template.Spec.ActiveDeadlineSeconds = nil // filter spec.tolerations @@ -102,6 +101,7 @@ func filterInPlaceFields(src *corev1.PodTemplateSpec) *corev1.PodTemplateSpec { delete(template.Spec.Containers[i].Resources.Limits, corev1.ResourceCPU) delete(template.Spec.Containers[i].Resources.Limits, corev1.ResourceMemory) } + normalizeKBAgentInitCommandForRevision(template) return template } From bf662dcde8a4d67a56d491b46501b316d9b30481 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 13 Aug 2026 11:46:59 +0800 Subject: [PATCH 08/10] refactor: centralize kbagent pod contract --- .../transformer_component_workload.go | 3 +- pkg/controller/component/kbagent.go | 24 ++---- pkg/controller/component/kbagent_test.go | 16 ++-- .../instanceset/in_place_update_util.go | 28 ++----- .../instanceset/in_place_update_util_test.go | 25 +++--- .../instanceset/reconciler_update_test.go | 17 ++-- pkg/kbagent/podspec.go | 77 +++++++++++++++++++ pkg/kbagent/podspec_test.go | 69 +++++++++++++++++ 8 files changed, 191 insertions(+), 68 deletions(-) create mode 100644 pkg/kbagent/podspec.go create mode 100644 pkg/kbagent/podspec_test.go diff --git a/controllers/apps/component/transformer_component_workload.go b/controllers/apps/component/transformer_component_workload.go index 57970aacd1f..65a16594aca 100644 --- a/controllers/apps/component/transformer_component_workload.go +++ b/controllers/apps/component/transformer_component_workload.go @@ -38,6 +38,7 @@ import ( "github.com/apecloud/kubeblocks/pkg/controller/graph" "github.com/apecloud/kubeblocks/pkg/controller/model" intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" + "github.com/apecloud/kubeblocks/pkg/kbagent" ) // componentWorkloadTransformer handles component workload generation @@ -327,7 +328,7 @@ func checkNRollbackProtoImages(itsObj, itsProto *workloads.InstanceSet) { images[i] = make(map[string]string) for _, c := range cc { // skip the kb-agent container - if component.IsKBAgentContainer(&c) { + if kbagent.IsContainer(&c) { continue } images[i][c.Name] = c.Image diff --git a/pkg/controller/component/kbagent.go b/pkg/controller/component/kbagent.go index 8a80d772e5d..1f4fa2745ba 100644 --- a/pkg/controller/component/kbagent.go +++ b/pkg/controller/component/kbagent.go @@ -40,11 +40,6 @@ import ( ) const ( - kbAgentCommand = "/bin/kbagent" - kbAgentInitCommand = "/bin/tini-static" - kbAgentSharedMountPath = "/kubeblocks" - kbAgentCommandOnSharedMount = "/kubeblocks/kbagent" - minAvailablePort = 1025 maxAvailablePort = 65535 @@ -56,14 +51,7 @@ const ( podRoleLabelFileName = "role" ) -var ( - sharedVolumeMount = corev1.VolumeMount{Name: "kubeblocks", MountPath: kbAgentSharedMountPath} - roleLabelVolumeMount = corev1.VolumeMount{Name: roleLabelVolumeName, MountPath: podMetadataMountPath, ReadOnly: true} -) - -func IsKBAgentContainer(c *corev1.Container) bool { - return c.Name == kbagent.ContainerName || c.Name == kbagent.ContainerName4Worker || c.Name == kbagent.InitContainerName -} +var roleLabelVolumeMount = corev1.VolumeMount{Name: roleLabelVolumeName, MountPath: podMetadataMountPath, ReadOnly: true} func UpdateKBAgentContainer4HostNetwork(synthesizedComp *SynthesizedComponent) { idx, c := intctrlutil.GetContainerByName(synthesizedComp.PodSpec.Containers, kbagent.ContainerName) @@ -140,7 +128,7 @@ func buildKBAgentContainer(synthesizedComp *SynthesizedComponent) error { b := builder.NewContainerBuilder(name). SetImage(viper.GetString(constant.KBToolsImage)). SetImagePullPolicy(corev1.PullIfNotPresent). - AddCommands(kbAgentCommand). + AddCommands(kbagent.BinaryPath). AddEnv(mergedActionEnv4KBAgent(synthesizedComp)...). AddEnv(envVars...). SetSecurityContext(corev1.SecurityContext{ @@ -461,15 +449,15 @@ func handleCustomImageNContainerDefined(synthesizedComp *SynthesizedComponent, c initContainer := builder.NewContainerBuilder(kbagent.InitContainerName). SetImage(viper.GetString(constant.KBToolsImage)). SetImagePullPolicy(corev1.PullIfNotPresent). - AddCommands([]string{"cp", "-r", kbAgentCommand, kbAgentInitCommand, kbAgentSharedMountPath + "/"}...). - AddVolumeMounts(sharedVolumeMount). + AddCommands(kbagent.CurrentInitCopyCommand()...). + AddVolumeMounts(kbagent.SharedVolumeMount()). GetObject() synthesizedComp.PodSpec.InitContainers = append(synthesizedComp.PodSpec.InitContainers, *initContainer) for _, container := range containers { container.Image = image - container.Command[0] = kbAgentCommandOnSharedMount - container.VolumeMounts = append(container.VolumeMounts, sharedVolumeMount) + container.Command[0] = kbagent.SharedBinaryPath + container.VolumeMounts = append(container.VolumeMounts, kbagent.SharedVolumeMount()) } } diff --git a/pkg/controller/component/kbagent_test.go b/pkg/controller/component/kbagent_test.go index 1f3058a4c5f..f992e82765c 100644 --- a/pkg/controller/component/kbagent_test.go +++ b/pkg/controller/component/kbagent_test.go @@ -302,14 +302,14 @@ var _ = Describe("kb-agent", func() { ic := kbAgentInitContainer() Expect(ic).ShouldNot(BeNil()) - Expect(ic.Command).Should(Equal([]string{"cp", "-r", kbAgentCommand, kbAgentInitCommand, kbAgentSharedMountPath + "/"})) + Expect(ic.Command).Should(Equal(kbagent.CurrentInitCopyCommand())) c := kbAgentContainer() Expect(c).ShouldNot(BeNil()) Expect(c.Image).Should(Equal(image)) - Expect(c.Command[0]).Should(Equal(kbAgentCommandOnSharedMount)) + Expect(c.Command[0]).Should(Equal(kbagent.SharedBinaryPath)) Expect(c.VolumeMounts).Should(HaveLen(2)) - Expect(c.VolumeMounts[0]).Should(Equal(sharedVolumeMount)) + Expect(c.VolumeMounts[0]).Should(Equal(kbagent.SharedVolumeMount())) Expect(c.VolumeMounts[1]).Should(Equal(roleLabelVolumeMount)) }) @@ -346,7 +346,7 @@ var _ = Describe("kb-agent", func() { c := kbAgentContainer() Expect(c).ShouldNot(BeNil()) Expect(c.Image).Should(Equal(viperx.GetString(constant.KBToolsImage))) - Expect(c.Command[0]).Should(Equal(kbAgentCommand)) + Expect(c.Command[0]).Should(Equal(kbagent.BinaryPath)) Expect(c.VolumeMounts).Should(HaveLen(1)) Expect(c.VolumeMounts[0]).Should(Equal(roleLabelVolumeMount)) }) @@ -424,9 +424,9 @@ var _ = Describe("kb-agent", func() { Expect(c).ShouldNot(BeNil()) Expect(c).ShouldNot(BeNil()) Expect(c.Image).Should(Equal(container.Image)) - Expect(c.Command[0]).Should(Equal(kbAgentCommandOnSharedMount)) + Expect(c.Command[0]).Should(Equal(kbagent.SharedBinaryPath)) Expect(c.VolumeMounts).Should(HaveLen(3)) - Expect(c.VolumeMounts[0]).Should(Equal(sharedVolumeMount)) + Expect(c.VolumeMounts[0]).Should(Equal(kbagent.SharedVolumeMount())) Expect(c.VolumeMounts[1]).Should(Equal(container.VolumeMounts[0])) Expect(c.VolumeMounts[2]).Should(Equal(roleLabelVolumeMount)) }) @@ -452,9 +452,9 @@ var _ = Describe("kb-agent", func() { c := kbAgentContainer() Expect(c.Image).Should(Equal(image)) - Expect(c.Command[0]).Should(Equal(kbAgentCommandOnSharedMount)) + Expect(c.Command[0]).Should(Equal(kbagent.SharedBinaryPath)) Expect(c.VolumeMounts).Should(HaveLen(3)) - Expect(c.VolumeMounts[0]).Should(Equal(sharedVolumeMount)) + Expect(c.VolumeMounts[0]).Should(Equal(kbagent.SharedVolumeMount())) Expect(c.VolumeMounts[1]).Should(Equal(container.VolumeMounts[0])) Expect(c.VolumeMounts[2]).Should(Equal(roleLabelVolumeMount)) }) diff --git a/pkg/controller/instanceset/in_place_update_util.go b/pkg/controller/instanceset/in_place_update_util.go index 31297955945..914396f1e4c 100644 --- a/pkg/controller/instanceset/in_place_update_util.go +++ b/pkg/controller/instanceset/in_place_update_util.go @@ -47,14 +47,7 @@ const ( inPlaceUpdatePolicy podUpdatePolicy = "inPlaceUpdate" ) -var ( - errTemplateNotFound = fmt.Errorf("no template found for pod") - - legacyKBAgentInitCopyCommand = []string{"cp", "-r", "/bin/kbagent", "/kubeblocks/"} - kbAgentInitCopyCommand = []string{"cp", "-r", "/bin/kbagent", "/bin/tini-static", "/kubeblocks/"} -) - -const kbAgentCommandOnSharedMount = "/kubeblocks/kbagent" +var errTemplateNotFound = fmt.Errorf("no template found for pod") func supportPodVerticalScaling() bool { return viper.GetBool(constant.FeatureGateInPlacePodVerticalScaling) @@ -110,32 +103,25 @@ func filterInPlaceFields(src *corev1.PodTemplateSpec) *corev1.PodTemplateSpec { // from changing existing custom-image Pod revisions. The actual InstanceSet template // remains untouched, so every newly created Pod uses the new copy command. func normalizeKBAgentInitCommandForRevision(template *corev1.PodTemplateSpec) { - if template == nil || !hasSharedKBAgentCommand(template.Spec.Containers) { + if template == nil || !kbagent.UsesSharedBinary(template.Spec.Containers) { return } for i := range template.Spec.InitContainers { initContainer := &template.Spec.InitContainers[i] - if initContainer.Name == kbagent.InitContainerName && reflect.DeepEqual(initContainer.Command, kbAgentInitCopyCommand) { - initContainer.Command = append([]string(nil), legacyKBAgentInitCopyCommand...) + if initContainer.Name == kbagent.InitContainerName && kbagent.IsCurrentInitCopyCommand(initContainer.Command) { + initContainer.Command = kbagent.LegacyInitCopyCommand() return } } } -func hasSharedKBAgentCommand(containers []corev1.Container) bool { - index := slices.IndexFunc(containers, func(container corev1.Container) bool { - return container.Name == kbagent.ContainerName - }) - return index >= 0 && reflect.DeepEqual(containers[index].Command, []string{kbAgentCommandOnSharedMount}) -} - // podForDeferredKBAgentInitMigration returns a desired Pod view that retains the // live init-kbagent image and command. This lets existing custom-image Pods keep // running without applying an immutable init command change, while the real // InstanceSet template stays on the new image and two-file copy command. func podForDeferredKBAgentInitMigration(old, desired *corev1.Pod) (*corev1.Pod, bool) { if old == nil || desired == nil || - !hasSharedKBAgentCommand(old.Spec.Containers) || !hasSharedKBAgentCommand(desired.Spec.Containers) { + !kbagent.UsesSharedBinary(old.Spec.Containers) || !kbagent.UsesSharedBinary(desired.Spec.Containers) { return desired, false } oldIndex := slices.IndexFunc(old.Spec.InitContainers, func(container corev1.Container) bool { @@ -149,8 +135,8 @@ func podForDeferredKBAgentInitMigration(old, desired *corev1.Pod) (*corev1.Pod, } oldInit := &old.Spec.InitContainers[oldIndex] desiredInit := &desired.Spec.InitContainers[desiredIndex] - if !reflect.DeepEqual(oldInit.Command, legacyKBAgentInitCopyCommand) || - !reflect.DeepEqual(desiredInit.Command, kbAgentInitCopyCommand) { + if !kbagent.IsLegacyInitCopyCommand(oldInit.Command) || + !kbagent.IsCurrentInitCopyCommand(desiredInit.Command) { return desired, false } diff --git a/pkg/controller/instanceset/in_place_update_util_test.go b/pkg/controller/instanceset/in_place_update_util_test.go index b4388dba9a0..91899b75151 100644 --- a/pkg/controller/instanceset/in_place_update_util_test.go +++ b/pkg/controller/instanceset/in_place_update_util_test.go @@ -36,6 +36,7 @@ import ( "github.com/apecloud/kubeblocks/pkg/constant" "github.com/apecloud/kubeblocks/pkg/controller/builder" "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" + "github.com/apecloud/kubeblocks/pkg/kbagent" viper "github.com/apecloud/kubeblocks/pkg/viperx" ) @@ -76,21 +77,21 @@ var _ = Describe("instance util test", func() { InitContainers: []corev1.Container{{ Name: "init-kbagent", Image: "docker.io/apecloud/kubeblocks-tools:1.0.0", - Command: append([]string(nil), legacyKBAgentInitCopyCommand...), + Command: kbagent.LegacyInitCopyCommand(), }}, Containers: []corev1.Container{ {Name: "app", Image: "mysql:8.0"}, - {Name: "kbagent", Image: "custom-action:1.0", Command: []string{"/kubeblocks/kbagent"}}, + {Name: "kbagent", Image: "custom-action:1.0", Command: []string{kbagent.SharedBinaryPath}}, }, }} newTemplate := oldTemplate.DeepCopy() newTemplate.Spec.InitContainers[0].Image = "mirror.local/apecloud/kubeblocks-tools:1.1.0" - newTemplate.Spec.InitContainers[0].Command = append([]string(nil), kbAgentInitCopyCommand...) + newTemplate.Spec.InitContainers[0].Command = kbagent.CurrentInitCopyCommand() oldFiltered := filterInPlaceFields(oldTemplate) newFiltered := filterInPlaceFields(newTemplate) Expect(newFiltered).Should(Equal(oldFiltered)) - Expect(newTemplate.Spec.InitContainers[0].Command).Should(Equal(kbAgentInitCopyCommand), + Expect(newTemplate.Spec.InitContainers[0].Command).Should(Equal(kbagent.CurrentInitCopyCommand()), "revision normalization must not modify the desired template") parent := builder.NewInstanceSetBuilder(namespace, name).SetUID(uid).GetObject() @@ -135,35 +136,35 @@ var _ = Describe("instance util test", func() { InitContainers: []corev1.Container{{ Name: "init-kbagent", Image: "docker.io/apecloud/kubeblocks-tools:1.0.0", - Command: append([]string(nil), legacyKBAgentInitCopyCommand...), + Command: kbagent.LegacyInitCopyCommand(), }}, Containers: []corev1.Container{ {Name: "app", Image: "mysql:8.0"}, - {Name: "kbagent", Image: "custom-action:1.0", Command: []string{"/kubeblocks/kbagent"}}, + {Name: "kbagent", Image: "custom-action:1.0", Command: []string{kbagent.SharedBinaryPath}}, }, }} desiredPod := oldPod.DeepCopy() desiredPod.Spec.InitContainers[0].Image = "mirror.local/apecloud/kubeblocks-tools:1.1.0" - desiredPod.Spec.InitContainers[0].Command = append([]string(nil), kbAgentInitCopyCommand...) + desiredPod.Spec.InitContainers[0].Command = kbagent.CurrentInitCopyCommand() normalized, ok := podForDeferredKBAgentInitMigration(oldPod, desiredPod) Expect(ok).Should(BeTrue()) Expect(normalized.Spec.InitContainers[0].Image).Should(Equal(oldPod.Spec.InitContainers[0].Image)) - Expect(normalized.Spec.InitContainers[0].Command).Should(Equal(legacyKBAgentInitCopyCommand)) + Expect(normalized.Spec.InitContainers[0].Command).Should(Equal(kbagent.LegacyInitCopyCommand())) Expect(desiredPod.Spec.InitContainers[0].Image).Should(Equal("mirror.local/apecloud/kubeblocks-tools:1.1.0")) - Expect(desiredPod.Spec.InitContainers[0].Command).Should(Equal(kbAgentInitCopyCommand)) + Expect(desiredPod.Spec.InitContainers[0].Command).Should(Equal(kbagent.CurrentInitCopyCommand())) appUpgrade := desiredPod.DeepCopy() appUpgrade.Spec.Containers[0].Image = "mysql:8.4" normalized, ok = podForDeferredKBAgentInitMigration(oldPod, appUpgrade) Expect(ok).Should(BeTrue(), "application image changes must not prevent deferring the init pair") Expect(normalized.Spec.InitContainers[0].Image).Should(Equal(oldPod.Spec.InitContainers[0].Image)) - Expect(normalized.Spec.InitContainers[0].Command).Should(Equal(legacyKBAgentInitCopyCommand)) + Expect(normalized.Spec.InitContainers[0].Command).Should(Equal(kbagent.LegacyInitCopyCommand())) Expect(normalized.Spec.Containers[0].Image).Should(Equal("mysql:8.4"), "the application image must remain at its desired value") defaultImagePod := desiredPod.DeepCopy() - defaultImagePod.Spec.Containers[1].Command = []string{"/bin/kbagent"} + defaultImagePod.Spec.Containers[1].Command = []string{kbagent.BinaryPath} _, ok = podForDeferredKBAgentInitMigration(oldPod, defaultImagePod) Expect(ok).Should(BeFalse(), "the compatibility path is only for custom-image workloads") }) @@ -219,7 +220,7 @@ var _ = Describe("instance util test", func() { oldPod := buildRandomPod() oldPod.Spec.Containers = []corev1.Container{ {Name: "app", Image: "docker.io/apecloud/redis:7.2"}, - {Name: "kbagent", Image: "docker.io/apecloud/kubeblocks-tools:1.0.0", Command: []string{"/bin/kbagent"}}, + {Name: "kbagent", Image: "docker.io/apecloud/kubeblocks-tools:1.0.0", Command: []string{kbagent.BinaryPath}}, } oldPod.Spec.InitContainers = []corev1.Container{ {Name: "init-kbagent", Image: "docker.io/apecloud/kubeblocks-tools:1.0.0", Command: []string{"cp"}}, diff --git a/pkg/controller/instanceset/reconciler_update_test.go b/pkg/controller/instanceset/reconciler_update_test.go index 2f423596eb0..052d5f0ee06 100644 --- a/pkg/controller/instanceset/reconciler_update_test.go +++ b/pkg/controller/instanceset/reconciler_update_test.go @@ -43,6 +43,7 @@ import ( "github.com/apecloud/kubeblocks/pkg/controller/kubebuilderx" "github.com/apecloud/kubeblocks/pkg/controller/lifecycle" intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil" + "github.com/apecloud/kubeblocks/pkg/kbagent" viper "github.com/apecloud/kubeblocks/pkg/viperx" ) @@ -560,7 +561,7 @@ var _ = Describe("update reconciler test", func() { its.Spec.Template.Spec.Containers = append(its.Spec.Template.Spec.Containers, corev1.Container{ Name: "kbagent", Image: "docker.io/apecloud/kubeblocks-tools:1.0.0", - Command: []string{"/bin/kbagent"}, + Command: []string{kbagent.BinaryPath}, }) its.Spec.LifecycleActions = &workloads.LifecycleActions{ Switchover: &kbappsv1.Action{ @@ -669,11 +670,11 @@ var _ = Describe("update reconciler test", func() { its.Spec.Template.Spec.InitContainers = []corev1.Container{{ Name: "init-kbagent", Image: "docker.io/apecloud/kubeblocks-tools:1.0.0", - Command: append([]string(nil), legacyKBAgentInitCopyCommand...), + Command: kbagent.LegacyInitCopyCommand(), }} its.Spec.Template.Spec.Containers = []corev1.Container{ {Name: "app", Image: "mysql:8.0"}, - {Name: "kbagent", Image: "custom-action:1.0", Command: []string{"/kubeblocks/kbagent"}}, + {Name: "kbagent", Image: "custom-action:1.0", Command: []string{kbagent.SharedBinaryPath}}, } tree := kubebuilderx.NewObjectTree() @@ -691,9 +692,9 @@ var _ = Describe("update reconciler test", func() { }) its.Spec.Template.Spec.InitContainers[0].Image = "mirror.local/apecloud/kubeblocks-tools:1.1.0" - its.Spec.Template.Spec.InitContainers[0].Command = append([]string(nil), kbAgentInitCopyCommand...) + its.Spec.Template.Spec.InitContainers[0].Command = kbagent.CurrentInitCopyCommand() its.Spec.Template.Spec.Containers[0].Image = "mysql:8.4" - Expect(its.Spec.Template.Spec.InitContainers[0].Command).Should(Equal(kbAgentInitCopyCommand), + Expect(its.Spec.Template.Spec.InitContainers[0].Command).Should(Equal(kbagent.CurrentInitCopyCommand()), "the desired template must retain the new command") reconciler := NewUpdateReconciler() @@ -706,7 +707,7 @@ var _ = Describe("update reconciler test", func() { livePod := postPods[0].(*corev1.Pod) Expect(livePod.UID).Should(Equal(pod.UID), "the existing Pod must not be recreated") Expect(livePod.Spec.InitContainers[0].Image).Should(Equal("docker.io/apecloud/kubeblocks-tools:1.0.0")) - Expect(livePod.Spec.InitContainers[0].Command).Should(Equal(legacyKBAgentInitCopyCommand), + Expect(livePod.Spec.InitContainers[0].Command).Should(Equal(kbagent.LegacyInitCopyCommand()), "the existing Pod must keep the old image and command as one pair") Expect(livePod.Spec.Containers[0].Image).Should(Equal("mysql:8.4"), "the application image must still be updated in place") @@ -718,7 +719,7 @@ var _ = Describe("update reconciler test", func() { replacement, err := buildInstancePodByTemplate(pod.Name, templates[0], its, "") Expect(err).ShouldNot(HaveOccurred()) Expect(replacement.Spec.InitContainers[0].Image).Should(Equal("mirror.local/apecloud/kubeblocks-tools:1.1.0")) - Expect(replacement.Spec.InitContainers[0].Command).Should(Equal(kbAgentInitCopyCommand), + Expect(replacement.Spec.InitContainers[0].Command).Should(Equal(kbagent.CurrentInitCopyCommand()), "a recreated Pod must copy both kbagent and tini-static") if its.Spec.Template.Annotations == nil { @@ -745,7 +746,7 @@ var _ = Describe("update reconciler test", func() { its.Spec.Template.Spec.Containers = append(its.Spec.Template.Spec.Containers, corev1.Container{ Name: "kbagent", Image: "docker.io/apecloud/kubeblocks-tools:1.0.0", - Command: []string{"/bin/kbagent"}, + Command: []string{kbagent.BinaryPath}, }) tree := kubebuilderx.NewObjectTree() diff --git a/pkg/kbagent/podspec.go b/pkg/kbagent/podspec.go new file mode 100644 index 00000000000..fbecd10b4ce --- /dev/null +++ b/pkg/kbagent/podspec.go @@ -0,0 +1,77 @@ +/* +Copyright (C) 2022-2026 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package kbagent + +import ( + "slices" + + corev1 "k8s.io/api/core/v1" +) + +const ( + BinaryPath = "/bin/kbagent" + TiniPath = "/bin/tini-static" + SharedMountPath = "/kubeblocks" + SharedBinaryPath = SharedMountPath + "/kbagent" + SharedVolumeName = "kubeblocks" +) + +// CurrentInitCopyCommand returns the current init-kbagent copy command. +func CurrentInitCopyCommand() []string { + return []string{"cp", "-r", BinaryPath, TiniPath, SharedMountPath + "/"} +} + +// LegacyInitCopyCommand returns the init-kbagent copy command used before +// tini-static became part of the shared kbagent runtime contract. +func LegacyInitCopyCommand() []string { + return []string{"cp", "-r", BinaryPath, SharedMountPath + "/"} +} + +// IsCurrentInitCopyCommand reports whether command implements the current +// init-kbagent copy contract. +func IsCurrentInitCopyCommand(command []string) bool { + return slices.Equal(command, CurrentInitCopyCommand()) +} + +// IsLegacyInitCopyCommand reports whether command implements the legacy +// init-kbagent copy contract. +func IsLegacyInitCopyCommand(command []string) bool { + return slices.Equal(command, LegacyInitCopyCommand()) +} + +// IsContainer reports whether container belongs to the kbagent runtime. +func IsContainer(container *corev1.Container) bool { + return container != nil && (container.Name == ContainerName || + container.Name == ContainerName4Worker || container.Name == InitContainerName) +} + +// UsesSharedBinary reports whether the kbagent server container runs the +// binary copied to the shared mount. +func UsesSharedBinary(containers []corev1.Container) bool { + index := slices.IndexFunc(containers, func(container corev1.Container) bool { + return container.Name == ContainerName + }) + return index >= 0 && slices.Equal(containers[index].Command, []string{SharedBinaryPath}) +} + +// SharedVolumeMount returns the volume mount used by the shared kbagent binary. +func SharedVolumeMount() corev1.VolumeMount { + return corev1.VolumeMount{Name: SharedVolumeName, MountPath: SharedMountPath} +} diff --git a/pkg/kbagent/podspec_test.go b/pkg/kbagent/podspec_test.go new file mode 100644 index 00000000000..c36286874a2 --- /dev/null +++ b/pkg/kbagent/podspec_test.go @@ -0,0 +1,69 @@ +/* +Copyright (C) 2022-2026 ApeCloud Co., Ltd + +This file is part of KubeBlocks project + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +package kbagent + +import ( + "slices" + "testing" + + corev1 "k8s.io/api/core/v1" +) + +func TestInitCopyCommandContract(t *testing.T) { + current := CurrentInitCopyCommand() + if !slices.Equal(current, []string{"cp", "-r", BinaryPath, TiniPath, SharedMountPath + "/"}) { + t.Fatalf("unexpected current copy command: %v", current) + } + if !IsCurrentInitCopyCommand(current) || IsLegacyInitCopyCommand(current) { + t.Fatalf("current copy command was not recognized correctly: %v", current) + } + + legacy := LegacyInitCopyCommand() + if !slices.Equal(legacy, []string{"cp", "-r", BinaryPath, SharedMountPath + "/"}) { + t.Fatalf("unexpected legacy copy command: %v", legacy) + } + if !IsLegacyInitCopyCommand(legacy) || IsCurrentInitCopyCommand(legacy) { + t.Fatalf("legacy copy command was not recognized correctly: %v", legacy) + } + + current[0] = "changed" + legacy[0] = "changed" + if CurrentInitCopyCommand()[0] != "cp" || LegacyInitCopyCommand()[0] != "cp" { + t.Fatal("copy command constructors returned shared mutable slices") + } +} + +func TestPodSpecPredicates(t *testing.T) { + containers := []corev1.Container{{ + Name: ContainerName, + Command: []string{SharedBinaryPath}, + }} + if !UsesSharedBinary(containers) { + t.Fatal("shared kbagent binary was not recognized") + } + if !IsContainer(&containers[0]) || IsContainer(nil) { + t.Fatal("kbagent container identity was not recognized correctly") + } + + mount := SharedVolumeMount() + if mount.Name != SharedVolumeName || mount.MountPath != SharedMountPath { + t.Fatalf("unexpected shared volume mount: %#v", mount) + } +} From 830eeb6711f28b6d26fdd49ac941bda897bcf5f4 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 13 Aug 2026 12:09:11 +0800 Subject: [PATCH 09/10] refactor: simplify kbagent init contract names --- pkg/controller/component/kbagent.go | 6 +++-- pkg/controller/component/kbagent_test.go | 2 +- .../instanceset/in_place_update_util.go | 12 +++++----- .../instanceset/in_place_update_util_test.go | 16 +++++++------- .../instanceset/reconciler_update_test.go | 10 ++++----- pkg/kbagent/podspec.go | 22 +++++++++---------- pkg/kbagent/podspec_test.go | 10 ++++----- 7 files changed, 41 insertions(+), 37 deletions(-) diff --git a/pkg/controller/component/kbagent.go b/pkg/controller/component/kbagent.go index 1f4fa2745ba..381210c7b10 100644 --- a/pkg/controller/component/kbagent.go +++ b/pkg/controller/component/kbagent.go @@ -51,7 +51,9 @@ const ( podRoleLabelFileName = "role" ) -var roleLabelVolumeMount = corev1.VolumeMount{Name: roleLabelVolumeName, MountPath: podMetadataMountPath, ReadOnly: true} +var ( + roleLabelVolumeMount = corev1.VolumeMount{Name: roleLabelVolumeName, MountPath: podMetadataMountPath, ReadOnly: true} +) func UpdateKBAgentContainer4HostNetwork(synthesizedComp *SynthesizedComponent) { idx, c := intctrlutil.GetContainerByName(synthesizedComp.PodSpec.Containers, kbagent.ContainerName) @@ -449,7 +451,7 @@ func handleCustomImageNContainerDefined(synthesizedComp *SynthesizedComponent, c initContainer := builder.NewContainerBuilder(kbagent.InitContainerName). SetImage(viper.GetString(constant.KBToolsImage)). SetImagePullPolicy(corev1.PullIfNotPresent). - AddCommands(kbagent.CurrentInitCopyCommand()...). + AddCommands(kbagent.InitCommand()...). AddVolumeMounts(kbagent.SharedVolumeMount()). GetObject() synthesizedComp.PodSpec.InitContainers = append(synthesizedComp.PodSpec.InitContainers, *initContainer) diff --git a/pkg/controller/component/kbagent_test.go b/pkg/controller/component/kbagent_test.go index f992e82765c..2f3b6f06188 100644 --- a/pkg/controller/component/kbagent_test.go +++ b/pkg/controller/component/kbagent_test.go @@ -302,7 +302,7 @@ var _ = Describe("kb-agent", func() { ic := kbAgentInitContainer() Expect(ic).ShouldNot(BeNil()) - Expect(ic.Command).Should(Equal(kbagent.CurrentInitCopyCommand())) + Expect(ic.Command).Should(Equal(kbagent.InitCommand())) c := kbAgentContainer() Expect(c).ShouldNot(BeNil()) diff --git a/pkg/controller/instanceset/in_place_update_util.go b/pkg/controller/instanceset/in_place_update_util.go index 914396f1e4c..b7fa44f773f 100644 --- a/pkg/controller/instanceset/in_place_update_util.go +++ b/pkg/controller/instanceset/in_place_update_util.go @@ -47,7 +47,9 @@ const ( inPlaceUpdatePolicy podUpdatePolicy = "inPlaceUpdate" ) -var errTemplateNotFound = fmt.Errorf("no template found for pod") +var ( + errTemplateNotFound = fmt.Errorf("no template found for pod") +) func supportPodVerticalScaling() bool { return viper.GetBool(constant.FeatureGateInPlacePodVerticalScaling) @@ -108,8 +110,8 @@ func normalizeKBAgentInitCommandForRevision(template *corev1.PodTemplateSpec) { } for i := range template.Spec.InitContainers { initContainer := &template.Spec.InitContainers[i] - if initContainer.Name == kbagent.InitContainerName && kbagent.IsCurrentInitCopyCommand(initContainer.Command) { - initContainer.Command = kbagent.LegacyInitCopyCommand() + if initContainer.Name == kbagent.InitContainerName && kbagent.IsInitCommand(initContainer.Command) { + initContainer.Command = kbagent.LegacyInitCommand() return } } @@ -135,8 +137,8 @@ func podForDeferredKBAgentInitMigration(old, desired *corev1.Pod) (*corev1.Pod, } oldInit := &old.Spec.InitContainers[oldIndex] desiredInit := &desired.Spec.InitContainers[desiredIndex] - if !kbagent.IsLegacyInitCopyCommand(oldInit.Command) || - !kbagent.IsCurrentInitCopyCommand(desiredInit.Command) { + if !kbagent.IsLegacyInitCommand(oldInit.Command) || + !kbagent.IsInitCommand(desiredInit.Command) { return desired, false } diff --git a/pkg/controller/instanceset/in_place_update_util_test.go b/pkg/controller/instanceset/in_place_update_util_test.go index 91899b75151..6e0bc18c2de 100644 --- a/pkg/controller/instanceset/in_place_update_util_test.go +++ b/pkg/controller/instanceset/in_place_update_util_test.go @@ -77,7 +77,7 @@ var _ = Describe("instance util test", func() { InitContainers: []corev1.Container{{ Name: "init-kbagent", Image: "docker.io/apecloud/kubeblocks-tools:1.0.0", - Command: kbagent.LegacyInitCopyCommand(), + Command: kbagent.LegacyInitCommand(), }}, Containers: []corev1.Container{ {Name: "app", Image: "mysql:8.0"}, @@ -86,12 +86,12 @@ var _ = Describe("instance util test", func() { }} newTemplate := oldTemplate.DeepCopy() newTemplate.Spec.InitContainers[0].Image = "mirror.local/apecloud/kubeblocks-tools:1.1.0" - newTemplate.Spec.InitContainers[0].Command = kbagent.CurrentInitCopyCommand() + newTemplate.Spec.InitContainers[0].Command = kbagent.InitCommand() oldFiltered := filterInPlaceFields(oldTemplate) newFiltered := filterInPlaceFields(newTemplate) Expect(newFiltered).Should(Equal(oldFiltered)) - Expect(newTemplate.Spec.InitContainers[0].Command).Should(Equal(kbagent.CurrentInitCopyCommand()), + Expect(newTemplate.Spec.InitContainers[0].Command).Should(Equal(kbagent.InitCommand()), "revision normalization must not modify the desired template") parent := builder.NewInstanceSetBuilder(namespace, name).SetUID(uid).GetObject() @@ -136,7 +136,7 @@ var _ = Describe("instance util test", func() { InitContainers: []corev1.Container{{ Name: "init-kbagent", Image: "docker.io/apecloud/kubeblocks-tools:1.0.0", - Command: kbagent.LegacyInitCopyCommand(), + Command: kbagent.LegacyInitCommand(), }}, Containers: []corev1.Container{ {Name: "app", Image: "mysql:8.0"}, @@ -145,21 +145,21 @@ var _ = Describe("instance util test", func() { }} desiredPod := oldPod.DeepCopy() desiredPod.Spec.InitContainers[0].Image = "mirror.local/apecloud/kubeblocks-tools:1.1.0" - desiredPod.Spec.InitContainers[0].Command = kbagent.CurrentInitCopyCommand() + desiredPod.Spec.InitContainers[0].Command = kbagent.InitCommand() normalized, ok := podForDeferredKBAgentInitMigration(oldPod, desiredPod) Expect(ok).Should(BeTrue()) Expect(normalized.Spec.InitContainers[0].Image).Should(Equal(oldPod.Spec.InitContainers[0].Image)) - Expect(normalized.Spec.InitContainers[0].Command).Should(Equal(kbagent.LegacyInitCopyCommand())) + Expect(normalized.Spec.InitContainers[0].Command).Should(Equal(kbagent.LegacyInitCommand())) Expect(desiredPod.Spec.InitContainers[0].Image).Should(Equal("mirror.local/apecloud/kubeblocks-tools:1.1.0")) - Expect(desiredPod.Spec.InitContainers[0].Command).Should(Equal(kbagent.CurrentInitCopyCommand())) + Expect(desiredPod.Spec.InitContainers[0].Command).Should(Equal(kbagent.InitCommand())) appUpgrade := desiredPod.DeepCopy() appUpgrade.Spec.Containers[0].Image = "mysql:8.4" normalized, ok = podForDeferredKBAgentInitMigration(oldPod, appUpgrade) Expect(ok).Should(BeTrue(), "application image changes must not prevent deferring the init pair") Expect(normalized.Spec.InitContainers[0].Image).Should(Equal(oldPod.Spec.InitContainers[0].Image)) - Expect(normalized.Spec.InitContainers[0].Command).Should(Equal(kbagent.LegacyInitCopyCommand())) + Expect(normalized.Spec.InitContainers[0].Command).Should(Equal(kbagent.LegacyInitCommand())) Expect(normalized.Spec.Containers[0].Image).Should(Equal("mysql:8.4"), "the application image must remain at its desired value") diff --git a/pkg/controller/instanceset/reconciler_update_test.go b/pkg/controller/instanceset/reconciler_update_test.go index 052d5f0ee06..4f5754ef164 100644 --- a/pkg/controller/instanceset/reconciler_update_test.go +++ b/pkg/controller/instanceset/reconciler_update_test.go @@ -670,7 +670,7 @@ var _ = Describe("update reconciler test", func() { its.Spec.Template.Spec.InitContainers = []corev1.Container{{ Name: "init-kbagent", Image: "docker.io/apecloud/kubeblocks-tools:1.0.0", - Command: kbagent.LegacyInitCopyCommand(), + Command: kbagent.LegacyInitCommand(), }} its.Spec.Template.Spec.Containers = []corev1.Container{ {Name: "app", Image: "mysql:8.0"}, @@ -692,9 +692,9 @@ var _ = Describe("update reconciler test", func() { }) its.Spec.Template.Spec.InitContainers[0].Image = "mirror.local/apecloud/kubeblocks-tools:1.1.0" - its.Spec.Template.Spec.InitContainers[0].Command = kbagent.CurrentInitCopyCommand() + its.Spec.Template.Spec.InitContainers[0].Command = kbagent.InitCommand() its.Spec.Template.Spec.Containers[0].Image = "mysql:8.4" - Expect(its.Spec.Template.Spec.InitContainers[0].Command).Should(Equal(kbagent.CurrentInitCopyCommand()), + Expect(its.Spec.Template.Spec.InitContainers[0].Command).Should(Equal(kbagent.InitCommand()), "the desired template must retain the new command") reconciler := NewUpdateReconciler() @@ -707,7 +707,7 @@ var _ = Describe("update reconciler test", func() { livePod := postPods[0].(*corev1.Pod) Expect(livePod.UID).Should(Equal(pod.UID), "the existing Pod must not be recreated") Expect(livePod.Spec.InitContainers[0].Image).Should(Equal("docker.io/apecloud/kubeblocks-tools:1.0.0")) - Expect(livePod.Spec.InitContainers[0].Command).Should(Equal(kbagent.LegacyInitCopyCommand()), + Expect(livePod.Spec.InitContainers[0].Command).Should(Equal(kbagent.LegacyInitCommand()), "the existing Pod must keep the old image and command as one pair") Expect(livePod.Spec.Containers[0].Image).Should(Equal("mysql:8.4"), "the application image must still be updated in place") @@ -719,7 +719,7 @@ var _ = Describe("update reconciler test", func() { replacement, err := buildInstancePodByTemplate(pod.Name, templates[0], its, "") Expect(err).ShouldNot(HaveOccurred()) Expect(replacement.Spec.InitContainers[0].Image).Should(Equal("mirror.local/apecloud/kubeblocks-tools:1.1.0")) - Expect(replacement.Spec.InitContainers[0].Command).Should(Equal(kbagent.CurrentInitCopyCommand()), + Expect(replacement.Spec.InitContainers[0].Command).Should(Equal(kbagent.InitCommand()), "a recreated Pod must copy both kbagent and tini-static") if its.Spec.Template.Annotations == nil { diff --git a/pkg/kbagent/podspec.go b/pkg/kbagent/podspec.go index fbecd10b4ce..9677543f23f 100644 --- a/pkg/kbagent/podspec.go +++ b/pkg/kbagent/podspec.go @@ -33,27 +33,27 @@ const ( SharedVolumeName = "kubeblocks" ) -// CurrentInitCopyCommand returns the current init-kbagent copy command. -func CurrentInitCopyCommand() []string { +// InitCommand returns the current init-kbagent copy command. +func InitCommand() []string { return []string{"cp", "-r", BinaryPath, TiniPath, SharedMountPath + "/"} } -// LegacyInitCopyCommand returns the init-kbagent copy command used before +// LegacyInitCommand returns the init-kbagent copy command used before // tini-static became part of the shared kbagent runtime contract. -func LegacyInitCopyCommand() []string { +func LegacyInitCommand() []string { return []string{"cp", "-r", BinaryPath, SharedMountPath + "/"} } -// IsCurrentInitCopyCommand reports whether command implements the current -// init-kbagent copy contract. -func IsCurrentInitCopyCommand(command []string) bool { - return slices.Equal(command, CurrentInitCopyCommand()) +// IsInitCommand reports whether command implements the current init-kbagent +// copy contract. +func IsInitCommand(command []string) bool { + return slices.Equal(command, InitCommand()) } -// IsLegacyInitCopyCommand reports whether command implements the legacy +// IsLegacyInitCommand reports whether command implements the legacy // init-kbagent copy contract. -func IsLegacyInitCopyCommand(command []string) bool { - return slices.Equal(command, LegacyInitCopyCommand()) +func IsLegacyInitCommand(command []string) bool { + return slices.Equal(command, LegacyInitCommand()) } // IsContainer reports whether container belongs to the kbagent runtime. diff --git a/pkg/kbagent/podspec_test.go b/pkg/kbagent/podspec_test.go index c36286874a2..3687303e251 100644 --- a/pkg/kbagent/podspec_test.go +++ b/pkg/kbagent/podspec_test.go @@ -27,25 +27,25 @@ import ( ) func TestInitCopyCommandContract(t *testing.T) { - current := CurrentInitCopyCommand() + current := InitCommand() if !slices.Equal(current, []string{"cp", "-r", BinaryPath, TiniPath, SharedMountPath + "/"}) { t.Fatalf("unexpected current copy command: %v", current) } - if !IsCurrentInitCopyCommand(current) || IsLegacyInitCopyCommand(current) { + if !IsInitCommand(current) || IsLegacyInitCommand(current) { t.Fatalf("current copy command was not recognized correctly: %v", current) } - legacy := LegacyInitCopyCommand() + legacy := LegacyInitCommand() if !slices.Equal(legacy, []string{"cp", "-r", BinaryPath, SharedMountPath + "/"}) { t.Fatalf("unexpected legacy copy command: %v", legacy) } - if !IsLegacyInitCopyCommand(legacy) || IsCurrentInitCopyCommand(legacy) { + if !IsLegacyInitCommand(legacy) || IsInitCommand(legacy) { t.Fatalf("legacy copy command was not recognized correctly: %v", legacy) } current[0] = "changed" legacy[0] = "changed" - if CurrentInitCopyCommand()[0] != "cp" || LegacyInitCopyCommand()[0] != "cp" { + if InitCommand()[0] != "cp" || LegacyInitCommand()[0] != "cp" { t.Fatal("copy command constructors returned shared mutable slices") } } From 6d910ef1e151532ee488d5043beff5c432781686 Mon Sep 17 00:00:00 2001 From: Leon Date: Thu, 13 Aug 2026 12:53:17 +0800 Subject: [PATCH 10/10] test: adapt kbagent migration coverage for 1.1 --- pkg/controller/instanceset/reconciler_update_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/controller/instanceset/reconciler_update_test.go b/pkg/controller/instanceset/reconciler_update_test.go index 4f5754ef164..aa81347f93d 100644 --- a/pkg/controller/instanceset/reconciler_update_test.go +++ b/pkg/controller/instanceset/reconciler_update_test.go @@ -728,7 +728,7 @@ var _ = Describe("update reconciler test", func() { its.Spec.Template.Annotations[constant.RestartAnnotationKey] = "2026-08-11T12:00:00Z" _, err = NewRevisionUpdateReconciler().Reconcile(tree) Expect(err).ShouldNot(HaveOccurred()) - policy, _, _, err := getPodUpdatePolicy(its, livePod) + policy, _, err := getPodUpdatePolicy(its, livePod) Expect(err).ShouldNot(HaveOccurred()) Expect(policy).Should(Equal(recreatePolicy), "an explicit restart must still recreate the Pod") })