fix(kbagent): reap zombie processes

cmd/kbagent/main.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"path/filepath"
 	"strings"
 	"syscall"
 
@@ -42,10 +43,30 @@ import (
 
 const (
 	defaultMaxConcurrency = 32
+	tiniBinaryName        = "tini-static"
 )
 
 var serverConfig server.Config
 
+type execProcessFunc func(argv0 string, argv []string, envv []string) error
+type executableFunc func() (string, error)
+
+func reexecUnderInit(pid int, args, env []string, executable executableFunc, execProcess execProcessFunc) error {
+	if pid != 1 {
+		return nil
+	}
+	exe, err := executable()
+	if err != nil {
+		return errors.Wrap(err, "resolve kbagent executable")
+	}
+	tini := filepath.Join(filepath.Dir(exe), tiniBinaryName)
+	argv := []string{tini, "--", exe}
+	if len(args) > 1 {
+		argv = append(argv, args[1:]...)
+	}
+	return errors.Wrap(execProcess(tini, argv, env), "exec tini-static")
+}
+
 func init() {
 	viper.AutomaticEnv()
 
@@ -60,6 +81,11 @@ func init() {
 }
 
 func main() {
+	if err := reexecUnderInit(os.Getpid(), os.Args, os.Environ(), os.Executable, syscall.Exec); err != nil {
+		_, _ = fmt.Fprintf(os.Stderr, "failed to launch kbagent under tini: %v\n", err)
+		os.Exit(1)
+	}
+
 	// Set GOMAXPROCS
 	_, _ = maxprocs.Set()
 

cmd/kbagent/main_test.go

@@ -0,0 +1,94 @@
+/*
+Copyright (C) 2022-2026 ApeCloud Co., Ltd
+
+This file is part of KubeBlocks project
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package main
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+)
+
+func TestReexecUnderInitSkipsNonPID1(t *testing.T) {
+	called := false
+	err := reexecUnderInit(2, []string{"/bin/kbagent"}, nil,
+		func() (string, error) {
+			called = true
+			return "", nil
+		},
+		func(string, []string, []string) error {
+			called = true
+			return nil
+		})
+	if err != nil || called {
+		t.Fatalf("reexecUnderInit() = %v, called = %v", err, called)
+	}
+}
+
+func TestReexecUnderInitPreservesArgumentsAndEnvironment(t *testing.T) {
+	tests := []struct {
+		name string
+		exe  string
+	}{
+		{name: "tools image", exe: "/bin/kbagent"},
+		{name: "custom action image", exe: "/kubeblocks/kbagent"},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			args := []string{tt.exe, "--port", "3501", "--server=false"}
+			env := []string{"A=B", "C=D"}
+			var gotPath string
+			var gotArgs, gotEnv []string
+			err := reexecUnderInit(1, args, env,
+				func() (string, error) { return tt.exe, nil },
+				func(path string, argv, envv []string) error {
+					gotPath = path
+					gotArgs = append([]string(nil), argv...)
+					gotEnv = append([]string(nil), envv...)
+					return nil
+				})
+			if err != nil {
+				t.Fatalf("reexecUnderInit() error = %v", err)
+			}
+			wantPath := tt.exe[:len(tt.exe)-len("kbagent")] + tiniBinaryName
+			wantArgs := []string{wantPath, "--", tt.exe, "--port", "3501", "--server=false"}
+			if gotPath != wantPath || !reflect.DeepEqual(gotArgs, wantArgs) || !reflect.DeepEqual(gotEnv, env) {
+				t.Fatalf("exec = (%q, %#v, %#v), want (%q, %#v, %#v)", gotPath, gotArgs, gotEnv, wantPath, wantArgs, env)
+			}
+		})
+	}
+}
+
+func TestReexecUnderInitReturnsStartupErrors(t *testing.T) {
+	resolveErr := errors.New("resolve")
+	err := reexecUnderInit(1, []string{"kbagent"}, nil,
+		func() (string, error) { return "", resolveErr },
+		func(string, []string, []string) error { return nil })
+	if !errors.Is(err, resolveErr) {
+		t.Fatalf("expected executable error, got %v", err)
+	}
+
+	execErr := errors.New("exec")
+	err = reexecUnderInit(1, []string{"/bin/kbagent"}, nil,
+		func() (string, error) { return "/bin/kbagent", nil },
+		func(string, []string, []string) error { return execErr })
+	if !errors.Is(err, execErr) {
+		t.Fatalf("expected exec error, got %v", err)
+	}
+}

controllers/apps/component/transformer_component_workload.go

@@ -38,6 +38,7 @@ import (
 	"github.com/apecloud/kubeblocks/pkg/controller/graph"
 	"github.com/apecloud/kubeblocks/pkg/controller/model"
 	intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil"
+	"github.com/apecloud/kubeblocks/pkg/kbagent"
 )
 
 // componentWorkloadTransformer handles component workload generation
@@ -327,7 +328,7 @@ func checkNRollbackProtoImages(itsObj, itsProto *workloads.InstanceSet) {
 		images[i] = make(map[string]string)
 		for _, c := range cc {
 			// skip the kb-agent container
-			if component.IsKBAgentContainer(&c) {
+			if kbagent.IsContainer(&c) {
 				continue
 			}
 			images[i][c.Name] = c.Image

docker/Dockerfile-tools

@@ -66,7 +66,8 @@ ENV APK_MIRROR=${APK_MIRROR}
 RUN if [ -n "${APK_MIRROR}" ]; then sed -i "s/dl-cdn.alpinelinux.org/${APK_MIRROR}/g" /etc/apk/repositories; fi
 RUN apk upgrade --no-cache
 # curl is used to send certain requests, such as reload in config-manager.
-RUN apk add --no-cache curl kubectl helm jq --allow-untrusted \
+RUN apk add --no-cache curl kubectl helm jq tini-static --allow-untrusted \
+	&& cp /sbin/tini-static /bin/tini-static \
     && rm -rf /var/cache/apk/*
 
 # copy kubeblocks tools

pkg/controller/component/kbagent.go

@@ -40,10 +40,6 @@ import (
 )
 
 const (
-	kbAgentCommand              = "/bin/kbagent"
-	kbAgentSharedMountPath      = "/kubeblocks"
-	kbAgentCommandOnSharedMount = "/kubeblocks/kbagent"
-
 	minAvailablePort = 1025
 	maxAvailablePort = 65535
 
@@ -56,14 +52,9 @@ const (
 )
 
 var (
-	sharedVolumeMount    = corev1.VolumeMount{Name: "kubeblocks", MountPath: kbAgentSharedMountPath}
 	roleLabelVolumeMount = corev1.VolumeMount{Name: roleLabelVolumeName, MountPath: podMetadataMountPath, ReadOnly: true}
 )
 
-func IsKBAgentContainer(c *corev1.Container) bool {
-	return c.Name == kbagent.ContainerName || c.Name == kbagent.ContainerName4Worker || c.Name == kbagent.InitContainerName
-}
-
 func UpdateKBAgentContainer4HostNetwork(synthesizedComp *SynthesizedComponent) {
 	idx, c := intctrlutil.GetContainerByName(synthesizedComp.PodSpec.Containers, kbagent.ContainerName)
 	if c == nil {
@@ -139,7 +130,7 @@ func buildKBAgentContainer(synthesizedComp *SynthesizedComponent) error {
 		b := builder.NewContainerBuilder(name).
 			SetImage(viper.GetString(constant.KBToolsImage)).
 			SetImagePullPolicy(corev1.PullIfNotPresent).
-			AddCommands(kbAgentCommand).
+			AddCommands(kbagent.BinaryPath).
 			AddEnv(mergedActionEnv4KBAgent(synthesizedComp)...).
 			AddEnv(envVars...).
 			SetSecurityContext(corev1.SecurityContext{
@@ -460,15 +451,15 @@ func handleCustomImageNContainerDefined(synthesizedComp *SynthesizedComponent, c
 		initContainer := builder.NewContainerBuilder(kbagent.InitContainerName).
 			SetImage(viper.GetString(constant.KBToolsImage)).
 			SetImagePullPolicy(corev1.PullIfNotPresent).
-			AddCommands([]string{"cp", "-r", kbAgentCommand, kbAgentSharedMountPath + "/"}...).
-			AddVolumeMounts(sharedVolumeMount).
+			AddCommands(kbagent.InitCommand()...).
+			AddVolumeMounts(kbagent.SharedVolumeMount()).
 			GetObject()
 		synthesizedComp.PodSpec.InitContainers = append(synthesizedComp.PodSpec.InitContainers, *initContainer)
 
 		for _, container := range containers {
 			container.Image = image
-			container.Command[0] = kbAgentCommandOnSharedMount
-			container.VolumeMounts = append(container.VolumeMounts, sharedVolumeMount)
+			container.Command[0] = kbagent.SharedBinaryPath
+			container.VolumeMounts = append(container.VolumeMounts, kbagent.SharedVolumeMount())
 		}
 	}
 

pkg/controller/component/kbagent_test.go

@@ -302,13 +302,14 @@ var _ = Describe("kb-agent", func() {
 
 			ic := kbAgentInitContainer()
 			Expect(ic).ShouldNot(BeNil())
+			Expect(ic.Command).Should(Equal(kbagent.InitCommand()))
 
 			c := kbAgentContainer()
 			Expect(c).ShouldNot(BeNil())
 			Expect(c.Image).Should(Equal(image))
-			Expect(c.Command[0]).Should(Equal(kbAgentCommandOnSharedMount))
+			Expect(c.Command[0]).Should(Equal(kbagent.SharedBinaryPath))
 			Expect(c.VolumeMounts).Should(HaveLen(2))
-			Expect(c.VolumeMounts[0]).Should(Equal(sharedVolumeMount))
+			Expect(c.VolumeMounts[0]).Should(Equal(kbagent.SharedVolumeMount()))
 			Expect(c.VolumeMounts[1]).Should(Equal(roleLabelVolumeMount))
 		})
 
@@ -345,7 +346,7 @@ var _ = Describe("kb-agent", func() {
 			c := kbAgentContainer()
 			Expect(c).ShouldNot(BeNil())
 			Expect(c.Image).Should(Equal(viperx.GetString(constant.KBToolsImage)))
-			Expect(c.Command[0]).Should(Equal(kbAgentCommand))
+			Expect(c.Command[0]).Should(Equal(kbagent.BinaryPath))
 			Expect(c.VolumeMounts).Should(HaveLen(1))
 			Expect(c.VolumeMounts[0]).Should(Equal(roleLabelVolumeMount))
 		})
@@ -423,9 +424,9 @@ var _ = Describe("kb-agent", func() {
 			Expect(c).ShouldNot(BeNil())
 			Expect(c).ShouldNot(BeNil())
 			Expect(c.Image).Should(Equal(container.Image))
-			Expect(c.Command[0]).Should(Equal(kbAgentCommandOnSharedMount))
+			Expect(c.Command[0]).Should(Equal(kbagent.SharedBinaryPath))
 			Expect(c.VolumeMounts).Should(HaveLen(3))
-			Expect(c.VolumeMounts[0]).Should(Equal(sharedVolumeMount))
+			Expect(c.VolumeMounts[0]).Should(Equal(kbagent.SharedVolumeMount()))
 			Expect(c.VolumeMounts[1]).Should(Equal(container.VolumeMounts[0]))
 			Expect(c.VolumeMounts[2]).Should(Equal(roleLabelVolumeMount))
 		})
@@ -451,9 +452,9 @@ var _ = Describe("kb-agent", func() {
 
 			c := kbAgentContainer()
 			Expect(c.Image).Should(Equal(image))
-			Expect(c.Command[0]).Should(Equal(kbAgentCommandOnSharedMount))
+			Expect(c.Command[0]).Should(Equal(kbagent.SharedBinaryPath))
 			Expect(c.VolumeMounts).Should(HaveLen(3))
-			Expect(c.VolumeMounts[0]).Should(Equal(sharedVolumeMount))
+			Expect(c.VolumeMounts[0]).Should(Equal(kbagent.SharedVolumeMount()))
 			Expect(c.VolumeMounts[1]).Should(Equal(container.VolumeMounts[0]))
 			Expect(c.VolumeMounts[2]).Should(Equal(roleLabelVolumeMount))
 		})

pkg/controller/instanceset/in_place_update_util.go

@@ -35,6 +35,7 @@ import (
 	"github.com/apecloud/kubeblocks/pkg/constant"
 	"github.com/apecloud/kubeblocks/pkg/controller/instancetemplate"
 	intctrlutil "github.com/apecloud/kubeblocks/pkg/controllerutil"
+	"github.com/apecloud/kubeblocks/pkg/kbagent"
 	viper "github.com/apecloud/kubeblocks/pkg/viperx"
 )
 
@@ -95,10 +96,72 @@ func filterInPlaceFields(src *corev1.PodTemplateSpec) *corev1.PodTemplateSpec {
 		delete(template.Spec.Containers[i].Resources.Limits, corev1.ResourceCPU)
 		delete(template.Spec.Containers[i].Resources.Limits, corev1.ResourceMemory)
 	}
+	normalizeKBAgentInitCommandForRevision(template)
 
 	return template
 }
 
+// normalizeKBAgentInitCommandForRevision keeps the one-time addition of tini-static
+// from changing existing custom-image Pod revisions. The actual InstanceSet template
+// remains untouched, so every newly created Pod uses the new copy command.
+func normalizeKBAgentInitCommandForRevision(template *corev1.PodTemplateSpec) {
+	if template == nil || !kbagent.UsesSharedBinary(template.Spec.Containers) {
+		return
+	}
+	for i := range template.Spec.InitContainers {
+		initContainer := &template.Spec.InitContainers[i]
+		if initContainer.Name == kbagent.InitContainerName && kbagent.IsInitCommand(initContainer.Command) {
+			initContainer.Command = kbagent.LegacyInitCommand()
+			return
+		}
+	}
+}
+
+// podForDeferredKBAgentInitMigration returns a desired Pod view that retains the
+// live init-kbagent image and command. This lets existing custom-image Pods keep
+// running without applying an immutable init command change, while the real
+// InstanceSet template stays on the new image and two-file copy command.
+func podForDeferredKBAgentInitMigration(old, desired *corev1.Pod) (*corev1.Pod, bool) {
+	if old == nil || desired == nil ||
+		!kbagent.UsesSharedBinary(old.Spec.Containers) || !kbagent.UsesSharedBinary(desired.Spec.Containers) {
+		return desired, false
+	}
+	oldIndex := slices.IndexFunc(old.Spec.InitContainers, func(container corev1.Container) bool {
+		return container.Name == kbagent.InitContainerName
+	})
+	desiredIndex := slices.IndexFunc(desired.Spec.InitContainers, func(container corev1.Container) bool {
+		return container.Name == kbagent.InitContainerName
+	})
+	if oldIndex < 0 || desiredIndex < 0 {
+		return desired, false
+	}
+	oldInit := &old.Spec.InitContainers[oldIndex]
+	desiredInit := &desired.Spec.InitContainers[desiredIndex]
+	if !kbagent.IsLegacyInitCommand(oldInit.Command) ||
+		!kbagent.IsInitCommand(desiredInit.Command) {
+		return desired, false
+	}
+
+	// If the image also changed, validate the init-kbagent image transition in
+	// isolation. Other containers may have legitimate image updates of their own;
+	// they must not prevent this init image and command from being retained as a
+	// pair on the live Pod.
+	if !intctrlutil.EqualContainerImageInSpec(oldInit.Image, desiredInit.Image) {
+		oldInitPod := &corev1.Pod{Spec: corev1.PodSpec{InitContainers: []corev1.Container{*oldInit.DeepCopy()}}}
+		desiredInitCopy := desiredInit.DeepCopy()
+		desiredInitCopy.Command = append([]string(nil), oldInit.Command...)
+		desiredInitPod := &corev1.Pod{Spec: corev1.PodSpec{InitContainers: []corev1.Container{*desiredInitCopy}}}
+		if !intctrlutil.OnlyKBManagedPodImagesChanged(oldInitPod, desiredInitPod) {
+			return desired, false
+		}
+	}
+
+	normalized := desired.DeepCopy()
+	normalized.Spec.InitContainers[desiredIndex].Image = oldInit.Image
+	normalized.Spec.InitContainers[desiredIndex].Command = append([]string(nil), oldInit.Command...)
+	return normalized, true
+}
+
 func copyRequestsNLimitsFields(container *corev1.Container) (corev1.ResourceList, corev1.ResourceList) {
 	requests := make(corev1.ResourceList)
 	limits := make(corev1.ResourceList)
@@ -417,7 +480,7 @@ func getPodUpdatePolicy(its *workloads.InstanceSet, pod *corev1.Pod) (podUpdateP
 	if index < 0 {
 		return noOpsPolicy, "", errors.Wrapf(errTemplateNotFound, "pod: %s/%s", pod.Namespace, pod.Name)
 	}
-	newPod, err := buildInstancePodByTemplate(pod.Name, templateList[index], its, getPodRevision(pod))
+	newPod, err := buildInstancePodByTemplateForUpdate(pod, templateList[index], its)
 	if err != nil {
 		return noOpsPolicy, "", err
 	}