feat(scheduler): add remote-gpu device backend for lupine-served GPUs

charts/hami/templates/_helpers.tpl

@@ -265,5 +265,16 @@ Returns a YAML list that can be used directly or converted to JSON via fromYaml
 {{- range .Values.devices.amd.customresources -}}
 {{- $resources = append $resources (dict "name" . "ignoredByScheduler" true) -}}
 {{- end -}}
+{{/* Remote GPU (lupine) resources. A client pod requests these on a node that
+     owns no GPU, so ignoredByScheduler is what keeps that node a candidate:
+     kube-scheduler skips the node-fit check and leaves placement to the
+     extender, which still gets called because the resource is listed here.
+     Both names come straight from the values the device config uses, so
+     renaming a resource cannot leave it off this list and silently reinstate
+     the node-fit check that rejects every GPU-less node. */}}
+{{- if .Values.devices.remotegpu.enabled -}}
+{{- $resources = append $resources (dict "name" .Values.devices.remotegpu.resourceCountName "ignoredByScheduler" true) -}}
+{{- $resources = append $resources (dict "name" .Values.devices.remotegpu.resourceMemoryName "ignoredByScheduler" true) -}}
+{{- end -}}
 {{- toYaml $resources -}}
 {{- end -}}

charts/hami/templates/scheduler/device-configmap.yaml

@@ -96,6 +96,12 @@ data:
       resourceCountName: iluvatar.ai/BI-V100-vgpu
       resourceMemoryName: iluvatar.ai/BI-V100.vMem
       resourceCoreName: iluvatar.ai/BI-V100.vCore
+    {{- if .Values.devices.remotegpu.enabled }}
+    remotegpu:
+      resourceCountName: {{ .Values.devices.remotegpu.resourceCountName }}
+      resourceMemoryName: {{ .Values.devices.remotegpu.resourceMemoryName }}
+      defaultPort: {{ .Values.devices.remotegpu.defaultPort }}
+    {{- end }}
     kunlun:
       resourceCountName: {{ .Values.kunlunResourceName }}
       resourceVCountName: {{ .Values.kunlunResourceVCountName }}

charts/hami/values.yaml

@@ -583,6 +583,14 @@ devices:
   nvidia:
     gpuCorePolicy: default
     libCudaLogLevel: 1
+  remotegpu:
+    # Schedule pods onto GPUs served over the network by a lupine server.
+    # Label each lupine node with hami.io/lupine-server, optionally setting the
+    # label value to the port lupine listens on.
+    enabled: false
+    defaultPort: 14833
+    resourceCountName: "nvidia.com/remote-gpu"
+    resourceMemoryName: "nvidia.com/remote-gpu-memory"
   ascend:
     enabled: false
     image: ""

cmd/scheduler/metrics.go

@@ -29,6 +29,7 @@ import (
 	klog "k8s.io/klog/v2"
 
 	"github.com/Project-HAMi/HAMi/pkg/device"
+	"github.com/Project-HAMi/HAMi/pkg/device/remotegpu"
 	versionmetrics "github.com/Project-HAMi/HAMi/pkg/metrics"
 	schedulerpkg "github.com/Project-HAMi/HAMi/pkg/scheduler"
 	"github.com/Project-HAMi/HAMi/pkg/util/leaderelection"
@@ -106,9 +107,20 @@ func (cc ClusterManagerCollector) Collect(ch chan<- prometheus.Metric) {
 	cc.collectNodeMetrics(ch, nu, legacy)
 	cc.collectQuotaMetrics(ch, legacy)
 	cc.collectContainerMetrics(ch, nu, legacy)
+	cc.collectRemoteGPUMetrics(ch)
 	cc.collectSchedulerStateMetrics(ch)
 }
 
+// remoteGPUPool reads the lupine fleet when the remote-gpu backend is
+// configured and reports nothing otherwise. Swappable for tests.
+var remoteGPUPool = func() []remotegpu.PoolDevice {
+	dev, ok := device.GetDevices()[remotegpu.RemoteGPUDevice].(*remotegpu.RemoteGPUDevices)
+	if !ok {
+		return nil
+	}
+	return dev.Inspect()
+}
+
 // collectNodeMetrics emits node-level GPU metrics (memory/core limits and
 // allocations, sharing counts, MIG instances and overview) for every device on
 // every node. AMD core values are normalized to a percentage.
@@ -210,6 +222,13 @@ func (cc ClusterManagerCollector) collectNodeMetrics(ch chan<- prometheus.Metric
 
 	for nodeID, val := range *nu {
 		for _, devs := range val.Devices.DeviceLists {
+			if devs.Device.Type == remotegpu.RemoteGPUCommonWord {
+				// The lupine pool is handed to every client node, so reporting
+				// it here would list each card once per node, under a node
+				// that does not own it. collectRemoteGPUMetrics reports the
+				// pool once, by server.
+				continue
+			}
 			coreLimit, coreAllocated := normalizeAMDCoreMetrics(devs.Device.Type, devs.Device.Totalcore, devs.Device.Usedcores)
 			if devs.Device.Mode == "mig" {
 				for _, allocation := range devs.Device.MigAllocationsInUse {
@@ -412,6 +431,45 @@ func (cc ClusterManagerCollector) collectContainerMetrics(ch chan<- prometheus.M
 	}
 }
 
+// collectRemoteGPUMetrics exports the lupine fleet once, keyed by the server
+// that owns each card rather than the client nodes that can reach it. A card
+// is handed out whole, so allocation is a flag and allocated memory is either
+// the whole card or nothing.
+func (cc ClusterManagerCollector) collectRemoteGPUMetrics(ch chan<- prometheus.Metric) {
+	labels := []string{"server", "endpoint", "device_uuid", "device_index", "device_type"}
+	memoryLimitDesc := prometheus.NewDesc(
+		"hami_remote_gpu_memory_limit_bytes",
+		"Device memory limit for a lupine-served GPU",
+		labels, nil,
+	)
+	allocatedDesc := prometheus.NewDesc(
+		"hami_remote_gpu_allocated",
+		"1 if a pod holds this lupine-served GPU, 0 if it is free",
+		labels, nil,
+	)
+	overviewDesc := prometheus.NewDesc(
+		"hami_remote_gpu_overview",
+		"Memory allocated on a lupine-served GPU, with its server and capacity",
+		[]string{"server", "endpoint", "device_uuid", "device_index", "device_cores", "device_memory_limit", "device_type"}, nil,
+	)
+	for _, pd := range remoteGPUPool() {
+		d := pd.Device
+		allocated, usedmem := 0.0, int32(0)
+		if pd.Reserved {
+			allocated, usedmem = 1, d.Devmem
+		}
+		if err := sendMetric(ch, memoryLimitDesc, prometheus.GaugeValue, mibToBytes(d.Devmem), pd.Server, pd.Endpoint, d.ID, fmt.Sprint(d.Index), d.Type); err != nil {
+			klog.V(4).Infof("Failed to send hami_remote_gpu_memory_limit_bytes metric: %v", err)
+		}
+		if err := sendMetric(ch, allocatedDesc, prometheus.GaugeValue, allocated, pd.Server, pd.Endpoint, d.ID, fmt.Sprint(d.Index), d.Type); err != nil {
+			klog.V(4).Infof("Failed to send hami_remote_gpu_allocated metric: %v", err)
+		}
+		if err := sendMetric(ch, overviewDesc, prometheus.GaugeValue, mibToBytes(usedmem), pd.Server, pd.Endpoint, d.ID, fmt.Sprint(d.Index), fmt.Sprint(d.Devcore), fmt.Sprint(d.Devmem), d.Type); err != nil {
+			klog.V(4).Infof("Failed to send hami_remote_gpu_overview metric: %v", err)
+		}
+	}
+}
+
 // collectSchedulerStateMetrics emits hami_scheduler_is_leader and
 // hami_scheduler_cache_synced as gauges (0 or 1) every scrape. These allow
 // operators to alert on leader-loss or cache-desync without log-scraping.

cmd/scheduler/metrics_test.go

@@ -27,6 +27,7 @@ import (
 	k8stypes "k8s.io/apimachinery/pkg/types"
 
 	"github.com/Project-HAMi/HAMi/pkg/device"
+	"github.com/Project-HAMi/HAMi/pkg/device/remotegpu"
 	schedulerpkg "github.com/Project-HAMi/HAMi/pkg/scheduler"
 	"github.com/Project-HAMi/HAMi/pkg/scheduler/policy"
 	"github.com/Project-HAMi/HAMi/pkg/util/leaderelection"
@@ -579,6 +580,75 @@ func TestSchedulerStateMetrics(t *testing.T) {
 	}
 }
 
+// The lupine pool is handed to every client node, so the per-node metrics
+// would list each remote card once per node. Those entries are dropped and the
+// pool is exported once, by the server that owns each card.
+func TestClusterManagerCollectorExportsRemoteGPUPoolOnce(t *testing.T) {
+	remote := func() *device.DeviceUsage {
+		return &device.DeviceUsage{ID: "gpu-a/GPU-1", Totalmem: 1024, Totalcore: 100, Type: remotegpu.RemoteGPUCommonWord, Health: true}
+	}
+	nodeUsage := map[string]*schedulerpkg.NodeUsage{
+		"cpu-1": {Devices: policy.DeviceUsageList{DeviceLists: []*policy.DeviceListsScore{
+			{Device: remote()},
+			{Device: &device.DeviceUsage{ID: "GPU-local", Totalmem: 2048, Totalcore: 100, Usedmem: 512, Type: "NVIDIA", Mode: "hami-core"}},
+		}}},
+		"cpu-2": {Devices: policy.DeviceUsageList{DeviceLists: []*policy.DeviceListsScore{{Device: remote()}}}},
+	}
+	prev := remoteGPUPool
+	remoteGPUPool = func() []remotegpu.PoolDevice {
+		return []remotegpu.PoolDevice{
+			{Server: "gpu-a", Endpoint: "10.0.0.5:14833", Reserved: true,
+				Device: device.DeviceInfo{ID: "gpu-a/GPU-1", Index: 0, Devmem: 1024, Devcore: 100, Type: remotegpu.RemoteGPUCommonWord}},
+			{Server: "gpu-a", Endpoint: "10.0.0.5:14833",
+				Device: device.DeviceInfo{ID: "gpu-a/GPU-2", Index: 1, Devmem: 1024, Devcore: 100, Type: remotegpu.RemoteGPUCommonWord}},
+		}
+	}
+	t.Cleanup(func() { remoteGPUPool = prev })
+
+	cc := ClusterManagerCollector{
+		ClusterManager: &ClusterManager{Zone: "test-zone"},
+		metricsProvider: &fakeMetricsProvider{
+			nodeUsage:    nodeUsage,
+			quotaManager: device.NewQuotaManager(),
+			podManager:   device.NewPodManager(),
+		},
+	}
+	want := `
+# HELP hami_gpu_memory_limit_bytes Device memory limit for a certain GPU
+# TYPE hami_gpu_memory_limit_bytes gauge
+hami_gpu_memory_limit_bytes{device_index="0",device_type="NVIDIA",device_uuid="GPU-local",node="cpu-1"} 2.147483648e+09
+# HELP hami_node_gpu_overview GPU overview on a certain node
+# TYPE hami_node_gpu_overview gauge
+hami_node_gpu_overview{device_cores="100",device_index="0",device_memory_limit="2048",device_type="NVIDIA",device_uuid="GPU-local",node="cpu-1"} 5.36870912e+08
+# HELP hami_remote_gpu_allocated 1 if a pod holds this lupine-served GPU, 0 if it is free
+# TYPE hami_remote_gpu_allocated gauge
+hami_remote_gpu_allocated{device_index="0",device_type="RemoteGPU",device_uuid="gpu-a/GPU-1",endpoint="10.0.0.5:14833",server="gpu-a"} 1
+hami_remote_gpu_allocated{device_index="1",device_type="RemoteGPU",device_uuid="gpu-a/GPU-2",endpoint="10.0.0.5:14833",server="gpu-a"} 0
+# HELP hami_remote_gpu_memory_limit_bytes Device memory limit for a lupine-served GPU
+# TYPE hami_remote_gpu_memory_limit_bytes gauge
+hami_remote_gpu_memory_limit_bytes{device_index="0",device_type="RemoteGPU",device_uuid="gpu-a/GPU-1",endpoint="10.0.0.5:14833",server="gpu-a"} 1.073741824e+09
+hami_remote_gpu_memory_limit_bytes{device_index="1",device_type="RemoteGPU",device_uuid="gpu-a/GPU-2",endpoint="10.0.0.5:14833",server="gpu-a"} 1.073741824e+09
+# HELP hami_remote_gpu_overview Memory allocated on a lupine-served GPU, with its server and capacity
+# TYPE hami_remote_gpu_overview gauge
+hami_remote_gpu_overview{device_cores="100",device_index="0",device_memory_limit="1024",device_type="RemoteGPU",device_uuid="gpu-a/GPU-1",endpoint="10.0.0.5:14833",server="gpu-a"} 1.073741824e+09
+hami_remote_gpu_overview{device_cores="100",device_index="1",device_memory_limit="1024",device_type="RemoteGPU",device_uuid="gpu-a/GPU-2",endpoint="10.0.0.5:14833",server="gpu-a"} 0
+`
+	if err := promtestutil.CollectAndCompare(cc, strings.NewReader(want),
+		"hami_node_gpu_overview", "hami_gpu_memory_limit_bytes",
+		"hami_remote_gpu_allocated", "hami_remote_gpu_memory_limit_bytes", "hami_remote_gpu_overview",
+	); err != nil {
+		t.Errorf("unexpected metrics:\n%s", err)
+	}
+}
+
+// Without the remote-gpu backend configured the collector reports nothing for
+// the pool and does not fail the scrape.
+func TestRemoteGPUPoolAbsentWhenBackendNotConfigured(t *testing.T) {
+	if got := remoteGPUPool(); got != nil {
+		t.Errorf("expected no pool without the backend, got %v", got)
+	}
+}
+
 // wantStr builds the expected Prometheus text for the two scheduler state
 // metrics given the expected gauge values (0.0 or 1.0).
 func wantStr(leaderVal, syncedVal float64) string {

pkg/device/nvidia/device.go

@@ -62,6 +62,10 @@ const (
 	MigMode      = "mig"
 	HamiCoreMode = "hami-core"
 	MpsMode      = "mps"
+	// RemoteMode marks a GPU this node serves over the network through lupine
+	// rather than to pods of its own. The remote-gpu backend hands such a card
+	// out cluster wide, so this backend must leave it alone.
+	RemoteMode = "remote"
 )
 
 var (
@@ -300,6 +304,29 @@ func (dev *NvidiaGPUDevices) GetNodeDevices(n corev1.Node) ([]*device.DeviceInfo
 		klog.InfoS("no nvidia gpu device found", "node", n.Name, "device annotation", devEncoded)
 		return []*device.DeviceInfo{}, errors.New("no gpu found on node")
 	}
+	// The device plugin publishes one registration annotation whatever mode it
+	// runs in, and the remote-gpu backend reads the same annotation. Claiming a
+	// card that lupine is already serving would let this node hand the same
+	// physical GPU to a second pod.
+	local := make([]*device.DeviceInfo, 0, len(nodedevices))
+	for _, nodedevice := range nodedevices {
+		if nodedevice == nil || nodedevice.Mode == RemoteMode {
+			continue
+		}
+		local = append(local, nodedevice)
+	}
+	if skipped := len(nodedevices) - len(local); skipped > 0 {
+		klog.InfoS("skipping GPUs this node serves remotely through lupine",
+			"node", n.Name, "skipped", skipped, "kept", len(local))
+	}
+	if len(local) == 0 {
+		// Zero devices rather than an error: Scheduler.register prunes a stale
+		// cache entry only on a successful empty result, and skips the pruning
+		// on error. An error would leave a node that has just switched to
+		// serving lupine still advertising its cards here.
+		return nil, nil
+	}
+	nodedevices = local
 	for idx := range nodedevices {
 		nodedevices[idx].DeviceVendor = dev.CommonWord()
 	}

pkg/device/nvidia/device_test.go

@@ -2844,6 +2844,35 @@ func TestGetNodeDevices_InvalidJSON(t *testing.T) {
 	assert.Assert(t, err != nil)
 }
 
+// A node serving its GPUs through lupine publishes the same registration
+// annotation the remote-gpu backend reads. Claiming those cards here too would
+// hand one physical GPU to two pods at once, which was reproduced on a real
+// cluster before this filter existed.
+func TestGetNodeDevices_SkipsRemotelyServedGPUs(t *testing.T) {
+	dev := &NvidiaGPUDevices{}
+	node := func(annos string) corev1.Node {
+		return corev1.Node{ObjectMeta: metav1.ObjectMeta{
+			Name:        "node-lupine",
+			Annotations: map[string]string{RegisterAnnos: annos},
+		}}
+	}
+
+	// Every card served remotely: zero devices and no error, so
+	// Scheduler.register prunes whatever this node had cached.
+	result, err := dev.GetNodeDevices(node(
+		`[{"id":"GPU-0","count":10,"devmem":81559,"devcore":100,"type":"NVIDIA-H100-80GB-HBM3","health":true,"mode":"remote"}]`))
+	assert.NilError(t, err)
+	assert.Equal(t, len(result), 0)
+
+	// A mixed node keeps the cards it still serves locally.
+	result, err = dev.GetNodeDevices(node(
+		`[{"id":"GPU-0","count":10,"devmem":81559,"devcore":100,"type":"NVIDIA-H100-80GB-HBM3","health":true,"mode":"remote"},` +
+			`{"id":"GPU-1","count":10,"devmem":81559,"devcore":100,"type":"NVIDIA-H100-80GB-HBM3","health":true,"mode":"hami-core"}]`))
+	assert.NilError(t, err)
+	assert.Equal(t, len(result), 1)
+	assert.Equal(t, result[0].ID, "GPU-1")
+}
+
 func TestGetNodeDevices_MigProfilesFromNode(t *testing.T) {
 	dev := &NvidiaGPUDevices{}
 	node := corev1.Node{

pkg/device/remotegpu/config.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2025 The HAMi Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package remotegpu
+
+// RemoteGPUConfig describes the resource names a client pod uses to ask for a
+// fraction of the lupine GPU pool, and the port lupine listens on when a
+// server node does not override it through the LupineServerLabel value.
+type RemoteGPUConfig struct {
+	ResourceCountName  string `yaml:"resourceCountName"`
+	ResourceMemoryName string `yaml:"resourceMemoryName"`
+	DefaultPort        int    `yaml:"defaultPort"`
+}