collector/slabinfo: index duplicate slab entries

/proc/slabinfo can list the same slab name more than once, for example one
cache per device instance. Labelling only by slab name therefore produces
duplicate label sets, causing the registry to reject the scrape.

Add an index label carrying each entry's position among entries sharing the
same slab name. This preserves one series per kernel cache and keeps each
entry's own counts and geometry.

The label name matches the existing ordinal index convention used by other
collectors. The ordinal follows /proc/slabinfo ordering and is not a stable
identity. It is assigned after filtering so excluded slabs do not consume an
index.

Fixes #3506

Signed-off-by: Kyle Mason <kyle@kylemason.org>

Author: machismo0311

collector/fixtures/e2e-64k-page-output.txt

@@ -3774,34 +3774,34 @@ node_scrape_collector_success{collector="zfs"} 1
 node_scrape_collector_success{collector="zoneinfo"} 1
 # HELP node_slabinfo_active_objects The number of objects that are currently active (i.e., in use).
 # TYPE node_slabinfo_active_objects gauge
-node_slabinfo_active_objects{slab="dmaengine-unmap-128"} 1206
-node_slabinfo_active_objects{slab="kmalloc-8192"} 132
-node_slabinfo_active_objects{slab="kmem_cache"} 320
-node_slabinfo_active_objects{slab="tw_sock_TCP"} 704
+node_slabinfo_active_objects{index="0",slab="dmaengine-unmap-128"} 1206
+node_slabinfo_active_objects{index="0",slab="kmalloc-8192"} 132
+node_slabinfo_active_objects{index="0",slab="kmem_cache"} 320
+node_slabinfo_active_objects{index="0",slab="tw_sock_TCP"} 704
 # HELP node_slabinfo_object_size_bytes The size of objects in this slab, in bytes.
 # TYPE node_slabinfo_object_size_bytes gauge
-node_slabinfo_object_size_bytes{slab="dmaengine-unmap-128"} 1088
-node_slabinfo_object_size_bytes{slab="kmalloc-8192"} 8192
-node_slabinfo_object_size_bytes{slab="kmem_cache"} 256
-node_slabinfo_object_size_bytes{slab="tw_sock_TCP"} 256
+node_slabinfo_object_size_bytes{index="0",slab="dmaengine-unmap-128"} 1088
+node_slabinfo_object_size_bytes{index="0",slab="kmalloc-8192"} 8192
+node_slabinfo_object_size_bytes{index="0",slab="kmem_cache"} 256
+node_slabinfo_object_size_bytes{index="0",slab="tw_sock_TCP"} 256
 # HELP node_slabinfo_objects The total number of allocated objects (i.e., objects that are both in use and not in use).
 # TYPE node_slabinfo_objects gauge
-node_slabinfo_objects{slab="dmaengine-unmap-128"} 1320
-node_slabinfo_objects{slab="kmalloc-8192"} 148
-node_slabinfo_objects{slab="kmem_cache"} 320
-node_slabinfo_objects{slab="tw_sock_TCP"} 864
+node_slabinfo_objects{index="0",slab="dmaengine-unmap-128"} 1320
+node_slabinfo_objects{index="0",slab="kmalloc-8192"} 148
+node_slabinfo_objects{index="0",slab="kmem_cache"} 320
+node_slabinfo_objects{index="0",slab="tw_sock_TCP"} 864
 # HELP node_slabinfo_objects_per_slab The number of objects stored in each slab.
 # TYPE node_slabinfo_objects_per_slab gauge
-node_slabinfo_objects_per_slab{slab="dmaengine-unmap-128"} 30
-node_slabinfo_objects_per_slab{slab="kmalloc-8192"} 4
-node_slabinfo_objects_per_slab{slab="kmem_cache"} 32
-node_slabinfo_objects_per_slab{slab="tw_sock_TCP"} 32
+node_slabinfo_objects_per_slab{index="0",slab="dmaengine-unmap-128"} 30
+node_slabinfo_objects_per_slab{index="0",slab="kmalloc-8192"} 4
+node_slabinfo_objects_per_slab{index="0",slab="kmem_cache"} 32
+node_slabinfo_objects_per_slab{index="0",slab="tw_sock_TCP"} 32
 # HELP node_slabinfo_pages_per_slab The number of pages allocated for each slab.
 # TYPE node_slabinfo_pages_per_slab gauge
-node_slabinfo_pages_per_slab{slab="dmaengine-unmap-128"} 8
-node_slabinfo_pages_per_slab{slab="kmalloc-8192"} 8
-node_slabinfo_pages_per_slab{slab="kmem_cache"} 2
-node_slabinfo_pages_per_slab{slab="tw_sock_TCP"} 2
+node_slabinfo_pages_per_slab{index="0",slab="dmaengine-unmap-128"} 8
+node_slabinfo_pages_per_slab{index="0",slab="kmalloc-8192"} 8
+node_slabinfo_pages_per_slab{index="0",slab="kmem_cache"} 2
+node_slabinfo_pages_per_slab{index="0",slab="tw_sock_TCP"} 2
 # HELP node_sockstat_FRAG6_inuse Number of FRAG6 sockets in state inuse.
 # TYPE node_sockstat_FRAG6_inuse gauge
 node_sockstat_FRAG6_inuse 0

collector/fixtures/e2e-output.txt

@@ -3806,34 +3806,34 @@ node_scrape_collector_success{collector="zfs"} 1
 node_scrape_collector_success{collector="zoneinfo"} 1
 # HELP node_slabinfo_active_objects The number of objects that are currently active (i.e., in use).
 # TYPE node_slabinfo_active_objects gauge
-node_slabinfo_active_objects{slab="dmaengine-unmap-128"} 1206
-node_slabinfo_active_objects{slab="kmalloc-8192"} 132
-node_slabinfo_active_objects{slab="kmem_cache"} 320
-node_slabinfo_active_objects{slab="tw_sock_TCP"} 704
+node_slabinfo_active_objects{index="0",slab="dmaengine-unmap-128"} 1206
+node_slabinfo_active_objects{index="0",slab="kmalloc-8192"} 132
+node_slabinfo_active_objects{index="0",slab="kmem_cache"} 320
+node_slabinfo_active_objects{index="0",slab="tw_sock_TCP"} 704
 # HELP node_slabinfo_object_size_bytes The size of objects in this slab, in bytes.
 # TYPE node_slabinfo_object_size_bytes gauge
-node_slabinfo_object_size_bytes{slab="dmaengine-unmap-128"} 1088
-node_slabinfo_object_size_bytes{slab="kmalloc-8192"} 8192
-node_slabinfo_object_size_bytes{slab="kmem_cache"} 256
-node_slabinfo_object_size_bytes{slab="tw_sock_TCP"} 256
+node_slabinfo_object_size_bytes{index="0",slab="dmaengine-unmap-128"} 1088
+node_slabinfo_object_size_bytes{index="0",slab="kmalloc-8192"} 8192
+node_slabinfo_object_size_bytes{index="0",slab="kmem_cache"} 256
+node_slabinfo_object_size_bytes{index="0",slab="tw_sock_TCP"} 256
 # HELP node_slabinfo_objects The total number of allocated objects (i.e., objects that are both in use and not in use).
 # TYPE node_slabinfo_objects gauge
-node_slabinfo_objects{slab="dmaengine-unmap-128"} 1320
-node_slabinfo_objects{slab="kmalloc-8192"} 148
-node_slabinfo_objects{slab="kmem_cache"} 320
-node_slabinfo_objects{slab="tw_sock_TCP"} 864
+node_slabinfo_objects{index="0",slab="dmaengine-unmap-128"} 1320
+node_slabinfo_objects{index="0",slab="kmalloc-8192"} 148
+node_slabinfo_objects{index="0",slab="kmem_cache"} 320
+node_slabinfo_objects{index="0",slab="tw_sock_TCP"} 864
 # HELP node_slabinfo_objects_per_slab The number of objects stored in each slab.
 # TYPE node_slabinfo_objects_per_slab gauge
-node_slabinfo_objects_per_slab{slab="dmaengine-unmap-128"} 30
-node_slabinfo_objects_per_slab{slab="kmalloc-8192"} 4
-node_slabinfo_objects_per_slab{slab="kmem_cache"} 32
-node_slabinfo_objects_per_slab{slab="tw_sock_TCP"} 32
+node_slabinfo_objects_per_slab{index="0",slab="dmaengine-unmap-128"} 30
+node_slabinfo_objects_per_slab{index="0",slab="kmalloc-8192"} 4
+node_slabinfo_objects_per_slab{index="0",slab="kmem_cache"} 32
+node_slabinfo_objects_per_slab{index="0",slab="tw_sock_TCP"} 32
 # HELP node_slabinfo_pages_per_slab The number of pages allocated for each slab.
 # TYPE node_slabinfo_pages_per_slab gauge
-node_slabinfo_pages_per_slab{slab="dmaengine-unmap-128"} 8
-node_slabinfo_pages_per_slab{slab="kmalloc-8192"} 8
-node_slabinfo_pages_per_slab{slab="kmem_cache"} 2
-node_slabinfo_pages_per_slab{slab="tw_sock_TCP"} 2
+node_slabinfo_pages_per_slab{index="0",slab="dmaengine-unmap-128"} 8
+node_slabinfo_pages_per_slab{index="0",slab="kmalloc-8192"} 8
+node_slabinfo_pages_per_slab{index="0",slab="kmem_cache"} 2
+node_slabinfo_pages_per_slab{index="0",slab="tw_sock_TCP"} 2
 # HELP node_sockstat_FRAG6_inuse Number of FRAG6 sockets in state inuse.
 # TYPE node_sockstat_FRAG6_inuse gauge
 node_sockstat_FRAG6_inuse 0

collector/slabinfo_linux.go

@@ -18,6 +18,7 @@ package collector
 import (
 	"fmt"
 	"log/slog"
+	"strconv"
 
 	"github.com/alecthomas/kingpin/v2"
 	"github.com/prometheus/client_golang/prometheus"
@@ -50,7 +51,7 @@ func NewSlabinfoCollector(logger *slog.Logger) (Collector, error) {
 	return &slabinfoCollector{logger: logger,
 		fs:             fs,
 		subsystem:      "slabinfo",
-		labels:         []string{"slab"},
+		labels:         []string{"slab", "index"},
 		slabNameFilter: newDeviceFilter(*slabNameExclude, *slabNameInclude),
 	}, nil
 }
@@ -61,71 +62,84 @@ func (c *slabinfoCollector) Update(ch chan<- prometheus.Metric) error {
 		return fmt.Errorf("couldn't get %s: %w", c.subsystem, err)
 	}
 
+	// /proc/slabinfo can list the same slab name more than once, for example one
+	// cache per device instance. The kernel permits this: the duplicate-name check
+	// in kmem_cache_sanity_check() only WARNs, and is compiled out unless
+	// CONFIG_DEBUG_VM is set. Labelling by name alone therefore produces duplicate
+	// label sets, which fails the entire scrape.
+	//
+	// Each entry is given its position among the entries sharing its name, so every
+	// cache keeps its own series and its own geometry. The ordinal reflects the
+	// order of /proc/slabinfo and is not a stable identity: if a cache is created
+	// or destroyed, subsequent entries shift.
+	seen := make(map[string]int, len(slabinfo.Slabs))
 	for _, slab := range slabinfo.Slabs {
 		if c.slabNameFilter.ignored(slab.Name) {
 			continue
 		}
-		ch <- c.activeObjects(slab.Name, slab.ObjActive)
-		ch <- c.objects(slab.Name, slab.ObjNum)
-		ch <- c.objectSizeBytes(slab.Name, slab.ObjSize)
-		ch <- c.objectsPerSlab(slab.Name, slab.ObjPerSlab)
-		ch <- c.pagesPerSlab(slab.Name, slab.PagesPerSlab)
+		index := strconv.Itoa(seen[slab.Name])
+		seen[slab.Name]++
+		ch <- c.activeObjects(slab.Name, index, slab.ObjActive)
+		ch <- c.objects(slab.Name, index, slab.ObjNum)
+		ch <- c.objectSizeBytes(slab.Name, index, slab.ObjSize)
+		ch <- c.objectsPerSlab(slab.Name, index, slab.ObjPerSlab)
+		ch <- c.pagesPerSlab(slab.Name, index, slab.PagesPerSlab)
 	}
 
 	return nil
 }
 
-func (c *slabinfoCollector) activeObjects(label string, val int64) prometheus.Metric {
+func (c *slabinfoCollector) activeObjects(label, index string, val int64) prometheus.Metric {
 	desc := prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, c.subsystem, "active_objects"),
 		"The number of objects that are currently active (i.e., in use).",
 		c.labels, nil)
 
 	return prometheus.MustNewConstMetric(
-		desc, prometheus.GaugeValue, float64(val), label,
+		desc, prometheus.GaugeValue, float64(val), label, index,
 	)
 }
 
-func (c *slabinfoCollector) objects(label string, val int64) prometheus.Metric {
+func (c *slabinfoCollector) objects(label, index string, val int64) prometheus.Metric {
 	desc := prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, c.subsystem, "objects"),
 		"The total number of allocated objects (i.e., objects that are both in use and not in use).",
 		c.labels, nil)
 
 	return prometheus.MustNewConstMetric(
-		desc, prometheus.GaugeValue, float64(val), label,
+		desc, prometheus.GaugeValue, float64(val), label, index,
 	)
 }
 
-func (c *slabinfoCollector) objectSizeBytes(label string, val int64) prometheus.Metric {
+func (c *slabinfoCollector) objectSizeBytes(label, index string, val int64) prometheus.Metric {
 	desc := prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, c.subsystem, "object_size_bytes"),
 		"The size of objects in this slab, in bytes.",
 		c.labels, nil)
 
 	return prometheus.MustNewConstMetric(
-		desc, prometheus.GaugeValue, float64(val), label,
+		desc, prometheus.GaugeValue, float64(val), label, index,
 	)
 }
 
-func (c *slabinfoCollector) objectsPerSlab(label string, val int64) prometheus.Metric {
+func (c *slabinfoCollector) objectsPerSlab(label, index string, val int64) prometheus.Metric {
 	desc := prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, c.subsystem, "objects_per_slab"),
 		"The number of objects stored in each slab.",
 		c.labels, nil)
 
 	return prometheus.MustNewConstMetric(
-		desc, prometheus.GaugeValue, float64(val), label,
+		desc, prometheus.GaugeValue, float64(val), label, index,
 	)
 }
 
-func (c *slabinfoCollector) pagesPerSlab(label string, val int64) prometheus.Metric {
+func (c *slabinfoCollector) pagesPerSlab(label, index string, val int64) prometheus.Metric {
 	desc := prometheus.NewDesc(
 		prometheus.BuildFQName(namespace, c.subsystem, "pages_per_slab"),
 		"The number of pages allocated for each slab.",
 		c.labels, nil)
 
 	return prometheus.MustNewConstMetric(
-		desc, prometheus.GaugeValue, float64(val), label,
+		desc, prometheus.GaugeValue, float64(val), label, index,
 	)
 }

collector/slabinfo_linux_test.go

@@ -0,0 +1,169 @@
+// Copyright 2026 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !noslabinfo
+
+package collector
+
+import (
+	"log/slog"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const slabinfoHeader = `slabinfo - version: 2.1
+# name            <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab> : tunables <limit> <batchcount> <sharedfactor> : slabdata <active_slabs> <num_slabs> <sharedavail>
+`
+
+// writeSlabinfo writes body into a temporary procfs and points the collector at
+// it for the duration of the test. procPath is a package-global flag value, so
+// it is restored afterwards to keep these tests independent of execution order.
+// These tests must not call t.Parallel() for the same reason.
+func writeSlabinfo(t *testing.T, body string) {
+	t.Helper()
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "slabinfo"), []byte(slabinfoHeader+body), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	orig := *procPath
+	t.Cleanup(func() { *procPath = orig })
+	*procPath = dir
+}
+
+// gatherSlabinfo collects through a registry, which is what surfaces duplicate
+// label sets, and keys the result by "metric/slab/index".
+func gatherSlabinfo(t *testing.T) map[string]float64 {
+	t.Helper()
+	c, err := NewSlabinfoCollector(slog.New(slog.NewTextHandler(os.Stderr, nil)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	reg := prometheus.NewPedanticRegistry()
+	if err := reg.Register(prometheus.CollectorFunc(func(ch chan<- prometheus.Metric) {
+		if err := c.Update(ch); err != nil {
+			t.Errorf("Update failed: %v", err)
+		}
+	})); err != nil {
+		t.Fatal(err)
+	}
+	mfs, err := reg.Gather()
+	if err != nil {
+		t.Fatalf("Gather failed: %v", err)
+	}
+	got := map[string]float64{}
+	for _, mf := range mfs {
+		for _, m := range mf.Metric {
+			var name, index string
+			for _, l := range m.Label {
+				switch l.GetName() {
+				case "slab":
+					name = l.GetValue()
+				case "index":
+					index = l.GetValue()
+				}
+			}
+			got[mf.GetName()+"/"+name+"/"+index] = m.GetGauge().GetValue()
+		}
+	}
+	return got
+}
+
+func checkSlabinfo(t *testing.T, got map[string]float64, want map[string]float64) {
+	t.Helper()
+	for key, w := range want {
+		if g, ok := got[key]; !ok {
+			t.Errorf("%s: missing", key)
+		} else if g != w {
+			t.Errorf("%s: got %v, want %v", key, g, w)
+		}
+	}
+}
+
+// A slab name may appear more than once, for example one cache per device
+// instance. Each entry keeps its own series, distinguished by the index label.
+func TestSlabinfoDuplicateSlabNames(t *testing.T) {
+	writeSlabinfo(t, `kmem_cache           320    320    256   32    2 : tunables    0    0    0 : slabdata     10     10      0
+mlx5_fs_ftes          44     44    736   44    8 : tunables    0    0    0 : slabdata      1      1      0
+mlx5_fs_fgs           42     42    776   42    8 : tunables    0    0    0 : slabdata      1      1      0
+mlx5_fs_ftes         396    396    736   44    8 : tunables    0    0    0 : slabdata      9      9      0
+mlx5_fs_fgs           84     84    776   42    8 : tunables    0    0    0 : slabdata      2      2      0
+`)
+	checkSlabinfo(t, gatherSlabinfo(t), map[string]float64{
+		"node_slabinfo_active_objects/mlx5_fs_ftes/0":    44,
+		"node_slabinfo_active_objects/mlx5_fs_ftes/1":    396,
+		"node_slabinfo_objects/mlx5_fs_ftes/1":           396,
+		"node_slabinfo_active_objects/mlx5_fs_fgs/0":     42,
+		"node_slabinfo_active_objects/mlx5_fs_fgs/1":     84,
+		"node_slabinfo_object_size_bytes/mlx5_fs_ftes/1": 736,
+		"node_slabinfo_objects_per_slab/mlx5_fs_ftes/1":  44,
+		"node_slabinfo_pages_per_slab/mlx5_fs_ftes/1":    8,
+		"node_slabinfo_active_objects/kmem_cache/0":      320,
+	})
+}
+
+// Slabs with distinct names each get index 0.
+func TestSlabinfoDistinctSlabNames(t *testing.T) {
+	writeSlabinfo(t, `tw_sock_TCP          704    864    256   32    2 : tunables    0    0    0 : slabdata     27     27      0
+kmem_cache           320    320    256   32    2 : tunables    0    0    0 : slabdata     10     10      0
+`)
+	checkSlabinfo(t, gatherSlabinfo(t), map[string]float64{
+		"node_slabinfo_active_objects/tw_sock_TCP/0": 704,
+		"node_slabinfo_objects/tw_sock_TCP/0":        864,
+		"node_slabinfo_active_objects/kmem_cache/0":  320,
+		"node_slabinfo_objects/kmem_cache/0":         320,
+	})
+}
+
+// The kernel does not guarantee that same-named caches share geometry: the
+// duplicate-name check in kmem_cache_sanity_check() only WARNs and is compiled
+// out without CONFIG_DEBUG_VM, and object_size is per-cache. Indexing keeps each
+// cache's own geometry, so no representative value has to be chosen.
+func TestSlabinfoDuplicateSlabNamesDifferentGeometry(t *testing.T) {
+	writeSlabinfo(t, `weird_cache          10     10    128   32    2 : tunables    0    0    0 : slabdata      1      1      0
+weird_cache          20     20    256   16    4 : tunables    0    0    0 : slabdata      2      2      0
+`)
+	checkSlabinfo(t, gatherSlabinfo(t), map[string]float64{
+		"node_slabinfo_active_objects/weird_cache/0":    10,
+		"node_slabinfo_object_size_bytes/weird_cache/0": 128,
+		"node_slabinfo_objects_per_slab/weird_cache/0":  32,
+		"node_slabinfo_pages_per_slab/weird_cache/0":    2,
+		"node_slabinfo_active_objects/weird_cache/1":    20,
+		"node_slabinfo_object_size_bytes/weird_cache/1": 256,
+		"node_slabinfo_objects_per_slab/weird_cache/1":  16,
+		"node_slabinfo_pages_per_slab/weird_cache/1":    4,
+	})
+}
+
+// Excluded slabs must not consume an index, so the ordinals of the remaining
+// entries stay contiguous.
+func TestSlabinfoIndexIgnoresFilteredSlabs(t *testing.T) {
+	orig := *slabNameExclude
+	t.Cleanup(func() { *slabNameExclude = orig })
+	*slabNameExclude = "^drop_me$"
+
+	writeSlabinfo(t, `dup_cache            10     10    128   32    2 : tunables    0    0    0 : slabdata      1      1      0
+drop_me              99     99    999   99    9 : tunables    0    0    0 : slabdata      9      9      0
+dup_cache            20     20    128   32    2 : tunables    0    0    0 : slabdata      2      2      0
+`)
+	got := gatherSlabinfo(t)
+	checkSlabinfo(t, got, map[string]float64{
+		"node_slabinfo_active_objects/dup_cache/0": 10,
+		"node_slabinfo_active_objects/dup_cache/1": 20,
+	})
+	if _, ok := got["node_slabinfo_active_objects/drop_me/0"]; ok {
+		t.Error("excluded slab was collected")
+	}
+}