We have removed the partial slab usage from allocation paths. Now remove
the whole config option and associated code.
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
---
mm/Kconfig | 11 ---
mm/slab.h | 29 ------
mm/slub.c | 321 ++++---------------------------------------------------------
3 files changed, 19 insertions(+), 342 deletions(-)
diff --git a/mm/Kconfig b/mm/Kconfig
index bd0ea5454af8..08593674cd20 100644
--- a/mm/Kconfig
+++ b/mm/Kconfig
@@ -247,17 +247,6 @@ config SLUB_STATS
out which slabs are relevant to a particular load.
Try running: slabinfo -DA
-config SLUB_CPU_PARTIAL
- default y
- depends on SMP && !SLUB_TINY
- bool "Enable per cpu partial caches"
- help
- Per cpu partial caches accelerate objects allocation and freeing
- that is local to a processor at the price of more indeterminism
- in the latency of the free. On overflow these caches will be cleared
- which requires the taking of locks that may cause latency spikes.
- Typically one would choose no for a realtime system.
-
config RANDOM_KMALLOC_CACHES
default n
depends on !SLUB_TINY
diff --git a/mm/slab.h b/mm/slab.h
index cb48ce5014ba..e77260720994 100644
--- a/mm/slab.h
+++ b/mm/slab.h
@@ -77,12 +77,6 @@ struct slab {
struct llist_node llnode;
void *flush_freelist;
};
-#ifdef CONFIG_SLUB_CPU_PARTIAL
- struct {
- struct slab *next;
- int slabs; /* Nr of slabs left */
- };
-#endif
};
/* Double-word boundary */
struct freelist_counters;
@@ -188,23 +182,6 @@ static inline size_t slab_size(const struct slab *slab)
return PAGE_SIZE << slab_order(slab);
}
-#ifdef CONFIG_SLUB_CPU_PARTIAL
-#define slub_percpu_partial(c) ((c)->partial)
-
-#define slub_set_percpu_partial(c, p) \
-({ \
- slub_percpu_partial(c) = (p)->next; \
-})
-
-#define slub_percpu_partial_read_once(c) READ_ONCE(slub_percpu_partial(c))
-#else
-#define slub_percpu_partial(c) NULL
-
-#define slub_set_percpu_partial(c, p)
-
-#define slub_percpu_partial_read_once(c) NULL
-#endif // CONFIG_SLUB_CPU_PARTIAL
-
/*
* Word size structure that can be atomically updated or read and that
* contains both the order and the number of objects that a slab of the
@@ -228,12 +205,6 @@ struct kmem_cache {
unsigned int object_size; /* Object size without metadata */
struct reciprocal_value reciprocal_size;
unsigned int offset; /* Free pointer offset */
-#ifdef CONFIG_SLUB_CPU_PARTIAL
- /* Number of per cpu partial objects to keep around */
- unsigned int cpu_partial;
- /* Number of per cpu partial slabs to keep around */
- unsigned int cpu_partial_slabs;
-#endif
unsigned int sheaf_capacity;
struct kmem_cache_order_objects oo;
diff --git a/mm/slub.c b/mm/slub.c
index 698c0d940f06..6b1280f7900a 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -263,15 +263,6 @@ void *fixup_red_left(struct kmem_cache *s, void *p)
return p;
}
-static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
-{
-#ifdef CONFIG_SLUB_CPU_PARTIAL
- return !kmem_cache_debug(s);
-#else
- return false;
-#endif
-}
-
/*
* Issues still to be resolved:
*
@@ -426,9 +417,6 @@ struct freelist_tid {
struct kmem_cache_cpu {
struct freelist_tid;
struct slab *slab; /* The slab from which we are allocating */
-#ifdef CONFIG_SLUB_CPU_PARTIAL
- struct slab *partial; /* Partially allocated slabs */
-#endif
local_trylock_t lock; /* Protects the fields above */
#ifdef CONFIG_SLUB_STATS
unsigned int stat[NR_SLUB_STAT_ITEMS];
@@ -673,29 +661,6 @@ static inline unsigned int oo_objects(struct kmem_cache_order_objects x)
return x.x & OO_MASK;
}
-#ifdef CONFIG_SLUB_CPU_PARTIAL
-static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
-{
- unsigned int nr_slabs;
-
- s->cpu_partial = nr_objects;
-
- /*
- * We take the number of objects but actually limit the number of
- * slabs on the per cpu partial list, in order to limit excessive
- * growth of the list. For simplicity we assume that the slabs will
- * be half-full.
- */
- nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo));
- s->cpu_partial_slabs = nr_slabs;
-}
-#elif defined(SLAB_SUPPORTS_SYSFS)
-static inline void
-slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
-{
-}
-#endif /* CONFIG_SLUB_CPU_PARTIAL */
-
/*
* If network-based swap is enabled, slub must keep track of whether memory
* were allocated from pfmemalloc reserves.
@@ -3474,12 +3439,6 @@ static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab,
return object;
}
-#ifdef CONFIG_SLUB_CPU_PARTIAL
-static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain);
-#else
-static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab,
- int drain) { }
-#endif
static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags);
static bool get_partial_node_bulk(struct kmem_cache *s,
@@ -3898,131 +3857,6 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
#define local_unlock_cpu_slab(s, flags) \
local_unlock_irqrestore(&(s)->cpu_slab->lock, flags)
-#ifdef CONFIG_SLUB_CPU_PARTIAL
-static void __put_partials(struct kmem_cache *s, struct slab *partial_slab)
-{
- struct kmem_cache_node *n = NULL, *n2 = NULL;
- struct slab *slab, *slab_to_discard = NULL;
- unsigned long flags = 0;
-
- while (partial_slab) {
- slab = partial_slab;
- partial_slab = slab->next;
-
- n2 = get_node(s, slab_nid(slab));
- if (n != n2) {
- if (n)
- spin_unlock_irqrestore(&n->list_lock, flags);
-
- n = n2;
- spin_lock_irqsave(&n->list_lock, flags);
- }
-
- if (unlikely(!slab->inuse && n->nr_partial >= s->min_partial)) {
- slab->next = slab_to_discard;
- slab_to_discard = slab;
- } else {
- add_partial(n, slab, DEACTIVATE_TO_TAIL);
- stat(s, FREE_ADD_PARTIAL);
- }
- }
-
- if (n)
- spin_unlock_irqrestore(&n->list_lock, flags);
-
- while (slab_to_discard) {
- slab = slab_to_discard;
- slab_to_discard = slab_to_discard->next;
-
- stat(s, DEACTIVATE_EMPTY);
- discard_slab(s, slab);
- stat(s, FREE_SLAB);
- }
-}
-
-/*
- * Put all the cpu partial slabs to the node partial list.
- */
-static void put_partials(struct kmem_cache *s)
-{
- struct slab *partial_slab;
- unsigned long flags;
-
- local_lock_irqsave(&s->cpu_slab->lock, flags);
- partial_slab = this_cpu_read(s->cpu_slab->partial);
- this_cpu_write(s->cpu_slab->partial, NULL);
- local_unlock_irqrestore(&s->cpu_slab->lock, flags);
-
- if (partial_slab)
- __put_partials(s, partial_slab);
-}
-
-static void put_partials_cpu(struct kmem_cache *s,
- struct kmem_cache_cpu *c)
-{
- struct slab *partial_slab;
-
- partial_slab = slub_percpu_partial(c);
- c->partial = NULL;
-
- if (partial_slab)
- __put_partials(s, partial_slab);
-}
-
-/*
- * Put a slab into a partial slab slot if available.
- *
- * If we did not find a slot then simply move all the partials to the
- * per node partial list.
- */
-static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain)
-{
- struct slab *oldslab;
- struct slab *slab_to_put = NULL;
- unsigned long flags;
- int slabs = 0;
-
- local_lock_cpu_slab(s, flags);
-
- oldslab = this_cpu_read(s->cpu_slab->partial);
-
- if (oldslab) {
- if (drain && oldslab->slabs >= s->cpu_partial_slabs) {
- /*
- * Partial array is full. Move the existing set to the
- * per node partial list. Postpone the actual unfreezing
- * outside of the critical section.
- */
- slab_to_put = oldslab;
- oldslab = NULL;
- } else {
- slabs = oldslab->slabs;
- }
- }
-
- slabs++;
-
- slab->slabs = slabs;
- slab->next = oldslab;
-
- this_cpu_write(s->cpu_slab->partial, slab);
-
- local_unlock_cpu_slab(s, flags);
-
- if (slab_to_put) {
- __put_partials(s, slab_to_put);
- stat(s, CPU_PARTIAL_DRAIN);
- }
-}
-
-#else /* CONFIG_SLUB_CPU_PARTIAL */
-
-static inline void put_partials(struct kmem_cache *s) { }
-static inline void put_partials_cpu(struct kmem_cache *s,
- struct kmem_cache_cpu *c) { }
-
-#endif /* CONFIG_SLUB_CPU_PARTIAL */
-
static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
{
unsigned long flags;
@@ -4060,8 +3894,6 @@ static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
deactivate_slab(s, slab, freelist);
stat(s, CPUSLAB_FLUSH);
}
-
- put_partials_cpu(s, c);
}
static inline void flush_this_cpu_slab(struct kmem_cache *s)
@@ -4070,15 +3902,13 @@ static inline void flush_this_cpu_slab(struct kmem_cache *s)
if (c->slab)
flush_slab(s, c);
-
- put_partials(s);
}
static bool has_cpu_slab(int cpu, struct kmem_cache *s)
{
struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
- return c->slab || slub_percpu_partial(c);
+ return c->slab;
}
static bool has_pcs_used(int cpu, struct kmem_cache *s)
@@ -5646,13 +5476,6 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
return;
}
- /*
- * It is enough to test IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) below
- * instead of kmem_cache_has_cpu_partial(s), because kmem_cache_debug(s)
- * is the only other reason it can be false, and it is already handled
- * above.
- */
-
do {
if (unlikely(n)) {
spin_unlock_irqrestore(&n->list_lock, flags);
@@ -5677,26 +5500,19 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
* Unless it's frozen.
*/
if ((!new.inuse || was_full) && !was_frozen) {
+
+ n = get_node(s, slab_nid(slab));
/*
- * If slab becomes non-full and we have cpu partial
- * lists, we put it there unconditionally to avoid
- * taking the list_lock. Otherwise we need it.
+ * Speculatively acquire the list_lock.
+ * If the cmpxchg does not succeed then we may
+ * drop the list_lock without any processing.
+ *
+ * Otherwise the list_lock will synchronize with
+ * other processors updating the list of slabs.
*/
- if (!(IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && was_full)) {
-
- n = get_node(s, slab_nid(slab));
- /*
- * Speculatively acquire the list_lock.
- * If the cmpxchg does not succeed then we may
- * drop the list_lock without any processing.
- *
- * Otherwise the list_lock will synchronize with
- * other processors updating the list of slabs.
- */
- spin_lock_irqsave(&n->list_lock, flags);
-
- on_node_partial = slab_test_node_partial(slab);
- }
+ spin_lock_irqsave(&n->list_lock, flags);
+
+ on_node_partial = slab_test_node_partial(slab);
}
} while (!slab_update_freelist(s, slab, &old, &new, "__slab_free"));
@@ -5709,13 +5525,6 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
* activity can be necessary.
*/
stat(s, FREE_FROZEN);
- } else if (IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && was_full) {
- /*
- * If we started with a full slab then put it onto the
- * per cpu partial list.
- */
- put_cpu_partial(s, slab, 1);
- stat(s, CPU_PARTIAL_FREE);
}
/*
@@ -5744,10 +5553,9 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
/*
* Objects left in the slab. If it was not on the partial list before
- * then add it. This can only happen when cache has no per cpu partial
- * list otherwise we would have put it there.
+ * then add it.
*/
- if (!IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && unlikely(was_full)) {
+ if (unlikely(was_full)) {
add_partial(n, slab, DEACTIVATE_TO_TAIL);
stat(s, FREE_ADD_PARTIAL);
}
@@ -6396,8 +6204,8 @@ static __always_inline void do_slab_free(struct kmem_cache *s,
if (unlikely(!allow_spin)) {
/*
* __slab_free() can locklessly cmpxchg16 into a slab,
- * but then it might need to take spin_lock or local_lock
- * in put_cpu_partial() for further processing.
+ * but then it might need to take spin_lock
+ * for further processing.
* Avoid the complexity and simply add to a deferred list.
*/
defer_free(s, head);
@@ -7707,39 +7515,6 @@ static int init_kmem_cache_nodes(struct kmem_cache *s)
return 1;
}
-static void set_cpu_partial(struct kmem_cache *s)
-{
-#ifdef CONFIG_SLUB_CPU_PARTIAL
- unsigned int nr_objects;
-
- /*
- * cpu_partial determined the maximum number of objects kept in the
- * per cpu partial lists of a processor.
- *
- * Per cpu partial lists mainly contain slabs that just have one
- * object freed. If they are used for allocation then they can be
- * filled up again with minimal effort. The slab will never hit the
- * per node partial lists and therefore no locking will be required.
- *
- * For backwards compatibility reasons, this is determined as number
- * of objects, even though we now limit maximum number of pages, see
- * slub_set_cpu_partial()
- */
- if (!kmem_cache_has_cpu_partial(s))
- nr_objects = 0;
- else if (s->size >= PAGE_SIZE)
- nr_objects = 6;
- else if (s->size >= 1024)
- nr_objects = 24;
- else if (s->size >= 256)
- nr_objects = 52;
- else
- nr_objects = 120;
-
- slub_set_cpu_partial(s, nr_objects);
-#endif
-}
-
static unsigned int calculate_sheaf_capacity(struct kmem_cache *s,
struct kmem_cache_args *args)
@@ -8595,8 +8370,6 @@ int do_kmem_cache_create(struct kmem_cache *s, const char *name,
s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2);
s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial);
- set_cpu_partial(s);
-
s->cpu_sheaves = alloc_percpu(struct slub_percpu_sheaves);
if (!s->cpu_sheaves) {
err = -ENOMEM;
@@ -8960,20 +8733,6 @@ static ssize_t show_slab_objects(struct kmem_cache *s,
total += x;
nodes[node] += x;
-#ifdef CONFIG_SLUB_CPU_PARTIAL
- slab = slub_percpu_partial_read_once(c);
- if (slab) {
- node = slab_nid(slab);
- if (flags & SO_TOTAL)
- WARN_ON_ONCE(1);
- else if (flags & SO_OBJECTS)
- WARN_ON_ONCE(1);
- else
- x = data_race(slab->slabs);
- total += x;
- nodes[node] += x;
- }
-#endif
}
}
@@ -9108,12 +8867,7 @@ SLAB_ATTR(min_partial);
static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
{
- unsigned int nr_partial = 0;
-#ifdef CONFIG_SLUB_CPU_PARTIAL
- nr_partial = s->cpu_partial;
-#endif
-
- return sysfs_emit(buf, "%u\n", nr_partial);
+ return sysfs_emit(buf, "0\n");
}
static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
@@ -9125,11 +8879,9 @@ static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
err = kstrtouint(buf, 10, &objects);
if (err)
return err;
- if (objects && !kmem_cache_has_cpu_partial(s))
+ if (objects)
return -EINVAL;
- slub_set_cpu_partial(s, objects);
- flush_all(s);
return length;
}
SLAB_ATTR(cpu_partial);
@@ -9168,42 +8920,7 @@ SLAB_ATTR_RO(objects_partial);
static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
{
- int objects = 0;
- int slabs = 0;
- int cpu __maybe_unused;
- int len = 0;
-
-#ifdef CONFIG_SLUB_CPU_PARTIAL
- for_each_online_cpu(cpu) {
- struct slab *slab;
-
- slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
-
- if (slab)
- slabs += data_race(slab->slabs);
- }
-#endif
-
- /* Approximate half-full slabs, see slub_set_cpu_partial() */
- objects = (slabs * oo_objects(s->oo)) / 2;
- len += sysfs_emit_at(buf, len, "%d(%d)", objects, slabs);
-
-#ifdef CONFIG_SLUB_CPU_PARTIAL
- for_each_online_cpu(cpu) {
- struct slab *slab;
-
- slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
- if (slab) {
- slabs = data_race(slab->slabs);
- objects = (slabs * oo_objects(s->oo)) / 2;
- len += sysfs_emit_at(buf, len, " C%d=%d(%d)",
- cpu, objects, slabs);
- }
- }
-#endif
- len += sysfs_emit_at(buf, len, "\n");
-
- return len;
+ return sysfs_emit(buf, "0(0)\n");
}
SLAB_ATTR_RO(slabs_cpu_partial);
--
2.52.0On Fri, Jan 16, 2026 at 2:40 PM Vlastimil Babka <vbabka@suse.cz> wrote:
>
> We have removed the partial slab usage from allocation paths. Now remove
> the whole config option and associated code.
>
> Reviewed-by: Suren Baghdasaryan <surenb@google.com>
I did? Well, if so, I missed some remaining mentions about cpu partial caches:
- slub.c has several hits on "cpu partial" in the comments.
- there is one hit on "put_cpu_partial" in slub.c in the comments.
Should we also update Documentation/ABI/testing/sysfs-kernel-slab to
say that from now on cpu_partial control always reads 0?
Once addressed, please feel free to keep my Reviewed-by.
> Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
> ---
> mm/Kconfig | 11 ---
> mm/slab.h | 29 ------
> mm/slub.c | 321 ++++---------------------------------------------------------
> 3 files changed, 19 insertions(+), 342 deletions(-)
>
> diff --git a/mm/Kconfig b/mm/Kconfig
> index bd0ea5454af8..08593674cd20 100644
> --- a/mm/Kconfig
> +++ b/mm/Kconfig
> @@ -247,17 +247,6 @@ config SLUB_STATS
> out which slabs are relevant to a particular load.
> Try running: slabinfo -DA
>
> -config SLUB_CPU_PARTIAL
> - default y
> - depends on SMP && !SLUB_TINY
> - bool "Enable per cpu partial caches"
> - help
> - Per cpu partial caches accelerate objects allocation and freeing
> - that is local to a processor at the price of more indeterminism
> - in the latency of the free. On overflow these caches will be cleared
> - which requires the taking of locks that may cause latency spikes.
> - Typically one would choose no for a realtime system.
> -
> config RANDOM_KMALLOC_CACHES
> default n
> depends on !SLUB_TINY
> diff --git a/mm/slab.h b/mm/slab.h
> index cb48ce5014ba..e77260720994 100644
> --- a/mm/slab.h
> +++ b/mm/slab.h
> @@ -77,12 +77,6 @@ struct slab {
> struct llist_node llnode;
> void *flush_freelist;
> };
> -#ifdef CONFIG_SLUB_CPU_PARTIAL
> - struct {
> - struct slab *next;
> - int slabs; /* Nr of slabs left */
> - };
> -#endif
> };
> /* Double-word boundary */
> struct freelist_counters;
> @@ -188,23 +182,6 @@ static inline size_t slab_size(const struct slab *slab)
> return PAGE_SIZE << slab_order(slab);
> }
>
> -#ifdef CONFIG_SLUB_CPU_PARTIAL
> -#define slub_percpu_partial(c) ((c)->partial)
> -
> -#define slub_set_percpu_partial(c, p) \
> -({ \
> - slub_percpu_partial(c) = (p)->next; \
> -})
> -
> -#define slub_percpu_partial_read_once(c) READ_ONCE(slub_percpu_partial(c))
> -#else
> -#define slub_percpu_partial(c) NULL
> -
> -#define slub_set_percpu_partial(c, p)
> -
> -#define slub_percpu_partial_read_once(c) NULL
> -#endif // CONFIG_SLUB_CPU_PARTIAL
> -
> /*
> * Word size structure that can be atomically updated or read and that
> * contains both the order and the number of objects that a slab of the
> @@ -228,12 +205,6 @@ struct kmem_cache {
> unsigned int object_size; /* Object size without metadata */
> struct reciprocal_value reciprocal_size;
> unsigned int offset; /* Free pointer offset */
> -#ifdef CONFIG_SLUB_CPU_PARTIAL
> - /* Number of per cpu partial objects to keep around */
> - unsigned int cpu_partial;
> - /* Number of per cpu partial slabs to keep around */
> - unsigned int cpu_partial_slabs;
> -#endif
> unsigned int sheaf_capacity;
> struct kmem_cache_order_objects oo;
>
> diff --git a/mm/slub.c b/mm/slub.c
> index 698c0d940f06..6b1280f7900a 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -263,15 +263,6 @@ void *fixup_red_left(struct kmem_cache *s, void *p)
> return p;
> }
>
> -static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
> -{
> -#ifdef CONFIG_SLUB_CPU_PARTIAL
> - return !kmem_cache_debug(s);
> -#else
> - return false;
> -#endif
> -}
> -
> /*
> * Issues still to be resolved:
> *
> @@ -426,9 +417,6 @@ struct freelist_tid {
> struct kmem_cache_cpu {
> struct freelist_tid;
> struct slab *slab; /* The slab from which we are allocating */
> -#ifdef CONFIG_SLUB_CPU_PARTIAL
> - struct slab *partial; /* Partially allocated slabs */
> -#endif
> local_trylock_t lock; /* Protects the fields above */
> #ifdef CONFIG_SLUB_STATS
> unsigned int stat[NR_SLUB_STAT_ITEMS];
> @@ -673,29 +661,6 @@ static inline unsigned int oo_objects(struct kmem_cache_order_objects x)
> return x.x & OO_MASK;
> }
>
> -#ifdef CONFIG_SLUB_CPU_PARTIAL
> -static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
> -{
> - unsigned int nr_slabs;
> -
> - s->cpu_partial = nr_objects;
> -
> - /*
> - * We take the number of objects but actually limit the number of
> - * slabs on the per cpu partial list, in order to limit excessive
> - * growth of the list. For simplicity we assume that the slabs will
> - * be half-full.
> - */
> - nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo));
> - s->cpu_partial_slabs = nr_slabs;
> -}
> -#elif defined(SLAB_SUPPORTS_SYSFS)
> -static inline void
> -slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
> -{
> -}
> -#endif /* CONFIG_SLUB_CPU_PARTIAL */
> -
> /*
> * If network-based swap is enabled, slub must keep track of whether memory
> * were allocated from pfmemalloc reserves.
> @@ -3474,12 +3439,6 @@ static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab,
> return object;
> }
>
> -#ifdef CONFIG_SLUB_CPU_PARTIAL
> -static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain);
> -#else
> -static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab,
> - int drain) { }
> -#endif
> static inline bool pfmemalloc_match(struct slab *slab, gfp_t gfpflags);
>
> static bool get_partial_node_bulk(struct kmem_cache *s,
> @@ -3898,131 +3857,6 @@ static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
> #define local_unlock_cpu_slab(s, flags) \
> local_unlock_irqrestore(&(s)->cpu_slab->lock, flags)
>
> -#ifdef CONFIG_SLUB_CPU_PARTIAL
> -static void __put_partials(struct kmem_cache *s, struct slab *partial_slab)
> -{
> - struct kmem_cache_node *n = NULL, *n2 = NULL;
> - struct slab *slab, *slab_to_discard = NULL;
> - unsigned long flags = 0;
> -
> - while (partial_slab) {
> - slab = partial_slab;
> - partial_slab = slab->next;
> -
> - n2 = get_node(s, slab_nid(slab));
> - if (n != n2) {
> - if (n)
> - spin_unlock_irqrestore(&n->list_lock, flags);
> -
> - n = n2;
> - spin_lock_irqsave(&n->list_lock, flags);
> - }
> -
> - if (unlikely(!slab->inuse && n->nr_partial >= s->min_partial)) {
> - slab->next = slab_to_discard;
> - slab_to_discard = slab;
> - } else {
> - add_partial(n, slab, DEACTIVATE_TO_TAIL);
> - stat(s, FREE_ADD_PARTIAL);
> - }
> - }
> -
> - if (n)
> - spin_unlock_irqrestore(&n->list_lock, flags);
> -
> - while (slab_to_discard) {
> - slab = slab_to_discard;
> - slab_to_discard = slab_to_discard->next;
> -
> - stat(s, DEACTIVATE_EMPTY);
> - discard_slab(s, slab);
> - stat(s, FREE_SLAB);
> - }
> -}
> -
> -/*
> - * Put all the cpu partial slabs to the node partial list.
> - */
> -static void put_partials(struct kmem_cache *s)
> -{
> - struct slab *partial_slab;
> - unsigned long flags;
> -
> - local_lock_irqsave(&s->cpu_slab->lock, flags);
> - partial_slab = this_cpu_read(s->cpu_slab->partial);
> - this_cpu_write(s->cpu_slab->partial, NULL);
> - local_unlock_irqrestore(&s->cpu_slab->lock, flags);
> -
> - if (partial_slab)
> - __put_partials(s, partial_slab);
> -}
> -
> -static void put_partials_cpu(struct kmem_cache *s,
> - struct kmem_cache_cpu *c)
> -{
> - struct slab *partial_slab;
> -
> - partial_slab = slub_percpu_partial(c);
> - c->partial = NULL;
> -
> - if (partial_slab)
> - __put_partials(s, partial_slab);
> -}
> -
> -/*
> - * Put a slab into a partial slab slot if available.
> - *
> - * If we did not find a slot then simply move all the partials to the
> - * per node partial list.
> - */
> -static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain)
> -{
> - struct slab *oldslab;
> - struct slab *slab_to_put = NULL;
> - unsigned long flags;
> - int slabs = 0;
> -
> - local_lock_cpu_slab(s, flags);
> -
> - oldslab = this_cpu_read(s->cpu_slab->partial);
> -
> - if (oldslab) {
> - if (drain && oldslab->slabs >= s->cpu_partial_slabs) {
> - /*
> - * Partial array is full. Move the existing set to the
> - * per node partial list. Postpone the actual unfreezing
> - * outside of the critical section.
> - */
> - slab_to_put = oldslab;
> - oldslab = NULL;
> - } else {
> - slabs = oldslab->slabs;
> - }
> - }
> -
> - slabs++;
> -
> - slab->slabs = slabs;
> - slab->next = oldslab;
> -
> - this_cpu_write(s->cpu_slab->partial, slab);
> -
> - local_unlock_cpu_slab(s, flags);
> -
> - if (slab_to_put) {
> - __put_partials(s, slab_to_put);
> - stat(s, CPU_PARTIAL_DRAIN);
> - }
> -}
> -
> -#else /* CONFIG_SLUB_CPU_PARTIAL */
> -
> -static inline void put_partials(struct kmem_cache *s) { }
> -static inline void put_partials_cpu(struct kmem_cache *s,
> - struct kmem_cache_cpu *c) { }
> -
> -#endif /* CONFIG_SLUB_CPU_PARTIAL */
> -
> static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
> {
> unsigned long flags;
> @@ -4060,8 +3894,6 @@ static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
> deactivate_slab(s, slab, freelist);
> stat(s, CPUSLAB_FLUSH);
> }
> -
> - put_partials_cpu(s, c);
> }
>
> static inline void flush_this_cpu_slab(struct kmem_cache *s)
> @@ -4070,15 +3902,13 @@ static inline void flush_this_cpu_slab(struct kmem_cache *s)
>
> if (c->slab)
> flush_slab(s, c);
> -
> - put_partials(s);
> }
>
> static bool has_cpu_slab(int cpu, struct kmem_cache *s)
> {
> struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
>
> - return c->slab || slub_percpu_partial(c);
> + return c->slab;
> }
>
> static bool has_pcs_used(int cpu, struct kmem_cache *s)
> @@ -5646,13 +5476,6 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
> return;
> }
>
> - /*
> - * It is enough to test IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) below
> - * instead of kmem_cache_has_cpu_partial(s), because kmem_cache_debug(s)
> - * is the only other reason it can be false, and it is already handled
> - * above.
> - */
> -
> do {
> if (unlikely(n)) {
> spin_unlock_irqrestore(&n->list_lock, flags);
> @@ -5677,26 +5500,19 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
> * Unless it's frozen.
> */
> if ((!new.inuse || was_full) && !was_frozen) {
> +
> + n = get_node(s, slab_nid(slab));
> /*
> - * If slab becomes non-full and we have cpu partial
> - * lists, we put it there unconditionally to avoid
> - * taking the list_lock. Otherwise we need it.
> + * Speculatively acquire the list_lock.
> + * If the cmpxchg does not succeed then we may
> + * drop the list_lock without any processing.
> + *
> + * Otherwise the list_lock will synchronize with
> + * other processors updating the list of slabs.
> */
> - if (!(IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && was_full)) {
> -
> - n = get_node(s, slab_nid(slab));
> - /*
> - * Speculatively acquire the list_lock.
> - * If the cmpxchg does not succeed then we may
> - * drop the list_lock without any processing.
> - *
> - * Otherwise the list_lock will synchronize with
> - * other processors updating the list of slabs.
> - */
> - spin_lock_irqsave(&n->list_lock, flags);
> -
> - on_node_partial = slab_test_node_partial(slab);
> - }
> + spin_lock_irqsave(&n->list_lock, flags);
> +
> + on_node_partial = slab_test_node_partial(slab);
> }
>
> } while (!slab_update_freelist(s, slab, &old, &new, "__slab_free"));
> @@ -5709,13 +5525,6 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
> * activity can be necessary.
> */
> stat(s, FREE_FROZEN);
> - } else if (IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && was_full) {
> - /*
> - * If we started with a full slab then put it onto the
> - * per cpu partial list.
> - */
> - put_cpu_partial(s, slab, 1);
> - stat(s, CPU_PARTIAL_FREE);
> }
>
> /*
> @@ -5744,10 +5553,9 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
>
> /*
> * Objects left in the slab. If it was not on the partial list before
> - * then add it. This can only happen when cache has no per cpu partial
> - * list otherwise we would have put it there.
> + * then add it.
> */
> - if (!IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && unlikely(was_full)) {
> + if (unlikely(was_full)) {
This is not really related to your change but I wonder why we check
for was_full to detect that the slab was not on partial list instead
of checking !on_node_partial... They might be equivalent at this point
but it's still a bit confusing.
> add_partial(n, slab, DEACTIVATE_TO_TAIL);
> stat(s, FREE_ADD_PARTIAL);
> }
> @@ -6396,8 +6204,8 @@ static __always_inline void do_slab_free(struct kmem_cache *s,
> if (unlikely(!allow_spin)) {
> /*
> * __slab_free() can locklessly cmpxchg16 into a slab,
> - * but then it might need to take spin_lock or local_lock
> - * in put_cpu_partial() for further processing.
> + * but then it might need to take spin_lock
> + * for further processing.
> * Avoid the complexity and simply add to a deferred list.
> */
> defer_free(s, head);
> @@ -7707,39 +7515,6 @@ static int init_kmem_cache_nodes(struct kmem_cache *s)
> return 1;
> }
>
> -static void set_cpu_partial(struct kmem_cache *s)
> -{
> -#ifdef CONFIG_SLUB_CPU_PARTIAL
> - unsigned int nr_objects;
> -
> - /*
> - * cpu_partial determined the maximum number of objects kept in the
> - * per cpu partial lists of a processor.
> - *
> - * Per cpu partial lists mainly contain slabs that just have one
> - * object freed. If they are used for allocation then they can be
> - * filled up again with minimal effort. The slab will never hit the
> - * per node partial lists and therefore no locking will be required.
> - *
> - * For backwards compatibility reasons, this is determined as number
> - * of objects, even though we now limit maximum number of pages, see
> - * slub_set_cpu_partial()
> - */
> - if (!kmem_cache_has_cpu_partial(s))
> - nr_objects = 0;
> - else if (s->size >= PAGE_SIZE)
> - nr_objects = 6;
> - else if (s->size >= 1024)
> - nr_objects = 24;
> - else if (s->size >= 256)
> - nr_objects = 52;
> - else
> - nr_objects = 120;
> -
> - slub_set_cpu_partial(s, nr_objects);
> -#endif
> -}
> -
> static unsigned int calculate_sheaf_capacity(struct kmem_cache *s,
> struct kmem_cache_args *args)
>
> @@ -8595,8 +8370,6 @@ int do_kmem_cache_create(struct kmem_cache *s, const char *name,
> s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2);
> s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial);
>
> - set_cpu_partial(s);
> -
> s->cpu_sheaves = alloc_percpu(struct slub_percpu_sheaves);
> if (!s->cpu_sheaves) {
> err = -ENOMEM;
> @@ -8960,20 +8733,6 @@ static ssize_t show_slab_objects(struct kmem_cache *s,
> total += x;
> nodes[node] += x;
>
> -#ifdef CONFIG_SLUB_CPU_PARTIAL
> - slab = slub_percpu_partial_read_once(c);
> - if (slab) {
> - node = slab_nid(slab);
> - if (flags & SO_TOTAL)
> - WARN_ON_ONCE(1);
> - else if (flags & SO_OBJECTS)
> - WARN_ON_ONCE(1);
> - else
> - x = data_race(slab->slabs);
> - total += x;
> - nodes[node] += x;
> - }
> -#endif
> }
> }
>
> @@ -9108,12 +8867,7 @@ SLAB_ATTR(min_partial);
>
> static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
> {
> - unsigned int nr_partial = 0;
> -#ifdef CONFIG_SLUB_CPU_PARTIAL
> - nr_partial = s->cpu_partial;
> -#endif
> -
> - return sysfs_emit(buf, "%u\n", nr_partial);
> + return sysfs_emit(buf, "0\n");
> }
>
> static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
> @@ -9125,11 +8879,9 @@ static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
> err = kstrtouint(buf, 10, &objects);
> if (err)
> return err;
> - if (objects && !kmem_cache_has_cpu_partial(s))
> + if (objects)
> return -EINVAL;
>
> - slub_set_cpu_partial(s, objects);
> - flush_all(s);
> return length;
> }
> SLAB_ATTR(cpu_partial);
> @@ -9168,42 +8920,7 @@ SLAB_ATTR_RO(objects_partial);
>
> static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
> {
> - int objects = 0;
> - int slabs = 0;
> - int cpu __maybe_unused;
> - int len = 0;
> -
> -#ifdef CONFIG_SLUB_CPU_PARTIAL
> - for_each_online_cpu(cpu) {
> - struct slab *slab;
> -
> - slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
> -
> - if (slab)
> - slabs += data_race(slab->slabs);
> - }
> -#endif
> -
> - /* Approximate half-full slabs, see slub_set_cpu_partial() */
> - objects = (slabs * oo_objects(s->oo)) / 2;
> - len += sysfs_emit_at(buf, len, "%d(%d)", objects, slabs);
> -
> -#ifdef CONFIG_SLUB_CPU_PARTIAL
> - for_each_online_cpu(cpu) {
> - struct slab *slab;
> -
> - slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
> - if (slab) {
> - slabs = data_race(slab->slabs);
> - objects = (slabs * oo_objects(s->oo)) / 2;
> - len += sysfs_emit_at(buf, len, " C%d=%d(%d)",
> - cpu, objects, slabs);
> - }
> - }
> -#endif
> - len += sysfs_emit_at(buf, len, "\n");
> -
> - return len;
> + return sysfs_emit(buf, "0(0)\n");
> }
> SLAB_ATTR_RO(slabs_cpu_partial);
>
>
> --
> 2.52.0
>
On 1/20/26 23:25, Suren Baghdasaryan wrote: > On Fri, Jan 16, 2026 at 2:40 PM Vlastimil Babka <vbabka@suse.cz> wrote: >> >> We have removed the partial slab usage from allocation paths. Now remove >> the whole config option and associated code. >> >> Reviewed-by: Suren Baghdasaryan <surenb@google.com> > > I did? Hmm looks like you didn't. Wonder if I screwed up, or b4 did. Sorry about that. > Well, if so, I missed some remaining mentions about cpu partial caches: > - slub.c has several hits on "cpu partial" in the comments. > - there is one hit on "put_cpu_partial" in slub.c in the comments. Should be addressed later by [PATCH v3 18/21] slab: update overview comments. I'll grep the result if anything is missing. > Should we also update Documentation/ABI/testing/sysfs-kernel-slab to > say that from now on cpu_partial control always reads 0? Uh those weird files. Does anyone care? I'd do that separately as well... > Once addressed, please feel free to keep my Reviewed-by. Thanks!
On Wed, Jan 21, 2026 at 2:22 PM Vlastimil Babka <vbabka@suse.cz> wrote: > > On 1/20/26 23:25, Suren Baghdasaryan wrote: > > On Fri, Jan 16, 2026 at 2:40 PM Vlastimil Babka <vbabka@suse.cz> wrote: > >> > >> We have removed the partial slab usage from allocation paths. Now remove > >> the whole config option and associated code. > >> > >> Reviewed-by: Suren Baghdasaryan <surenb@google.com> > > > > I did? > > Hmm looks like you didn't. Wonder if I screwed up, or b4 did. Sorry about that. No worries. > > > Well, if so, I missed some remaining mentions about cpu partial caches: > > - slub.c has several hits on "cpu partial" in the comments. > > - there is one hit on "put_cpu_partial" in slub.c in the comments. > > Should be addressed later by [PATCH v3 18/21] slab: update overview > comments. I'll grep the result if anything is missing. > > > Should we also update Documentation/ABI/testing/sysfs-kernel-slab to > > say that from now on cpu_partial control always reads 0? > > Uh those weird files. Does anyone care? I'd do that separately as well... I'm fine either way. Thanks! > > > Once addressed, please feel free to keep my Reviewed-by. > > Thanks! >
On 1/21/26 15:22, Vlastimil Babka wrote: > On 1/20/26 23:25, Suren Baghdasaryan wrote: >> On Fri, Jan 16, 2026 at 2:40 PM Vlastimil Babka <vbabka@suse.cz> wrote: >>> >>> We have removed the partial slab usage from allocation paths. Now remove >>> the whole config option and associated code. >>> >>> Reviewed-by: Suren Baghdasaryan <surenb@google.com> >> >> I did? > > Hmm looks like you didn't. Wonder if I screwed up, or b4 did. Sorry about that. Seems like it was b4 and did for all patches, damn. Sorry, will fix up to match reality.
On Tue, Jan 20, 2026 at 10:25:27PM +0000, Suren Baghdasaryan wrote:
> On Fri, Jan 16, 2026 at 2:40 PM Vlastimil Babka <vbabka@suse.cz> wrote:
> > @@ -5744,10 +5553,9 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
> >
> > /*
> > * Objects left in the slab. If it was not on the partial list before
> > - * then add it. This can only happen when cache has no per cpu partial
> > - * list otherwise we would have put it there.
> > + * then add it.
> > */
> > - if (!IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && unlikely(was_full)) {
> > + if (unlikely(was_full)) {
>
> This is not really related to your change but I wonder why we check
> for was_full to detect that the slab was not on partial list instead
> of checking !on_node_partial... They might be equivalent at this point
> but it's still a bit confusing.
If we only know that a slab is not on the partial list, we cannot
manipulate its list because it may be on a linked list that cannot
handle list manipulation outside function
(e.g., pc.slabs in __refill_objects()).
If it's not on the partial list, we can safely manipulate the list
only when we know it was full. It's safe because full slabs are not
supposed to be on any list (except for debug caches, where frees are
done via free_to_partial_list()).
--
Cheers,
Harry / Hyeonggon
On Wed, Jan 21, 2026 at 12:59 AM Harry Yoo <harry.yoo@oracle.com> wrote:
>
> On Tue, Jan 20, 2026 at 10:25:27PM +0000, Suren Baghdasaryan wrote:
> > On Fri, Jan 16, 2026 at 2:40 PM Vlastimil Babka <vbabka@suse.cz> wrote:
> > > @@ -5744,10 +5553,9 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
> > >
> > > /*
> > > * Objects left in the slab. If it was not on the partial list before
> > > - * then add it. This can only happen when cache has no per cpu partial
> > > - * list otherwise we would have put it there.
> > > + * then add it.
> > > */
> > > - if (!IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && unlikely(was_full)) {
> > > + if (unlikely(was_full)) {
> >
> > This is not really related to your change but I wonder why we check
> > for was_full to detect that the slab was not on partial list instead
> > of checking !on_node_partial... They might be equivalent at this point
> > but it's still a bit confusing.
>
> If we only know that a slab is not on the partial list, we cannot
> manipulate its list because it may be on a linked list that cannot
> handle list manipulation outside function
> (e.g., pc.slabs in __refill_objects()).
>
> If it's not on the partial list, we can safely manipulate the list
> only when we know it was full. It's safe because full slabs are not
> supposed to be on any list (except for debug caches, where frees are
> done via free_to_partial_list()).
Ack. I guess I'm reading the above comment too literally. Thanks!
>
> --
> Cheers,
> Harry / Hyeonggon
On Wed, Jan 21, 2026 at 09:58:40AM +0900, Harry Yoo wrote:
> On Tue, Jan 20, 2026 at 10:25:27PM +0000, Suren Baghdasaryan wrote:
> > On Fri, Jan 16, 2026 at 2:40 PM Vlastimil Babka <vbabka@suse.cz> wrote:
> > > @@ -5744,10 +5553,9 @@ static void __slab_free(struct kmem_cache *s, struct slab *slab,
> > >
> > > /*
> > > * Objects left in the slab. If it was not on the partial list before
> > > - * then add it. This can only happen when cache has no per cpu partial
> > > - * list otherwise we would have put it there.
> > > + * then add it.
> > > */
> > > - if (!IS_ENABLED(CONFIG_SLUB_CPU_PARTIAL) && unlikely(was_full)) {
> > > + if (unlikely(was_full)) {
> >
> > This is not really related to your change but I wonder why we check
> > for was_full to detect that the slab was not on partial list instead
> > of checking !on_node_partial... They might be equivalent at this point
> > but it's still a bit confusing.
>
> If we only know that a slab is not on the partial list, we cannot
> manipulate its list because it may be on a linked list that cannot
> handle list manipulation outside function
> (e.g., pc.slabs in __refill_objects()).
>
> If it's not on the partial list, we can safely manipulate the list
> only when we know it was full. It's safe because full slabs are not
> supposed to be on any list (except for debug caches, where frees are
> done via free_to_partial_list()).
Of course, when a slab was frozen, this doesn't apply and __slab_free()
explicitly handles that case.
--
Cheers,
Harry / Hyeonggon
On Fri, Jan 16, 2026 at 03:40:31PM +0100, Vlastimil Babka wrote: > We have removed the partial slab usage from allocation paths. Now remove > the whole config option and associated code. > > Reviewed-by: Suren Baghdasaryan <surenb@google.com> > Signed-off-by: Vlastimil Babka <vbabka@suse.cz> > --- > mm/Kconfig | 11 --- > mm/slab.h | 29 ------ > mm/slub.c | 321 ++++--------------------------------------------------------- > 3 files changed, 19 insertions(+), 342 deletions(-) > Looks good to me. Reviewed-by: Hao Li <hao.li@linux.dev> -- Thanks, Hao
On Fri, Jan 16, 2026 at 03:40:31PM +0100, Vlastimil Babka wrote: > We have removed the partial slab usage from allocation paths. Now remove > the whole config option and associated code. > > Reviewed-by: Suren Baghdasaryan <surenb@google.com> > Signed-off-by: Vlastimil Babka <vbabka@suse.cz> > --- Looks good to me, Reviewed-by: Harry Yoo <harry.yoo@oracle.com> -- Cheers, Harry / Hyeonggon
© 2016 - 2026 Red Hat, Inc.