Enable sheaves for kmalloc caches. For other types than KMALLOC_NORMAL,
we can simply allow them in calculate_sizes() as they are created later
than KMALLOC_NORMAL caches and can allocate sheaves and barns from
those.
For KMALLOC_NORMAL caches we perform additional step after first
creating them without sheaves. Then bootstrap_cache_sheaves() simply
allocates and initializes barns and sheaves and finally sets
s->sheaf_capacity to make them actually used.
Afterwards the only caches left without sheaves (unless SLUB_TINY or
debugging is enabled) are kmem_cache and kmem_cache_node. These are only
used when creating or destroying other kmem_caches. Thus they are not
performance critical and we can simply leave it that way.
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
---
mm/slub.c | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---
1 file changed, 84 insertions(+), 4 deletions(-)
diff --git a/mm/slub.c b/mm/slub.c
index 5d0b2cf66520..a84027fbca78 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2588,7 +2588,8 @@ static void *setup_object(struct kmem_cache *s, void *object)
return object;
}
-static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)
+static struct slab_sheaf *__alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp,
+ unsigned int capacity)
{
struct slab_sheaf *sheaf;
size_t sheaf_size;
@@ -2606,7 +2607,7 @@ static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)
if (s->flags & SLAB_KMALLOC)
gfp |= __GFP_NO_OBJ_EXT;
- sheaf_size = struct_size(sheaf, objects, s->sheaf_capacity);
+ sheaf_size = struct_size(sheaf, objects, capacity);
sheaf = kzalloc(sheaf_size, gfp);
if (unlikely(!sheaf))
@@ -2619,6 +2620,12 @@ static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)
return sheaf;
}
+static inline struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s,
+ gfp_t gfp)
+{
+ return __alloc_empty_sheaf(s, gfp, s->sheaf_capacity);
+}
+
static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf)
{
kfree(sheaf);
@@ -8064,8 +8071,11 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s)
if (s->flags & SLAB_RECLAIM_ACCOUNT)
s->allocflags |= __GFP_RECLAIMABLE;
- /* kmalloc caches need extra care to support sheaves */
- if (!is_kmalloc_cache(s))
+ /*
+ * For KMALLOC_NORMAL caches we enable sheaves later by
+ * bootstrap_kmalloc_sheaves() to avoid recursion
+ */
+ if (!is_kmalloc_normal(s))
s->sheaf_capacity = calculate_sheaf_capacity(s, args);
/*
@@ -8549,6 +8559,74 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
return s;
}
+/*
+ * Finish the sheaves initialization done normally by init_percpu_sheaves() and
+ * init_kmem_cache_nodes(). For normal kmalloc caches we have to bootstrap it
+ * since sheaves and barns are allocated by kmalloc.
+ */
+static void __init bootstrap_cache_sheaves(struct kmem_cache *s)
+{
+ struct kmem_cache_args empty_args = {};
+ unsigned int capacity;
+ bool failed = false;
+ int node, cpu;
+
+ capacity = calculate_sheaf_capacity(s, &empty_args);
+
+ /* capacity can be 0 due to debugging or SLUB_TINY */
+ if (!capacity)
+ return;
+
+ for_each_node_mask(node, slab_nodes) {
+ struct node_barn *barn;
+
+ barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, node);
+
+ if (!barn) {
+ failed = true;
+ goto out;
+ }
+
+ barn_init(barn);
+ get_node(s, node)->barn = barn;
+ }
+
+ for_each_possible_cpu(cpu) {
+ struct slub_percpu_sheaves *pcs;
+
+ pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
+
+ pcs->main = __alloc_empty_sheaf(s, GFP_KERNEL, capacity);
+
+ if (!pcs->main) {
+ failed = true;
+ break;
+ }
+ }
+
+out:
+ /*
+ * It's still early in boot so treat this like same as a failure to
+ * create the kmalloc cache in the first place
+ */
+ if (failed)
+ panic("Out of memory when creating kmem_cache %s\n", s->name);
+
+ s->sheaf_capacity = capacity;
+}
+
+static void __init bootstrap_kmalloc_sheaves(void)
+{
+ enum kmalloc_cache_type type;
+
+ for (type = KMALLOC_NORMAL; type <= KMALLOC_RANDOM_END; type++) {
+ for (int idx = 0; idx < KMALLOC_SHIFT_HIGH + 1; idx++) {
+ if (kmalloc_caches[type][idx])
+ bootstrap_cache_sheaves(kmalloc_caches[type][idx]);
+ }
+ }
+}
+
void __init kmem_cache_init(void)
{
static __initdata struct kmem_cache boot_kmem_cache,
@@ -8592,6 +8670,8 @@ void __init kmem_cache_init(void)
setup_kmalloc_cache_index_table();
create_kmalloc_caches();
+ bootstrap_kmalloc_sheaves();
+
/* Setup random freelists for each cache */
init_freelist_randomization();
--
2.51.1On Thu, Oct 23, 2025 at 03:52:30PM +0200, Vlastimil Babka wrote:
> Enable sheaves for kmalloc caches. For other types than KMALLOC_NORMAL,
> we can simply allow them in calculate_sizes() as they are created later
> than KMALLOC_NORMAL caches and can allocate sheaves and barns from
> those.
>
> For KMALLOC_NORMAL caches we perform additional step after first
> creating them without sheaves. Then bootstrap_cache_sheaves() simply
> allocates and initializes barns and sheaves and finally sets
> s->sheaf_capacity to make them actually used.
>
> Afterwards the only caches left without sheaves (unless SLUB_TINY or
> debugging is enabled) are kmem_cache and kmem_cache_node. These are only
> used when creating or destroying other kmem_caches. Thus they are not
> performance critical and we can simply leave it that way.
>
> Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
> ---
> mm/slub.c | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---
> 1 file changed, 84 insertions(+), 4 deletions(-)
>
> diff --git a/mm/slub.c b/mm/slub.c
> index 5d0b2cf66520..a84027fbca78 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
>
> static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf)
> {
> kfree(sheaf);
> @@ -8064,8 +8071,11 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s)
> if (s->flags & SLAB_RECLAIM_ACCOUNT)
> s->allocflags |= __GFP_RECLAIMABLE;
>
> - /* kmalloc caches need extra care to support sheaves */
> - if (!is_kmalloc_cache(s))
> + /*
> + * For KMALLOC_NORMAL caches we enable sheaves later by
> + * bootstrap_kmalloc_sheaves() to avoid recursion
> + */
> + if (!is_kmalloc_normal(s))
> s->sheaf_capacity = calculate_sheaf_capacity(s, args);
I was going to say we should differentiate KMALLOC_NORMAL caches that
are created for kmalloc buckets.... but no, they don't have the SLAB_KMALLOC
flag.
> /*
> @@ -8549,6 +8559,74 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
> return s;
> }
>
> +/*
> + * Finish the sheaves initialization done normally by init_percpu_sheaves() and
> + * init_kmem_cache_nodes(). For normal kmalloc caches we have to bootstrap it
> + * since sheaves and barns are allocated by kmalloc.
> + */
> +static void __init bootstrap_cache_sheaves(struct kmem_cache *s)
> +{
> + struct kmem_cache_args empty_args = {};
> + unsigned int capacity;
> + bool failed = false;
> + int node, cpu;
> +
> + capacity = calculate_sheaf_capacity(s, &empty_args);
> +
> + /* capacity can be 0 due to debugging or SLUB_TINY */
> + if (!capacity)
> + return;
I think pcs->main should still be !NULL in this case?
--
Cheers,
Harry / Hyeonggon
> +
> + for_each_node_mask(node, slab_nodes) {
> + struct node_barn *barn;
> +
> + barn = kmalloc_node(sizeof(*barn), GFP_KERNEL, node);
> +
> + if (!barn) {
> + failed = true;
> + goto out;
> + }
> +
> + barn_init(barn);
> + get_node(s, node)->barn = barn;
> + }
> +
> + for_each_possible_cpu(cpu) {
> + struct slub_percpu_sheaves *pcs;
> +
> + pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
> +
> + pcs->main = __alloc_empty_sheaf(s, GFP_KERNEL, capacity);
> +
> + if (!pcs->main) {
> + failed = true;
> + break;
> + }
> + }
> +
> +out:
> + /*
> + * It's still early in boot so treat this like same as a failure to
> + * create the kmalloc cache in the first place
> + */
> + if (failed)
> + panic("Out of memory when creating kmem_cache %s\n", s->name);
> +
> + s->sheaf_capacity = capacity;
> +}
On 10/27/25 07:12, Harry Yoo wrote:
>> @@ -8549,6 +8559,74 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
>> return s;
>> }
>>
>> +/*
>> + * Finish the sheaves initialization done normally by init_percpu_sheaves() and
>> + * init_kmem_cache_nodes(). For normal kmalloc caches we have to bootstrap it
>> + * since sheaves and barns are allocated by kmalloc.
>> + */
>> +static void __init bootstrap_cache_sheaves(struct kmem_cache *s)
>> +{
>> + struct kmem_cache_args empty_args = {};
>> + unsigned int capacity;
>> + bool failed = false;
>> + int node, cpu;
>> +
>> + capacity = calculate_sheaf_capacity(s, &empty_args);
>> +
>> + /* capacity can be 0 due to debugging or SLUB_TINY */
>> + if (!capacity)
>> + return;
>
> I think pcs->main should still be !NULL in this case?
It will remain to be set to bootstrap_sheaf, and with s->sheaf_capacity
things will continue to work.
On 10/29/25 21:06, Vlastimil Babka wrote:
> On 10/27/25 07:12, Harry Yoo wrote:
>>> @@ -8549,6 +8559,74 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
>>> return s;
>>> }
>>>
>>> +/*
>>> + * Finish the sheaves initialization done normally by init_percpu_sheaves() and
>>> + * init_kmem_cache_nodes(). For normal kmalloc caches we have to bootstrap it
>>> + * since sheaves and barns are allocated by kmalloc.
>>> + */
>>> +static void __init bootstrap_cache_sheaves(struct kmem_cache *s)
>>> +{
>>> + struct kmem_cache_args empty_args = {};
>>> + unsigned int capacity;
>>> + bool failed = false;
>>> + int node, cpu;
>>> +
>>> + capacity = calculate_sheaf_capacity(s, &empty_args);
>>> +
>>> + /* capacity can be 0 due to debugging or SLUB_TINY */
>>> + if (!capacity)
>>> + return;
>>
>> I think pcs->main should still be !NULL in this case?
>
> It will remain to be set to bootstrap_sheaf, and with s->sheaf_capacity
... s->sheaf_capacity remaining 0
> things will continue to work.
On Wed, Oct 29, 2025 at 09:06:45PM +0100, Vlastimil Babka wrote:
> On 10/29/25 21:06, Vlastimil Babka wrote:
> > On 10/27/25 07:12, Harry Yoo wrote:
> >>> @@ -8549,6 +8559,74 @@ static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
> >>> return s;
> >>> }
> >>>
> >>> +/*
> >>> + * Finish the sheaves initialization done normally by init_percpu_sheaves() and
> >>> + * init_kmem_cache_nodes(). For normal kmalloc caches we have to bootstrap it
> >>> + * since sheaves and barns are allocated by kmalloc.
> >>> + */
> >>> +static void __init bootstrap_cache_sheaves(struct kmem_cache *s)
> >>> +{
> >>> + struct kmem_cache_args empty_args = {};
> >>> + unsigned int capacity;
> >>> + bool failed = false;
> >>> + int node, cpu;
> >>> +
> >>> + capacity = calculate_sheaf_capacity(s, &empty_args);
> >>> +
> >>> + /* capacity can be 0 due to debugging or SLUB_TINY */
> >>> + if (!capacity)
> >>> + return;
> >>
> >> I think pcs->main should still be !NULL in this case?
> >
> > It will remain to be set to bootstrap_sheaf, and with s->sheaf_capacity
>
> ... s->sheaf_capacity remaining 0
>
> > things will continue to work.
Oh right. it's set to bootstrap_sheaf in init_percpu_sheaves() before
bootstrap_cache_sheaves() is called. Looks good then!
--
Cheers,
Harry / Hyeonggon
© 2016 - 2026 Red Hat, Inc.