From: "Liam R. Howlett" <Liam.Howlett@Oracle.com>
Make testing work for the slab and rcu changes that have come in with
the sheaves work.
This only works with one kmem_cache, and only the first one used.
Subsequent setting of kmem_cache will not update the active kmem_cache
and will be silently dropped because there are other tests which happen
after the kmem_cache of interest is set.
The saved active kmem_cache is used in the rcu callback, which passes
the object to be freed.
The rcu call takes the rcu_head, which is passed in as the field in the
struct (in this case rcu in the maple tree node), which is calculated by
pointer math. The offset of which is saved (in a global variable) for
restoring the node pointer on the callback after the rcu grace period
expires.
Don't use any of this outside of testing, please.
Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
---
tools/include/linux/slab.h | 41 ++++++++++++++++++++++++++++++++---
tools/testing/shared/linux.c | 24 ++++++++++++++++----
tools/testing/shared/linux/rcupdate.h | 22 +++++++++++++++++++
3 files changed, 80 insertions(+), 7 deletions(-)
diff --git a/tools/include/linux/slab.h b/tools/include/linux/slab.h
index c87051e2b26f5a7fee0362697fae067076b8e84d..d1444e79f2685edb828adbce8b3fbb500c0f8844 100644
--- a/tools/include/linux/slab.h
+++ b/tools/include/linux/slab.h
@@ -23,6 +23,12 @@ enum slab_state {
FULL
};
+struct kmem_cache_args {
+ unsigned int align;
+ unsigned int sheaf_capacity;
+ void (*ctor)(void *);
+};
+
static inline void *kzalloc(size_t size, gfp_t gfp)
{
return kmalloc(size, gfp | __GFP_ZERO);
@@ -37,9 +43,38 @@ static inline void *kmem_cache_alloc(struct kmem_cache *cachep, int flags)
}
void kmem_cache_free(struct kmem_cache *cachep, void *objp);
-struct kmem_cache *kmem_cache_create(const char *name, unsigned int size,
- unsigned int align, unsigned int flags,
- void (*ctor)(void *));
+
+struct kmem_cache *
+__kmem_cache_create_args(const char *name, unsigned int size,
+ struct kmem_cache_args *args, unsigned int flags);
+
+/* If NULL is passed for @args, use this variant with default arguments. */
+static inline struct kmem_cache *
+__kmem_cache_default_args(const char *name, unsigned int size,
+ struct kmem_cache_args *args, unsigned int flags)
+{
+ struct kmem_cache_args kmem_default_args = {};
+
+ return __kmem_cache_create_args(name, size, &kmem_default_args, flags);
+}
+
+static inline struct kmem_cache *
+__kmem_cache_create(const char *name, unsigned int size, unsigned int align,
+ unsigned int flags, void (*ctor)(void *))
+{
+ struct kmem_cache_args kmem_args = {
+ .align = align,
+ .ctor = ctor,
+ };
+
+ return __kmem_cache_create_args(name, size, &kmem_args, flags);
+}
+
+#define kmem_cache_create(__name, __object_size, __args, ...) \
+ _Generic((__args), \
+ struct kmem_cache_args *: __kmem_cache_create_args, \
+ void *: __kmem_cache_default_args, \
+ default: __kmem_cache_create)(__name, __object_size, __args, __VA_ARGS__)
void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list);
int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
diff --git a/tools/testing/shared/linux.c b/tools/testing/shared/linux.c
index 0f97fb0d19e19c327aa4843a35b45cc086f4f366..f998555a1b2af4a899a468a652b04622df459ed3 100644
--- a/tools/testing/shared/linux.c
+++ b/tools/testing/shared/linux.c
@@ -20,6 +20,7 @@ struct kmem_cache {
pthread_mutex_t lock;
unsigned int size;
unsigned int align;
+ unsigned int sheaf_capacity;
int nr_objs;
void *objs;
void (*ctor)(void *);
@@ -31,6 +32,8 @@ struct kmem_cache {
void *private;
};
+static struct kmem_cache *kmem_active = NULL;
+
void kmem_cache_set_callback(struct kmem_cache *cachep, void (*callback)(void *))
{
cachep->callback = callback;
@@ -147,6 +150,14 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp)
pthread_mutex_unlock(&cachep->lock);
}
+void kmem_cache_free_active(void *objp)
+{
+ if (!kmem_active)
+ printf("WARNING: No active kmem_cache\n");
+
+ kmem_cache_free(kmem_active, objp);
+}
+
void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list)
{
if (kmalloc_verbose)
@@ -234,23 +245,28 @@ int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
}
struct kmem_cache *
-kmem_cache_create(const char *name, unsigned int size, unsigned int align,
- unsigned int flags, void (*ctor)(void *))
+__kmem_cache_create_args(const char *name, unsigned int size,
+ struct kmem_cache_args *args,
+ unsigned int flags)
{
struct kmem_cache *ret = malloc(sizeof(*ret));
pthread_mutex_init(&ret->lock, NULL);
ret->size = size;
- ret->align = align;
+ ret->align = args->align;
+ ret->sheaf_capacity = args->sheaf_capacity;
ret->nr_objs = 0;
ret->nr_allocated = 0;
ret->nr_tallocated = 0;
ret->objs = NULL;
- ret->ctor = ctor;
+ ret->ctor = args->ctor;
ret->non_kernel = 0;
ret->exec_callback = false;
ret->callback = NULL;
ret->private = NULL;
+ if (!kmem_active)
+ kmem_active = ret;
+
return ret;
}
diff --git a/tools/testing/shared/linux/rcupdate.h b/tools/testing/shared/linux/rcupdate.h
index fed468fb0c78db6f33fb1900c7110ab5f3c19c65..c95e2f0bbd93798e544d7d34e0823ed68414f924 100644
--- a/tools/testing/shared/linux/rcupdate.h
+++ b/tools/testing/shared/linux/rcupdate.h
@@ -9,4 +9,26 @@
#define rcu_dereference_check(p, cond) rcu_dereference(p)
#define RCU_INIT_POINTER(p, v) do { (p) = (v); } while (0)
+void kmem_cache_free_active(void *objp);
+static unsigned long kfree_cb_offset = 0;
+
+static inline void kfree_rcu_cb(struct rcu_head *head)
+{
+ void *objp = (void *) ((unsigned long)head - kfree_cb_offset);
+
+ kmem_cache_free_active(objp);
+}
+
+#ifndef offsetof
+#define offsetof(TYPE, MEMBER) __builtin_offsetof(TYPE, MEMBER)
+#endif
+
+#define kfree_rcu(ptr, rhv) \
+do { \
+ if (!kfree_cb_offset) \
+ kfree_cb_offset = offsetof(typeof(*(ptr)), rhv); \
+ \
+ call_rcu(&ptr->rhv, kfree_rcu_cb); \
+} while (0)
+
#endif
--
2.50.1On Wed, Jul 23, 2025 at 6:35 AM Vlastimil Babka <vbabka@suse.cz> wrote:
>
> From: "Liam R. Howlett" <Liam.Howlett@Oracle.com>
>
> Make testing work for the slab and rcu changes that have come in with
> the sheaves work.
>
> This only works with one kmem_cache, and only the first one used.
> Subsequent setting of kmem_cache will not update the active kmem_cache
> and will be silently dropped because there are other tests which happen
> after the kmem_cache of interest is set.
>
> The saved active kmem_cache is used in the rcu callback, which passes
> the object to be freed.
>
> The rcu call takes the rcu_head, which is passed in as the field in the
> struct (in this case rcu in the maple tree node), which is calculated by
> pointer math. The offset of which is saved (in a global variable) for
> restoring the node pointer on the callback after the rcu grace period
> expires.
>
> Don't use any of this outside of testing, please.
>
> Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
> Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Couple nits but otherwise LGTM.
Reviewed-by: Suren Baghdasaryan <surenb@google.com>
> ---
> tools/include/linux/slab.h | 41 ++++++++++++++++++++++++++++++++---
> tools/testing/shared/linux.c | 24 ++++++++++++++++----
> tools/testing/shared/linux/rcupdate.h | 22 +++++++++++++++++++
> 3 files changed, 80 insertions(+), 7 deletions(-)
>
> diff --git a/tools/include/linux/slab.h b/tools/include/linux/slab.h
> index c87051e2b26f5a7fee0362697fae067076b8e84d..d1444e79f2685edb828adbce8b3fbb500c0f8844 100644
> --- a/tools/include/linux/slab.h
> +++ b/tools/include/linux/slab.h
> @@ -23,6 +23,12 @@ enum slab_state {
> FULL
> };
>
> +struct kmem_cache_args {
> + unsigned int align;
> + unsigned int sheaf_capacity;
> + void (*ctor)(void *);
> +};
> +
> static inline void *kzalloc(size_t size, gfp_t gfp)
> {
> return kmalloc(size, gfp | __GFP_ZERO);
> @@ -37,9 +43,38 @@ static inline void *kmem_cache_alloc(struct kmem_cache *cachep, int flags)
> }
> void kmem_cache_free(struct kmem_cache *cachep, void *objp);
>
> -struct kmem_cache *kmem_cache_create(const char *name, unsigned int size,
> - unsigned int align, unsigned int flags,
> - void (*ctor)(void *));
> +
> +struct kmem_cache *
> +__kmem_cache_create_args(const char *name, unsigned int size,
> + struct kmem_cache_args *args, unsigned int flags);
> +
> +/* If NULL is passed for @args, use this variant with default arguments. */
> +static inline struct kmem_cache *
> +__kmem_cache_default_args(const char *name, unsigned int size,
> + struct kmem_cache_args *args, unsigned int flags)
> +{
> + struct kmem_cache_args kmem_default_args = {};
> +
> + return __kmem_cache_create_args(name, size, &kmem_default_args, flags);
> +}
> +
> +static inline struct kmem_cache *
> +__kmem_cache_create(const char *name, unsigned int size, unsigned int align,
> + unsigned int flags, void (*ctor)(void *))
> +{
> + struct kmem_cache_args kmem_args = {
> + .align = align,
> + .ctor = ctor,
> + };
> +
> + return __kmem_cache_create_args(name, size, &kmem_args, flags);
> +}
> +
> +#define kmem_cache_create(__name, __object_size, __args, ...) \
> + _Generic((__args), \
> + struct kmem_cache_args *: __kmem_cache_create_args, \
> + void *: __kmem_cache_default_args, \
> + default: __kmem_cache_create)(__name, __object_size, __args, __VA_ARGS__)
>
> void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list);
> int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
> diff --git a/tools/testing/shared/linux.c b/tools/testing/shared/linux.c
> index 0f97fb0d19e19c327aa4843a35b45cc086f4f366..f998555a1b2af4a899a468a652b04622df459ed3 100644
> --- a/tools/testing/shared/linux.c
> +++ b/tools/testing/shared/linux.c
> @@ -20,6 +20,7 @@ struct kmem_cache {
> pthread_mutex_t lock;
> unsigned int size;
> unsigned int align;
> + unsigned int sheaf_capacity;
> int nr_objs;
> void *objs;
> void (*ctor)(void *);
> @@ -31,6 +32,8 @@ struct kmem_cache {
> void *private;
> };
>
> +static struct kmem_cache *kmem_active = NULL;
> +
> void kmem_cache_set_callback(struct kmem_cache *cachep, void (*callback)(void *))
> {
> cachep->callback = callback;
> @@ -147,6 +150,14 @@ void kmem_cache_free(struct kmem_cache *cachep, void *objp)
> pthread_mutex_unlock(&cachep->lock);
> }
>
> +void kmem_cache_free_active(void *objp)
> +{
> + if (!kmem_active)
> + printf("WARNING: No active kmem_cache\n");
> +
> + kmem_cache_free(kmem_active, objp);
> +}
> +
> void kmem_cache_free_bulk(struct kmem_cache *cachep, size_t size, void **list)
> {
> if (kmalloc_verbose)
> @@ -234,23 +245,28 @@ int kmem_cache_alloc_bulk(struct kmem_cache *cachep, gfp_t gfp, size_t size,
> }
>
> struct kmem_cache *
> -kmem_cache_create(const char *name, unsigned int size, unsigned int align,
> - unsigned int flags, void (*ctor)(void *))
> +__kmem_cache_create_args(const char *name, unsigned int size,
> + struct kmem_cache_args *args,
> + unsigned int flags)
> {
> struct kmem_cache *ret = malloc(sizeof(*ret));
>
> pthread_mutex_init(&ret->lock, NULL);
> ret->size = size;
> - ret->align = align;
> + ret->align = args->align;
> + ret->sheaf_capacity = args->sheaf_capacity;
> ret->nr_objs = 0;
> ret->nr_allocated = 0;
> ret->nr_tallocated = 0;
> ret->objs = NULL;
> - ret->ctor = ctor;
> + ret->ctor = args->ctor;
> ret->non_kernel = 0;
> ret->exec_callback = false;
> ret->callback = NULL;
> ret->private = NULL;
> + if (!kmem_active)
> + kmem_active = ret;
> +
> return ret;
> }
>
> diff --git a/tools/testing/shared/linux/rcupdate.h b/tools/testing/shared/linux/rcupdate.h
> index fed468fb0c78db6f33fb1900c7110ab5f3c19c65..c95e2f0bbd93798e544d7d34e0823ed68414f924 100644
> --- a/tools/testing/shared/linux/rcupdate.h
> +++ b/tools/testing/shared/linux/rcupdate.h
> @@ -9,4 +9,26 @@
> #define rcu_dereference_check(p, cond) rcu_dereference(p)
> #define RCU_INIT_POINTER(p, v) do { (p) = (v); } while (0)
>
> +void kmem_cache_free_active(void *objp);
> +static unsigned long kfree_cb_offset = 0;
> +
> +static inline void kfree_rcu_cb(struct rcu_head *head)
> +{
> + void *objp = (void *) ((unsigned long)head - kfree_cb_offset);
> +
> + kmem_cache_free_active(objp);
> +}
> +
> +#ifndef offsetof
> +#define offsetof(TYPE, MEMBER) __builtin_offsetof(TYPE, MEMBER)
> +#endif
> +
We need a comment here that concurrent kfree_rcu() calls are not
supported because they would override each other's kfree_cb_offset.
Kinda obvious but I think unusual limitations should be explicitly
called out.
> +#define kfree_rcu(ptr, rhv) \
> +do { \
> + if (!kfree_cb_offset) \
> + kfree_cb_offset = offsetof(typeof(*(ptr)), rhv); \
> + \
> + call_rcu(&ptr->rhv, kfree_rcu_cb); \
> +} while (0)
Any specific reason kfree_rcu() is a macro and not a static inline function?
> +
> #endif
>
> --
> 2.50.1
>
On 8/22/25 18:28, Suren Baghdasaryan wrote:
>> diff --git a/tools/testing/shared/linux/rcupdate.h b/tools/testing/shared/linux/rcupdate.h
>> index fed468fb0c78db6f33fb1900c7110ab5f3c19c65..c95e2f0bbd93798e544d7d34e0823ed68414f924 100644
>> --- a/tools/testing/shared/linux/rcupdate.h
>> +++ b/tools/testing/shared/linux/rcupdate.h
>> @@ -9,4 +9,26 @@
>> #define rcu_dereference_check(p, cond) rcu_dereference(p)
>> #define RCU_INIT_POINTER(p, v) do { (p) = (v); } while (0)
>>
>> +void kmem_cache_free_active(void *objp);
>> +static unsigned long kfree_cb_offset = 0;
>> +
>> +static inline void kfree_rcu_cb(struct rcu_head *head)
>> +{
>> + void *objp = (void *) ((unsigned long)head - kfree_cb_offset);
>> +
>> + kmem_cache_free_active(objp);
>> +}
>> +
>> +#ifndef offsetof
>> +#define offsetof(TYPE, MEMBER) __builtin_offsetof(TYPE, MEMBER)
>> +#endif
>> +
>
> We need a comment here that concurrent kfree_rcu() calls are not
> supported because they would override each other's kfree_cb_offset.
I think it's a bit more complex and related to the commit log sentence "This
only works with one kmem_cache, and only the first one used.". The first
call to kfree_rcu sets kfree_cb_offset (but what if the rhv offset is
actually 0?) so the others won't update it. So concurrent calls will work as
far as from the same cache thus same offset. But I'd like Liam's
confirmation and the comment text, if possible :)
> Kinda obvious but I think unusual limitations should be explicitly
> called out.
>
>> +#define kfree_rcu(ptr, rhv) \
>> +do { \
>> + if (!kfree_cb_offset) \
>> + kfree_cb_offset = offsetof(typeof(*(ptr)), rhv); \
>> + \
>> + call_rcu(&ptr->rhv, kfree_rcu_cb); \
>> +} while (0)
>
> Any specific reason kfree_rcu() is a macro and not a static inline function?
Think it's needed for the typeof() to work. The kernel's kfree_rcu() is
similar in this aspect.
>> +
>> #endif
>>
>> --
>> 2.50.1
>>
On Tue, Aug 26, 2025 at 2:32 AM Vlastimil Babka <vbabka@suse.cz> wrote:
>
> On 8/22/25 18:28, Suren Baghdasaryan wrote:
> >> diff --git a/tools/testing/shared/linux/rcupdate.h b/tools/testing/shared/linux/rcupdate.h
> >> index fed468fb0c78db6f33fb1900c7110ab5f3c19c65..c95e2f0bbd93798e544d7d34e0823ed68414f924 100644
> >> --- a/tools/testing/shared/linux/rcupdate.h
> >> +++ b/tools/testing/shared/linux/rcupdate.h
> >> @@ -9,4 +9,26 @@
> >> #define rcu_dereference_check(p, cond) rcu_dereference(p)
> >> #define RCU_INIT_POINTER(p, v) do { (p) = (v); } while (0)
> >>
> >> +void kmem_cache_free_active(void *objp);
> >> +static unsigned long kfree_cb_offset = 0;
> >> +
> >> +static inline void kfree_rcu_cb(struct rcu_head *head)
> >> +{
> >> + void *objp = (void *) ((unsigned long)head - kfree_cb_offset);
> >> +
> >> + kmem_cache_free_active(objp);
> >> +}
> >> +
> >> +#ifndef offsetof
> >> +#define offsetof(TYPE, MEMBER) __builtin_offsetof(TYPE, MEMBER)
> >> +#endif
> >> +
> >
> > We need a comment here that concurrent kfree_rcu() calls are not
> > supported because they would override each other's kfree_cb_offset.
>
> I think it's a bit more complex and related to the commit log sentence "This
> only works with one kmem_cache, and only the first one used.". The first
> call to kfree_rcu sets kfree_cb_offset (but what if the rhv offset is
> actually 0?) so the others won't update it. So concurrent calls will work as
> far as from the same cache thus same offset. But I'd like Liam's
> confirmation and the comment text, if possible :)
>
> > Kinda obvious but I think unusual limitations should be explicitly
> > called out.
> >
> >> +#define kfree_rcu(ptr, rhv) \
> >> +do { \
> >> + if (!kfree_cb_offset) \
> >> + kfree_cb_offset = offsetof(typeof(*(ptr)), rhv); \
> >> + \
> >> + call_rcu(&ptr->rhv, kfree_rcu_cb); \
> >> +} while (0)
> >
> > Any specific reason kfree_rcu() is a macro and not a static inline function?
>
> Think it's needed for the typeof() to work. The kernel's kfree_rcu() is
> similar in this aspect.
Ah, got it. Thanks!
>
> >> +
> >> #endif
> >>
> >> --
> >> 2.50.1
> >>
>
© 2016 - 2025 Red Hat, Inc.