[RFC PATCH V3 7/7] mm/slab: place slabobj_ext metadata in unused space within s->size

Harry Yoo posted 7 patches 3 months, 2 weeks ago
[RFC PATCH V3 7/7] mm/slab: place slabobj_ext metadata in unused space within s->size
Posted by Harry Yoo 3 months, 2 weeks ago
When a cache has high s->align value and s->object_size is not aligned
to it, each object ends up with some unused space because of alignment.
If this wasted space is big enough, we can use it to store the
slabobj_ext metadata instead of wasting it.

On my system, this happens with caches like kmem_cache, mm_struct, pid,
task_struct, sighand_cache, xfs_inode, and others.

To place the slabobj_ext metadata within each object, the existing
slab_obj_ext() logic can still be used by setting:

  - slab->obj_exts = slab_address(slab) + s->red_left_zone +
                     (slabobj_ext offset)
  - stride = s->size

slab_obj_ext() doesn't need know where the metadata is stored,
so this method works without adding extra overhead to slab_obj_ext().

A good example benefiting from this optimization is xfs_inode
(object_size: 992, align: 64). To measure memory savings, 2 millions of
files were created on XFS.

[ MEMCG=y, MEM_ALLOC_PROFILING=n ]

Before patch (creating 2M directories on xfs):
  Slab:            6693844 kB
  SReclaimable:    6016332 kB
  SUnreclaim:       677512 kB

After patch (creating 2M directories on xfs):
  Slab:            6697572 kB
  SReclaimable:    6034744 kB
  SUnreclaim:       662828 kB (-14.3 MiB)

Enjoy the memory savings!

Suggested-by: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: Harry Yoo <harry.yoo@oracle.com>
---
 include/linux/slab.h |  9 ++++++
 mm/slab_common.c     |  6 ++--
 mm/slub.c            | 72 ++++++++++++++++++++++++++++++++++++++++++--
 3 files changed, 82 insertions(+), 5 deletions(-)

diff --git a/include/linux/slab.h b/include/linux/slab.h
index 561597dd2164..fd09674cc117 100644
--- a/include/linux/slab.h
+++ b/include/linux/slab.h
@@ -59,6 +59,9 @@ enum _slab_flag_bits {
 	_SLAB_CMPXCHG_DOUBLE,
 #ifdef CONFIG_SLAB_OBJ_EXT
 	_SLAB_NO_OBJ_EXT,
+#endif
+#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
+	_SLAB_OBJ_EXT_IN_OBJ,
 #endif
 	_SLAB_FLAGS_LAST_BIT
 };
@@ -244,6 +247,12 @@ enum _slab_flag_bits {
 #define SLAB_NO_OBJ_EXT		__SLAB_FLAG_UNUSED
 #endif
 
+#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
+#define SLAB_OBJ_EXT_IN_OBJ	__SLAB_FLAG_BIT(_SLAB_OBJ_EXT_IN_OBJ)
+#else
+#define SLAB_OBJ_EXT_IN_OBJ	__SLAB_FLAG_UNUSED
+#endif
+
 /*
  * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests.
  *
diff --git a/mm/slab_common.c b/mm/slab_common.c
index 2c2ed2452271..bfe2f498e622 100644
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -43,11 +43,13 @@ DEFINE_MUTEX(slab_mutex);
 struct kmem_cache *kmem_cache;
 
 /*
- * Set of flags that will prevent slab merging
+ * Set of flags that will prevent slab merging.
+ * Any flag that adds per-object metadata should be included,
+ * since slab merging can update s->inuse that affects the metadata layout.
  */
 #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
 		SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \
-		SLAB_FAILSLAB | SLAB_NO_MERGE)
+		SLAB_FAILSLAB | SLAB_NO_MERGE | SLAB_OBJ_EXT_IN_OBJ)
 
 #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \
 			 SLAB_CACHE_DMA32 | SLAB_ACCOUNT)
diff --git a/mm/slub.c b/mm/slub.c
index 8101df5fdccf..7de6e8f8f8c2 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -970,6 +970,40 @@ static inline bool obj_exts_in_slab(struct kmem_cache *s, struct slab *slab)
 {
 	return false;
 }
+
+#endif
+
+#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
+static bool obj_exts_in_object(struct kmem_cache *s)
+{
+	return s->flags & SLAB_OBJ_EXT_IN_OBJ;
+}
+
+static unsigned int obj_exts_offset_in_object(struct kmem_cache *s)
+{
+	unsigned int offset = get_info_end(s);
+
+	if (kmem_cache_debug_flags(s, SLAB_STORE_USER))
+		offset += sizeof(struct track) * 2;
+
+	if (slub_debug_orig_size(s))
+		offset += ALIGN(sizeof(unsigned int),
+				__alignof__(unsigned long));
+
+	offset += kasan_metadata_size(s, false);
+
+	return offset;
+}
+#else
+static inline bool obj_exts_in_object(struct kmem_cache *s)
+{
+	return false;
+}
+
+static inline unsigned int obj_exts_offset_in_object(struct kmem_cache *s)
+{
+	return 0;
+}
 #endif
 
 #ifdef CONFIG_SLUB_DEBUG
@@ -1270,6 +1304,9 @@ static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p)
 
 	off += kasan_metadata_size(s, false);
 
+	if (obj_exts_in_object(s))
+		off += sizeof(struct slabobj_ext);
+
 	if (off != size_from_object(s))
 		/* Beginning of the filler is the free pointer */
 		print_section(KERN_ERR, "Padding  ", p + off,
@@ -1439,7 +1476,10 @@ check_bytes_and_report(struct kmem_cache *s, struct slab *slab,
  * 	A. Free pointer (if we cannot overwrite object on free)
  * 	B. Tracking data for SLAB_STORE_USER
  *	C. Original request size for kmalloc object (SLAB_STORE_USER enabled)
- *	D. Padding to reach required alignment boundary or at minimum
+ *	D. KASAN alloc metadata (KASAN enabled)
+ *	E. struct slabobj_ext to store accounting metadata
+ *	   (SLAB_OBJ_EXT_IN_OBJ enabled)
+ *	F. Padding to reach required alignment boundary or at minimum
  * 		one word if debugging is on to be able to detect writes
  * 		before the word boundary.
  *
@@ -1468,6 +1508,9 @@ static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p)
 
 	off += kasan_metadata_size(s, false);
 
+	if (obj_exts_in_object(s))
+		off += sizeof(struct slabobj_ext);
+
 	if (size_from_object(s) == off)
 		return 1;
 
@@ -2250,7 +2293,8 @@ static inline void free_slab_obj_exts(struct slab *slab)
 	if (!obj_exts)
 		return;
 
-	if (obj_exts_in_slab(slab->slab_cache, slab)) {
+	if (obj_exts_in_slab(slab->slab_cache, slab) ||
+			obj_exts_in_object(slab->slab_cache)) {
 		slab->obj_exts = 0;
 		return;
 	}
@@ -2291,6 +2335,21 @@ static void alloc_slab_obj_exts_early(struct kmem_cache *s, struct slab *slab)
 		if (IS_ENABLED(CONFIG_MEMCG))
 			slab->obj_exts |= MEMCG_DATA_OBJEXTS;
 		slab_set_stride(slab, sizeof(struct slabobj_ext));
+	} else if (obj_exts_in_object(s)) {
+		unsigned int offset = obj_exts_offset_in_object(s);
+
+		slab->obj_exts = (unsigned long)slab_address(slab);
+		slab->obj_exts += s->red_left_pad;
+		slab->obj_exts += obj_exts_offset_in_object(s);
+		if (IS_ENABLED(CONFIG_MEMCG))
+			slab->obj_exts |= MEMCG_DATA_OBJEXTS;
+		slab_set_stride(slab, s->size);
+
+		for_each_object(addr, s, slab_address(slab), slab->objects) {
+			kasan_unpoison_range(addr + offset,
+					     sizeof(struct slabobj_ext));
+			memset(addr + offset, 0, sizeof(struct slabobj_ext));
+		}
 	}
 	metadata_access_disable();
 }
@@ -7883,6 +7942,7 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s)
 {
 	slab_flags_t flags = s->flags;
 	unsigned int size = s->object_size;
+	unsigned int aligned_size;
 	unsigned int order;
 
 	/*
@@ -7997,7 +8057,13 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s)
 	 * offset 0. In order to align the objects we have to simply size
 	 * each object to conform to the alignment.
 	 */
-	size = ALIGN(size, s->align);
+	aligned_size = ALIGN(size, s->align);
+#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
+	if (aligned_size - size >= sizeof(struct slabobj_ext))
+		s->flags |= SLAB_OBJ_EXT_IN_OBJ;
+#endif
+	size = aligned_size;
+
 	s->size = size;
 	s->reciprocal_size = reciprocal_value(size);
 	order = calculate_order(size);
-- 
2.43.0
Re: [RFC PATCH V3 7/7] mm/slab: place slabobj_ext metadata in unused space within s->size
Posted by Andrey Ryabinin 3 months, 1 week ago

On 10/27/25 1:28 PM, Harry Yoo wrote:

>  			slab->obj_exts |= MEMCG_DATA_OBJEXTS;
>  		slab_set_stride(slab, sizeof(struct slabobj_ext));
> +	} else if (obj_exts_in_object(s)) {
> +		unsigned int offset = obj_exts_offset_in_object(s);
> +
> +		slab->obj_exts = (unsigned long)slab_address(slab);
> +		slab->obj_exts += s->red_left_pad;
> +		slab->obj_exts += obj_exts_offset_in_object(s);
> +		if (IS_ENABLED(CONFIG_MEMCG))
> +			slab->obj_exts |= MEMCG_DATA_OBJEXTS;
> +		slab_set_stride(slab, s->size);
> +
> +		for_each_object(addr, s, slab_address(slab), slab->objects) {
> +			kasan_unpoison_range(addr + offset,
> +					     sizeof(struct slabobj_ext));

Is this leftover from previous version? Otherwise I don't get why we unpoison this.

> +			memset(addr + offset, 0, sizeof(struct slabobj_ext));
> +		}
>  	}
>  	metadata_access_disable();
>  }
Re: [RFC PATCH V3 7/7] mm/slab: place slabobj_ext metadata in unused space within s->size
Posted by Harry Yoo 3 months, 1 week ago
On Wed, Oct 29, 2025 at 07:19:29PM +0100, Andrey Ryabinin wrote:
> 
> 
> On 10/27/25 1:28 PM, Harry Yoo wrote:
> 
> >  			slab->obj_exts |= MEMCG_DATA_OBJEXTS;
> >  		slab_set_stride(slab, sizeof(struct slabobj_ext));
> > +	} else if (obj_exts_in_object(s)) {
> > +		unsigned int offset = obj_exts_offset_in_object(s);
> > +
> > +		slab->obj_exts = (unsigned long)slab_address(slab);
> > +		slab->obj_exts += s->red_left_pad;
> > +		slab->obj_exts += obj_exts_offset_in_object(s);
> > +		if (IS_ENABLED(CONFIG_MEMCG))
> > +			slab->obj_exts |= MEMCG_DATA_OBJEXTS;
> > +		slab_set_stride(slab, s->size);
> > +
> > +		for_each_object(addr, s, slab_address(slab), slab->objects) {
> > +			kasan_unpoison_range(addr + offset,
> > +					     sizeof(struct slabobj_ext));
> 
> Is this leftover from previous version? Otherwise I don't get why we unpoison this.

Oh god, yes! Thanks for catching. Will fix in the next version.

> > +			memset(addr + offset, 0, sizeof(struct slabobj_ext));
> > +		}
> >  	}
> >  	metadata_access_disable();
> >  }

-- 
Cheers,
Harry / Hyeonggon
Re: [RFC PATCH V3 7/7] mm/slab: place slabobj_ext metadata in unused space within s->size
Posted by Yeoreum Yun 3 months, 1 week ago
Hi Harry,
> On Wed, Oct 29, 2025 at 07:19:29PM +0100, Andrey Ryabinin wrote:
> >
> >
> > On 10/27/25 1:28 PM, Harry Yoo wrote:
> >
> > >  			slab->obj_exts |= MEMCG_DATA_OBJEXTS;
> > >  		slab_set_stride(slab, sizeof(struct slabobj_ext));
> > > +	} else if (obj_exts_in_object(s)) {
> > > +		unsigned int offset = obj_exts_offset_in_object(s);
> > > +
> > > +		slab->obj_exts = (unsigned long)slab_address(slab);
> > > +		slab->obj_exts += s->red_left_pad;
> > > +		slab->obj_exts += obj_exts_offset_in_object(s);
> > > +		if (IS_ENABLED(CONFIG_MEMCG))
> > > +			slab->obj_exts |= MEMCG_DATA_OBJEXTS;
> > > +		slab_set_stride(slab, s->size);
> > > +
> > > +		for_each_object(addr, s, slab_address(slab), slab->objects) {
> > > +			kasan_unpoison_range(addr + offset,
> > > +					     sizeof(struct slabobj_ext));
> >
> > Is this leftover from previous version? Otherwise I don't get why we unpoison this.
>
> Oh god, yes! Thanks for catching. Will fix in the next version.
>

Not only this, there would be possible case for WARN_ON() in the
kasan_unpoison_range() for unaligned address with KASAN_GRANULE_SIZE
when:
    - No debug information.
    - object size = 24 byte.
    - align  = 32 bytes.
    - sizeof(struct slabobj_ext) = 8 (CONFIG_MEMCG=y && CONFIG_MEM_ALLOC_PROFILING=n)
    - using KASAN_HW_TAG (KASAN_GRANULE_SIZE = 16 bytes).

Thanks.

--
Sincerely,
Yeoreum Yun
Re: [RFC PATCH V3 7/7] mm/slab: place slabobj_ext metadata in unused space within s->size
Posted by Suren Baghdasaryan 3 months, 1 week ago
On Mon, Oct 27, 2025 at 5:29 AM Harry Yoo <harry.yoo@oracle.com> wrote:
>
> When a cache has high s->align value and s->object_size is not aligned
> to it, each object ends up with some unused space because of alignment.
> If this wasted space is big enough, we can use it to store the
> slabobj_ext metadata instead of wasting it.
>
> On my system, this happens with caches like kmem_cache, mm_struct, pid,
> task_struct, sighand_cache, xfs_inode, and others.
>
> To place the slabobj_ext metadata within each object, the existing
> slab_obj_ext() logic can still be used by setting:
>
>   - slab->obj_exts = slab_address(slab) + s->red_left_zone +
>                      (slabobj_ext offset)
>   - stride = s->size
>
> slab_obj_ext() doesn't need know where the metadata is stored,
> so this method works without adding extra overhead to slab_obj_ext().
>
> A good example benefiting from this optimization is xfs_inode
> (object_size: 992, align: 64). To measure memory savings, 2 millions of
> files were created on XFS.
>
> [ MEMCG=y, MEM_ALLOC_PROFILING=n ]
>
> Before patch (creating 2M directories on xfs):
>   Slab:            6693844 kB
>   SReclaimable:    6016332 kB
>   SUnreclaim:       677512 kB
>
> After patch (creating 2M directories on xfs):
>   Slab:            6697572 kB
>   SReclaimable:    6034744 kB
>   SUnreclaim:       662828 kB (-14.3 MiB)
>
> Enjoy the memory savings!
>
> Suggested-by: Vlastimil Babka <vbabka@suse.cz>
> Signed-off-by: Harry Yoo <harry.yoo@oracle.com>
> ---
>  include/linux/slab.h |  9 ++++++
>  mm/slab_common.c     |  6 ++--
>  mm/slub.c            | 72 ++++++++++++++++++++++++++++++++++++++++++--
>  3 files changed, 82 insertions(+), 5 deletions(-)
>
> diff --git a/include/linux/slab.h b/include/linux/slab.h
> index 561597dd2164..fd09674cc117 100644
> --- a/include/linux/slab.h
> +++ b/include/linux/slab.h
> @@ -59,6 +59,9 @@ enum _slab_flag_bits {
>         _SLAB_CMPXCHG_DOUBLE,
>  #ifdef CONFIG_SLAB_OBJ_EXT
>         _SLAB_NO_OBJ_EXT,
> +#endif
> +#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
> +       _SLAB_OBJ_EXT_IN_OBJ,
>  #endif
>         _SLAB_FLAGS_LAST_BIT
>  };
> @@ -244,6 +247,12 @@ enum _slab_flag_bits {
>  #define SLAB_NO_OBJ_EXT                __SLAB_FLAG_UNUSED
>  #endif
>
> +#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
> +#define SLAB_OBJ_EXT_IN_OBJ    __SLAB_FLAG_BIT(_SLAB_OBJ_EXT_IN_OBJ)
> +#else
> +#define SLAB_OBJ_EXT_IN_OBJ    __SLAB_FLAG_UNUSED
> +#endif
> +
>  /*
>   * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests.
>   *
> diff --git a/mm/slab_common.c b/mm/slab_common.c
> index 2c2ed2452271..bfe2f498e622 100644
> --- a/mm/slab_common.c
> +++ b/mm/slab_common.c
> @@ -43,11 +43,13 @@ DEFINE_MUTEX(slab_mutex);
>  struct kmem_cache *kmem_cache;
>
>  /*
> - * Set of flags that will prevent slab merging
> + * Set of flags that will prevent slab merging.
> + * Any flag that adds per-object metadata should be included,
> + * since slab merging can update s->inuse that affects the metadata layout.
>   */
>  #define SLAB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \
>                 SLAB_TRACE | SLAB_TYPESAFE_BY_RCU | SLAB_NOLEAKTRACE | \
> -               SLAB_FAILSLAB | SLAB_NO_MERGE)
> +               SLAB_FAILSLAB | SLAB_NO_MERGE | SLAB_OBJ_EXT_IN_OBJ)
>
>  #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \
>                          SLAB_CACHE_DMA32 | SLAB_ACCOUNT)
> diff --git a/mm/slub.c b/mm/slub.c
> index 8101df5fdccf..7de6e8f8f8c2 100644
> --- a/mm/slub.c
> +++ b/mm/slub.c
> @@ -970,6 +970,40 @@ static inline bool obj_exts_in_slab(struct kmem_cache *s, struct slab *slab)
>  {
>         return false;
>  }
> +
> +#endif
> +
> +#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
> +static bool obj_exts_in_object(struct kmem_cache *s)
> +{
> +       return s->flags & SLAB_OBJ_EXT_IN_OBJ;
> +}
> +
> +static unsigned int obj_exts_offset_in_object(struct kmem_cache *s)
> +{
> +       unsigned int offset = get_info_end(s);
> +
> +       if (kmem_cache_debug_flags(s, SLAB_STORE_USER))
> +               offset += sizeof(struct track) * 2;
> +
> +       if (slub_debug_orig_size(s))
> +               offset += ALIGN(sizeof(unsigned int),
> +                               __alignof__(unsigned long));
> +
> +       offset += kasan_metadata_size(s, false);
> +
> +       return offset;
> +}
> +#else
> +static inline bool obj_exts_in_object(struct kmem_cache *s)
> +{
> +       return false;
> +}
> +
> +static inline unsigned int obj_exts_offset_in_object(struct kmem_cache *s)
> +{
> +       return 0;
> +}
>  #endif
>
>  #ifdef CONFIG_SLUB_DEBUG
> @@ -1270,6 +1304,9 @@ static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p)
>
>         off += kasan_metadata_size(s, false);
>
> +       if (obj_exts_in_object(s))
> +               off += sizeof(struct slabobj_ext);
> +
>         if (off != size_from_object(s))
>                 /* Beginning of the filler is the free pointer */
>                 print_section(KERN_ERR, "Padding  ", p + off,
> @@ -1439,7 +1476,10 @@ check_bytes_and_report(struct kmem_cache *s, struct slab *slab,
>   *     A. Free pointer (if we cannot overwrite object on free)
>   *     B. Tracking data for SLAB_STORE_USER
>   *     C. Original request size for kmalloc object (SLAB_STORE_USER enabled)
> - *     D. Padding to reach required alignment boundary or at minimum
> + *     D. KASAN alloc metadata (KASAN enabled)
> + *     E. struct slabobj_ext to store accounting metadata
> + *        (SLAB_OBJ_EXT_IN_OBJ enabled)
> + *     F. Padding to reach required alignment boundary or at minimum
>   *             one word if debugging is on to be able to detect writes
>   *             before the word boundary.
>   *
> @@ -1468,6 +1508,9 @@ static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p)
>
>         off += kasan_metadata_size(s, false);
>
> +       if (obj_exts_in_object(s))
> +               off += sizeof(struct slabobj_ext);
> +
>         if (size_from_object(s) == off)
>                 return 1;
>
> @@ -2250,7 +2293,8 @@ static inline void free_slab_obj_exts(struct slab *slab)
>         if (!obj_exts)
>                 return;
>
> -       if (obj_exts_in_slab(slab->slab_cache, slab)) {
> +       if (obj_exts_in_slab(slab->slab_cache, slab) ||
> +                       obj_exts_in_object(slab->slab_cache)) {


I think you need a check for obj_exts_in_object() inside
alloc_slab_obj_exts() to avoid allocating the vector.

>                 slab->obj_exts = 0;
>                 return;
>         }
> @@ -2291,6 +2335,21 @@ static void alloc_slab_obj_exts_early(struct kmem_cache *s, struct slab *slab)
>                 if (IS_ENABLED(CONFIG_MEMCG))
>                         slab->obj_exts |= MEMCG_DATA_OBJEXTS;
>                 slab_set_stride(slab, sizeof(struct slabobj_ext));
> +       } else if (obj_exts_in_object(s)) {
> +               unsigned int offset = obj_exts_offset_in_object(s);
> +
> +               slab->obj_exts = (unsigned long)slab_address(slab);
> +               slab->obj_exts += s->red_left_pad;
> +               slab->obj_exts += obj_exts_offset_in_object(s);
> +               if (IS_ENABLED(CONFIG_MEMCG))
> +                       slab->obj_exts |= MEMCG_DATA_OBJEXTS;
> +               slab_set_stride(slab, s->size);
> +
> +               for_each_object(addr, s, slab_address(slab), slab->objects) {
> +                       kasan_unpoison_range(addr + offset,
> +                                            sizeof(struct slabobj_ext));
> +                       memset(addr + offset, 0, sizeof(struct slabobj_ext));
> +               }
>         }
>         metadata_access_disable();
>  }
> @@ -7883,6 +7942,7 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s)
>  {
>         slab_flags_t flags = s->flags;
>         unsigned int size = s->object_size;
> +       unsigned int aligned_size;
>         unsigned int order;
>
>         /*
> @@ -7997,7 +8057,13 @@ static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s)
>          * offset 0. In order to align the objects we have to simply size
>          * each object to conform to the alignment.
>          */
> -       size = ALIGN(size, s->align);
> +       aligned_size = ALIGN(size, s->align);
> +#if defined(CONFIG_SLAB_OBJ_EXT) && defined(CONFIG_64BIT)
> +       if (aligned_size - size >= sizeof(struct slabobj_ext))
> +               s->flags |= SLAB_OBJ_EXT_IN_OBJ;
> +#endif
> +       size = aligned_size;
> +
>         s->size = size;
>         s->reciprocal_size = reciprocal_value(size);
>         order = calculate_order(size);
> --
> 2.43.0
>
Re: [RFC PATCH V3 7/7] mm/slab: place slabobj_ext metadata in unused space within s->size
Posted by Harry Yoo 1 month, 2 weeks ago
On Tue, Oct 28, 2025 at 08:19:59PM -0700, Suren Baghdasaryan wrote:
> On Mon, Oct 27, 2025 at 5:29 AM Harry Yoo <harry.yoo@oracle.com> wrote:
> >
> > When a cache has high s->align value and s->object_size is not aligned
> > to it, each object ends up with some unused space because of alignment.
> > If this wasted space is big enough, we can use it to store the
> > slabobj_ext metadata instead of wasting it.
> >
> > On my system, this happens with caches like kmem_cache, mm_struct, pid,
> > task_struct, sighand_cache, xfs_inode, and others.
> >
> > To place the slabobj_ext metadata within each object, the existing
> > slab_obj_ext() logic can still be used by setting:
> >
> >   - slab->obj_exts = slab_address(slab) + s->red_left_zone +
> >                      (slabobj_ext offset)
> >   - stride = s->size
> >
> > slab_obj_ext() doesn't need know where the metadata is stored,
> > so this method works without adding extra overhead to slab_obj_ext().
> >
> > A good example benefiting from this optimization is xfs_inode
> > (object_size: 992, align: 64). To measure memory savings, 2 millions of
> > files were created on XFS.
> >
> > [ MEMCG=y, MEM_ALLOC_PROFILING=n ]
> >
> > Before patch (creating 2M directories on xfs):
> >   Slab:            6693844 kB
> >   SReclaimable:    6016332 kB
> >   SUnreclaim:       677512 kB
> >
> > After patch (creating 2M directories on xfs):
> >   Slab:            6697572 kB
> >   SReclaimable:    6034744 kB
> >   SUnreclaim:       662828 kB (-14.3 MiB)
> >
> > Enjoy the memory savings!
> >
> > Suggested-by: Vlastimil Babka <vbabka@suse.cz>
> > Signed-off-by: Harry Yoo <harry.yoo@oracle.com>
> > ---
> > @@ -2250,7 +2293,8 @@ static inline void free_slab_obj_exts(struct slab *slab)
> >         if (!obj_exts)
> >                 return;
> >
> > -       if (obj_exts_in_slab(slab->slab_cache, slab)) {
> > +       if (obj_exts_in_slab(slab->slab_cache, slab) ||
> > +                       obj_exts_in_object(slab->slab_cache)) {
> 

Hi Suren, thanks for the comment.
I should have replied earlier, sorry.
 
> I think you need a check for obj_exts_in_object() inside
> alloc_slab_obj_exts() to avoid allocating the vector.

But slab_obj_exts() check before alloc_slab_obj_exts() should have
returned nonzero if obj_exts_in_object() returns true?

> >                 slab->obj_exts = 0;
> >                 return;
> >         }

-- 
Cheers,
Harry / Hyeonggon