From nobody Wed Oct 8 13:24:56 2025 Received: from mailrelay3-3.pub.mailoutpod3-cph3.one.com (mailrelay3-3.pub.mailoutpod3-cph3.one.com [46.30.212.2]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 4488221ADAE for ; Sat, 28 Jun 2025 10:26:02 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=46.30.212.2 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751106365; cv=none; b=XSlbrbEaoGCGCuxDumxAR5S5WGKAv7GG2vVQB6eWWldg/NEruJDGY9KkxRUqgXhPn3p8JmM+KRlS414C2o5l88jcaJkimEA5aE+yuQrHYB8YBg0yvpwwyKffypVdQQzxs8a9cCUDt1UM8ghzVs6ylmNmW5W7bFT5zLUwhvySha4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1751106365; c=relaxed/simple; bh=NoHgq+K26IsAvEj6C2qrVRM6ACWDYkgMMicsGvtZPXI=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=ss3ggLlKFzz4NvbxBtknwIN5jU35XGcGu555jEt6QaRNJiX1UO/SXEY4WL7jZ3Pe2/LLjN832MemvGzWiFgE5g5dxrab1nJ1OaSYcyBNOGS/jiHvmFJ9X5JQ/X+wG3uuifqI+E0zMW2WwPFrhrvnUPCU+TBLlqtzy3Vpynttggs= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=konsulko.se; spf=none smtp.mailfrom=konsulko.se; dkim=pass (2048-bit key) header.d=konsulko.se header.i=@konsulko.se header.b=CY72p/GA; dkim=permerror (0-bit key) header.d=konsulko.se header.i=@konsulko.se header.b=7quBsTVQ; arc=none smtp.client-ip=46.30.212.2 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=konsulko.se Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=konsulko.se Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=konsulko.se header.i=@konsulko.se header.b="CY72p/GA"; dkim=permerror (0-bit key) header.d=konsulko.se header.i=@konsulko.se header.b="7quBsTVQ" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; t=1751106360; x=1751711160; d=konsulko.se; s=rsa1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=4ZIeiPu/zic9E4s7Al+vArUw1KwvYwIAaepoA+hvdNI=; b=CY72p/GA46qhHX+X7eqno86I9FBm9pjk8KHgj9fIMIAUucDlWZ7HlB3gbj2emqV2sbufq8CgK43WI C8hjYPNUhLMC8hfCQXyUklvZuBITan2biDIZUAnMHDGQ6o8uvzPHfuCcFdPyIC6DWR0Nxko8txVmrk T3qXe6hayKCYrJ4T0AdUoiXIpx/Z29HYyUpMmRwt7u4VbN4llUFBOniuxryhKs51Yt3aoshetBHbTx CHVnNrXL6uHGaDQJBi10h6Gk8lBtVaWgpUVyngYtstzZgVmA0wxBOdzip1WqgvD1Z0WAZ/tRV3a4zu cicXQkKBI8MMfMgyUr+f0wzbkSNKkhg== DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed; t=1751106360; x=1751711160; d=konsulko.se; s=ed1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=4ZIeiPu/zic9E4s7Al+vArUw1KwvYwIAaepoA+hvdNI=; b=7quBsTVQt9qd1r8VSm44KNmBO+N55FIXNrmHMJqAb/DZU1OmNVyExxgdDFrYUmFg5AxHSYyu17sNS IxXsGPLDQ== X-HalOne-ID: 47d75440-540a-11f0-af5d-2b8368a4d5c5 Received: from slottsdator.home (host-90-238-19-233.mobileonline.telia.com [90.238.19.233]) by mailrelay3.pub.mailoutpod3-cph3.one.com (Halon) with ESMTPSA id 47d75440-540a-11f0-af5d-2b8368a4d5c5; Sat, 28 Jun 2025 10:25:59 +0000 (UTC) From: Vitaly Wool To: linux-mm@kvack.org Cc: akpm@linux-foundation.org, linux-kernel@vger.kernel.org, Uladzislau Rezki , Danilo Krummrich , Alice Ryhl , rust-for-linux@vger.kernel.org, Vitaly Wool Subject: [PATCH v8 2/4] mm/slub: allow to set node and align in k[v]realloc Date: Sat, 28 Jun 2025 12:25:58 +0200 Message-Id: <20250628102558.2542886-1-vitaly.wool@konsulko.se> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20250628102315.2542656-1-vitaly.wool@konsulko.se> References: <20250628102315.2542656-1-vitaly.wool@konsulko.se> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Reimplement k[v]realloc_node() to be able to set node and alignment should a user need to do so. In order to do that while retaining the maximal backward compatibility, add k[v]realloc_node_align() functions and redefine the rest of API using these new ones. With that change we also provide the ability for the Rust part of the kernel to set node and aligmnent in its K[v]xxx [re]allocations. Signed-off-by: Vitaly Wool --- include/linux/slab.h | 40 +++++++++++++++++++--------- mm/nommu.c | 3 ++- mm/slub.c | 63 ++++++++++++++++++++++++++++++-------------- 3 files changed, 72 insertions(+), 34 deletions(-) diff --git a/include/linux/slab.h b/include/linux/slab.h index d5a8ab98035c..13abcf4ada22 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -465,9 +465,15 @@ int kmem_cache_shrink(struct kmem_cache *s); /* * Common kmalloc functions provided by all allocators */ -void * __must_check krealloc_noprof(const void *objp, size_t new_size, - gfp_t flags) __realloc_size(2); -#define krealloc(...) alloc_hooks(krealloc_noprof(__VA_ARGS__)) +void * __must_check krealloc_node_align_noprof(const void *objp, size_t ne= w_size, + unsigned long align, + gfp_t flags, int nid) __realloc_size(2); +#define krealloc_node_noprof(_p, _s, _f, _n) \ + krealloc_node_align_noprof(_p, _s, 1, _f, _n) +#define krealloc_noprof(...) krealloc_node_noprof(__VA_ARGS__, NUMA_NO_NO= DE) +#define krealloc_node_align(...) alloc_hooks(krealloc_node_align_noprof(__= VA_ARGS__)) +#define krealloc_node(...) alloc_hooks(krealloc_node_noprof(__VA_ARGS__)) +#define krealloc(...) alloc_hooks(krealloc_noprof(__VA_ARGS__)) =20 void kfree(const void *objp); void kfree_sensitive(const void *objp); @@ -1041,18 +1047,23 @@ static inline __alloc_size(1) void *kzalloc_noprof(= size_t size, gfp_t flags) #define kzalloc(...) alloc_hooks(kzalloc_noprof(__VA_ARGS__)) #define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__= GFP_ZERO, _node) =20 -void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int= node) __alloc_size(1); -#define kvmalloc_node_noprof(size, flags, node) \ - __kvmalloc_node_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node) +void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long al= ign, + gfp_t flags, int node) __alloc_size(1); +#define kvmalloc_node_align_noprof(_size, _align, _flags, _node) \ + __kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, NULL), _align, _flags, _= node) +#define kvmalloc_node_noprof(_size, _flags, _node) \ + kvmalloc_node_align_noprof(_size, 1, _flags, _node) +#define kvmalloc_node_align(...) \ + alloc_hooks(kvmalloc_node_align_noprof(__VA_ARGS__)) #define kvmalloc_node(...) alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__)) =20 -#define kvmalloc(_size, _flags) kvmalloc_node(_size, _flags, NUMA_NO_NOD= E) -#define kvmalloc_noprof(_size, _flags) kvmalloc_node_noprof(_size, _flags= , NUMA_NO_NODE) +#define kvmalloc_noprof(...) kvmalloc_node_noprof(__VA_ARGS__, NUMA_NO_N= ODE) +#define kvmalloc(...) alloc_hooks(kvmalloc_noprof(__VA_ARGS__)) #define kvzalloc(_size, _flags) kvmalloc(_size, (_flags)|__GFP_ZERO) =20 -#define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, (_flags)|= __GFP_ZERO, _node) +#define kvzalloc_node(_s, _f, _n) kvmalloc_node(_s, (_f)|__GFP_ZERO, _n) #define kmem_buckets_valloc(_b, _size, _flags) \ - alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags,= NUMA_NO_NODE)) + alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), 1, _fla= gs, NUMA_NO_NODE)) =20 static inline __alloc_size(1, 2) void * kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node) @@ -1068,13 +1079,16 @@ kvmalloc_array_node_noprof(size_t n, size_t size, g= fp_t flags, int node) #define kvmalloc_array_noprof(...) kvmalloc_array_node_noprof(__VA_ARGS__= , NUMA_NO_NODE) #define kvcalloc_node_noprof(_n,_s,_f,_node) kvmalloc_array_node_noprof(_n= ,_s,(_f)|__GFP_ZERO,_node) #define kvcalloc_noprof(...) kvcalloc_node_noprof(__VA_ARGS__, NUMA_NO_N= ODE) - #define kvmalloc_array(...) alloc_hooks(kvmalloc_array_noprof(__VA_ARGS_= _)) #define kvcalloc_node(...) alloc_hooks(kvcalloc_node_noprof(__VA_ARGS__)) #define kvcalloc(...) alloc_hooks(kvcalloc_noprof(__VA_ARGS__)) =20 -void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags) - __realloc_size(2); +void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned lon= g align, + gfp_t flags, int nid) __realloc_size(2); +#define kvrealloc_node_align(...) kvrealloc_node_align_noprof(__VA_ARGS__) +#define kvrealloc_node_noprof(_p, _s, _f, _n) kvrealloc_node_align_noprof(= _p, _s, 1, _f, _n) +#define kvrealloc_node(...) alloc_hooks(kvrealloc_node_noprof(__VA_ARGS_= _)) +#define kvrealloc_noprof(...) kvrealloc_node_noprof(__VA_ARGS__, NUMA_NO= _NODE) #define kvrealloc(...) alloc_hooks(kvrealloc_noprof(__VA_ARGS__)) =20 extern void kvfree(const void *addr); diff --git a/mm/nommu.c b/mm/nommu.c index 87e1acab0d64..8359b2025b9f 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -119,7 +119,8 @@ void *__vmalloc_noprof(unsigned long size, gfp_t gfp_ma= sk) } EXPORT_SYMBOL(__vmalloc_noprof); =20 -void *vrealloc_noprof(const void *p, size_t size, gfp_t flags) +void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long= align, + gfp_t flags, int node) { return krealloc_noprof(p, size, (flags | __GFP_COMP) & ~__GFP_HIGHMEM); } diff --git a/mm/slub.c b/mm/slub.c index c4b64821e680..ec355ce31965 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4845,7 +4845,7 @@ void kfree(const void *object) EXPORT_SYMBOL(kfree); =20 static __always_inline __realloc_size(2) void * -__do_krealloc(const void *p, size_t new_size, gfp_t flags) +__do_krealloc(const void *p, size_t new_size, unsigned long align, gfp_t f= lags, int nid) { void *ret; size_t ks =3D 0; @@ -4859,6 +4859,19 @@ __do_krealloc(const void *p, size_t new_size, gfp_t = flags) if (!kasan_check_byte(p)) return NULL; =20 + /* refuse to proceed if alignment is bigger than what kmalloc() provides = */ + if (((uintptr_t)p & (align - 1)) || new_size < align) + return NULL; + + /* + * it is possible to support reallocation with a different nid, but + * it doesn't go well with the concept of krealloc(). Such + * reallocation should be done explicitly instead. + */ + if (WARN(nid !=3D NUMA_NO_NODE && nid !=3D page_to_nid(virt_to_page(p)), + "krealloc() has mismatched nids\n")) + return NULL; + if (is_kfence_address(p)) { ks =3D orig_size =3D kfence_ksize(p); } else { @@ -4903,7 +4916,7 @@ __do_krealloc(const void *p, size_t new_size, gfp_t f= lags) return (void *)p; =20 alloc_new: - ret =3D kmalloc_node_track_caller_noprof(new_size, flags, NUMA_NO_NODE, _= RET_IP_); + ret =3D kmalloc_node_track_caller_noprof(new_size, flags, nid, _RET_IP_); if (ret && p) { /* Disable KASAN checks as the object's redzone is accessed. */ kasan_disable_current(); @@ -4915,10 +4928,12 @@ __do_krealloc(const void *p, size_t new_size, gfp_t= flags) } =20 /** - * krealloc - reallocate memory. The contents will remain unchanged. + * krealloc_node_align - reallocate memory. The contents will remain uncha= nged. * @p: object to reallocate memory for. * @new_size: how many bytes of memory are required. + * @align: desired alignment. * @flags: the type of memory to allocate. + * @nid: NUMA node or NUMA_NO_NODE * * If @p is %NULL, krealloc() behaves exactly like kmalloc(). If @new_size * is 0 and @p is not a %NULL pointer, the object pointed to is freed. @@ -4947,7 +4962,8 @@ __do_krealloc(const void *p, size_t new_size, gfp_t f= lags) * * Return: pointer to the allocated memory or %NULL in case of error */ -void *krealloc_noprof(const void *p, size_t new_size, gfp_t flags) +void *krealloc_node_align_noprof(const void *p, size_t new_size, unsigned = long align, + gfp_t flags, int nid) { void *ret; =20 @@ -4956,13 +4972,13 @@ void *krealloc_noprof(const void *p, size_t new_siz= e, gfp_t flags) return ZERO_SIZE_PTR; } =20 - ret =3D __do_krealloc(p, new_size, flags); + ret =3D __do_krealloc(p, new_size, align, flags, nid); if (ret && kasan_reset_tag(p) !=3D kasan_reset_tag(ret)) kfree(p); =20 return ret; } -EXPORT_SYMBOL(krealloc_noprof); +EXPORT_SYMBOL(krealloc_node_align_noprof); =20 static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size) { @@ -4993,6 +5009,7 @@ static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t s= ize) * failure, fall back to non-contiguous (vmalloc) allocation. * @size: size of the request. * @b: which set of kmalloc buckets to allocate from. + * @align: desired alignment. * @flags: gfp mask for the allocation - must be compatible (superset) wit= h GFP_KERNEL. * @node: numa node to allocate from * @@ -5005,19 +5022,22 @@ static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t= size) * * Return: pointer to the allocated memory of %NULL in case of failure */ -void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int= node) +void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long al= ign, + gfp_t flags, int node) { void *ret; =20 /* * It doesn't really make sense to fallback to vmalloc for sub page - * requests + * requests and small alignments */ - ret =3D __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), - kmalloc_gfp_adjust(flags, size), - node, _RET_IP_); - if (ret || size <=3D PAGE_SIZE) - return ret; + if (size >=3D align) { + ret =3D __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), + kmalloc_gfp_adjust(flags, size), + node, _RET_IP_); + if (ret || size <=3D PAGE_SIZE) + return ret; + } =20 /* non-sleeping allocations are not supported by vmalloc */ if (!gfpflags_allow_blocking(flags)) @@ -5035,7 +5055,7 @@ void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size,= b), gfp_t flags, int node) * about the resulting pointer, and cannot play * protection games. */ - return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END, + return __vmalloc_node_range_noprof(size, align, VMALLOC_START, VMALLOC_EN= D, flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, node, __builtin_return_address(0)); } @@ -5079,10 +5099,12 @@ void kvfree_sensitive(const void *addr, size_t len) EXPORT_SYMBOL(kvfree_sensitive); =20 /** - * kvrealloc - reallocate memory; contents remain unchanged + * kvrealloc_node_align - reallocate memory; contents remain unchanged * @p: object to reallocate memory for * @size: the size to reallocate + * @align: desired alignment * @flags: the flags for the page level allocator + * @nid: NUMA node id * * If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size i= s 0 * and @p is not a %NULL pointer, the object pointed to is freed. @@ -5100,17 +5122,18 @@ EXPORT_SYMBOL(kvfree_sensitive); * * Return: pointer to the allocated memory or %NULL in case of error */ -void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags) +void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned lon= g align, + gfp_t flags, int nid) { void *n; =20 if (is_vmalloc_addr(p)) - return vrealloc_noprof(p, size, flags); + return vrealloc_node_align_noprof(p, size, align, flags, nid); =20 - n =3D krealloc_noprof(p, size, kmalloc_gfp_adjust(flags, size)); + n =3D krealloc_node_align_noprof(p, size, align, kmalloc_gfp_adjust(flags= , size), nid); if (!n) { /* We failed to krealloc(), fall back to kvmalloc(). */ - n =3D kvmalloc_noprof(size, flags); + n =3D kvmalloc_node_align_noprof(size, align, flags, nid); if (!n) return NULL; =20 @@ -5126,7 +5149,7 @@ void *kvrealloc_noprof(const void *p, size_t size, gf= p_t flags) =20 return n; } -EXPORT_SYMBOL(kvrealloc_noprof); +EXPORT_SYMBOL(kvrealloc_node_align_noprof); =20 struct detached_freelist { struct slab *slab; --=20 2.39.2