From nobody Tue Oct 7 02:03:36 2025 Received: from mailrelay-egress16.pub.mailoutpod3-cph3.one.com (mailrelay-egress16.pub.mailoutpod3-cph3.one.com [46.30.212.3]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 060F12E3382 for ; Tue, 15 Jul 2025 13:58:07 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=46.30.212.3 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752587891; cv=none; b=tRWLWK1uMobs2BdEMCZ8igyrIOVFJbcGSbdZmgzcoodz/yA5sLayI1NCkMWWcnD1WPUljwn5oqh5mEjMMJgQrRItMxA4Ts759psrnzRXLxI2pB1keFdAbbEju2kZAID8GnNVqvhdMWcJYGuJufg6clAQEqA4/0D7800aVASirpI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752587891; c=relaxed/simple; bh=b11Dg6R/vlsqaqKkGCKON+PJNYHWYPrTti/tRvvl5Ps=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=Msvwoa2ojLdAvwAnCEAU+2M2YiGSmkfGHmLu/S0wno8ah5C8Kb95AmX1u0kDWpG3MvVaPVFRh+e6Ouxrf3Hvs4oTp8DYuzdHtSCDCVFQpv46QIWcrwabOHNCXkmrJDj653cNQuS2yDkZ0S9Elo7/dqtX+77q5Vdrf986Kq6Ct4E= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=konsulko.se; spf=none smtp.mailfrom=konsulko.se; dkim=pass (2048-bit key) header.d=konsulko.se header.i=@konsulko.se header.b=ZYkj/tH0; dkim=permerror (0-bit key) header.d=konsulko.se header.i=@konsulko.se header.b=K7s/z8ke; arc=none smtp.client-ip=46.30.212.3 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=konsulko.se Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=konsulko.se Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=konsulko.se header.i=@konsulko.se header.b="ZYkj/tH0"; dkim=permerror (0-bit key) header.d=konsulko.se header.i=@konsulko.se header.b="K7s/z8ke" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; t=1752587886; x=1753192686; d=konsulko.se; s=rsa1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=RvpazCzVxvXnIveADtjt84U407dyEylMhRtOiCnmbUA=; b=ZYkj/tH0/s0Co1qLPu9jhXAVZgLaM16V863OL7E+GKPk8A9exPRCUxW93Y1I9tNPVuS7PBk6ppW9b KGpn3f2TvWHpvpIxxhLVXunPZTyR/p58I7sdMBlQOpQM1wh1swpWRcmW2x8pYLEBo5KSz0NdpSuadb WD56pXT2bX4CDROy/8oa+iBijp/bCPKSevSHll86blURiM7fkCb92xCS4Br61Xy1EnXhn7VuASYESE JYStkaBCe18bu3jK3OfFEIffBrLxUxAOi31cEgQy1eMwicvLDcJFJFeWkP3XoaGqbiPd3Hef/a+B53 phyA+TJ5I6o/F3LGLm6bUjO6lvMtRvg== DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed; t=1752587886; x=1753192686; d=konsulko.se; s=ed1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=RvpazCzVxvXnIveADtjt84U407dyEylMhRtOiCnmbUA=; b=K7s/z8keFCDg2jxD/UnJCssqJywHAo1+Lm1JkRas/9lTcIvPj0eP5MtupCAWiBSiFDEoGzOqfGUrp iTJRct0CA== X-HalOne-ID: b99010bc-6183-11f0-8923-e90f2b8e16ca Received: from slottsdator.home (host-90-238-19-233.mobileonline.telia.com [90.238.19.233]) by mailrelay2.pub.mailoutpod2-cph3.one.com (Halon) with ESMTPSA id b99010bc-6183-11f0-8923-e90f2b8e16ca; Tue, 15 Jul 2025 13:58:05 +0000 (UTC) From: Vitaly Wool To: linux-mm@kvack.org Cc: akpm@linux-foundation.org, linux-kernel@vger.kernel.org, Uladzislau Rezki , Danilo Krummrich , Alice Ryhl , Vlastimil Babka , rust-for-linux@vger.kernel.org, Lorenzo Stoakes , "Liam R . Howlett" , Kent Overstreet , linux-bcachefs@vger.kernel.org, bpf@vger.kernel.org, Herbert Xu , Jann Horn , Pedro Falcato , Vitaly Wool Subject: [PATCH v13 1/4] mm/vmalloc: allow to set node and align in vrealloc Date: Tue, 15 Jul 2025 15:58:03 +0200 Message-Id: <20250715135803.2230193-1-vitaly.wool@konsulko.se> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20250715135645.2230065-1-vitaly.wool@konsulko.se> References: <20250715135645.2230065-1-vitaly.wool@konsulko.se> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Reimplement vrealloc() to be able to set node and alignment should a user need to do so. Rename the function to vrealloc_node_align() to better match what it actually does now and introduce macros for vrealloc() and friends for backward compatibility. With that change we also provide the ability for the Rust part of the kernel to set node and alignment in its allocations. Signed-off-by: Vitaly Wool Reviewed-by: Uladzislau Rezki (Sony) Reviewed-by: Vlastimil Babka Acked-by already. Reviewed-by: Liam R. Howlett --- include/linux/vmalloc.h | 12 +++++++++--- mm/nommu.c | 3 ++- mm/vmalloc.c | 29 ++++++++++++++++++++++++----- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index fdc9aeb74a44..68791f7cb3ba 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -197,9 +197,15 @@ extern void *__vcalloc_noprof(size_t n, size_t size, g= fp_t flags) __alloc_size(1 extern void *vcalloc_noprof(size_t n, size_t size) __alloc_size(1, 2); #define vcalloc(...) alloc_hooks(vcalloc_noprof(__VA_ARGS__)) =20 -void * __must_check vrealloc_noprof(const void *p, size_t size, gfp_t flag= s) - __realloc_size(2); -#define vrealloc(...) alloc_hooks(vrealloc_noprof(__VA_ARGS__)) +void *__must_check vrealloc_node_align_noprof(const void *p, size_t size, + unsigned long align, gfp_t flags, int nid) __realloc_size(2); +#define vrealloc_node_noprof(_p, _s, _f, _nid) \ + vrealloc_node_align_noprof(_p, _s, 1, _f, _nid) +#define vrealloc_noprof(_p, _s, _f) \ + vrealloc_node_align_noprof(_p, _s, 1, _f, NUMA_NO_NODE) +#define vrealloc_node_align(...) alloc_hooks(vrealloc_node_align_noprof(_= _VA_ARGS__)) +#define vrealloc_node(...) alloc_hooks(vrealloc_node_noprof(__VA_ARGS__)) +#define vrealloc(...) alloc_hooks(vrealloc_noprof(__VA_ARGS__)) =20 extern void vfree(const void *addr); extern void vfree_atomic(const void *addr); diff --git a/mm/nommu.c b/mm/nommu.c index b624acec6d2e..afde6c626b07 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -119,7 +119,8 @@ void *__vmalloc_noprof(unsigned long size, gfp_t gfp_ma= sk) } EXPORT_SYMBOL(__vmalloc_noprof); =20 -void *vrealloc_noprof(const void *p, size_t size, gfp_t flags) +void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long= align, + gfp_t flags, int node) { return krealloc_noprof(p, size, (flags | __GFP_COMP) & ~__GFP_HIGHMEM); } diff --git a/mm/vmalloc.c b/mm/vmalloc.c index ab986dd09b6a..e0a593651d96 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -4081,19 +4081,29 @@ void *vzalloc_node_noprof(unsigned long size, int n= ode) EXPORT_SYMBOL(vzalloc_node_noprof); =20 /** - * vrealloc - reallocate virtually contiguous memory; contents remain unch= anged + * vrealloc_node_align_noprof - reallocate virtually contiguous memory; co= ntents + * remain unchanged * @p: object to reallocate memory for * @size: the size to reallocate + * @align: requested alignment * @flags: the flags for the page level allocator + * @nid: node number of the target node + * + * If @p is %NULL, vrealloc_XXX() behaves exactly like vmalloc_XXX(). If @= size + * is 0 and @p is not a %NULL pointer, the object pointed to is freed. * - * If @p is %NULL, vrealloc() behaves exactly like vmalloc(). If @size is = 0 and - * @p is not a %NULL pointer, the object pointed to is freed. + * If the caller wants the new memory to be on specific node *only*, + * __GFP_THISNODE flag should be set, otherwise the function will try to a= void + * reallocation and possibly disregard the specified @nid. * * If __GFP_ZERO logic is requested, callers must ensure that, starting wi= th the * initial memory allocation, every subsequent call to this API for the sa= me * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible= that * __GFP_ZERO is not fully honored by this API. * + * Requesting an alignment that is bigger than the alignment of the existi= ng + * allocation will fail. + * * In any case, the contents of the object pointed to are preserved up to = the * lesser of the new and old sizes. * @@ -4103,7 +4113,8 @@ EXPORT_SYMBOL(vzalloc_node_noprof); * Return: pointer to the allocated memory; %NULL if @size is zero or in c= ase of * failure */ -void *vrealloc_noprof(const void *p, size_t size, gfp_t flags) +void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long= align, + gfp_t flags, int nid) { struct vm_struct *vm =3D NULL; size_t alloced_size =3D 0; @@ -4127,6 +4138,12 @@ void *vrealloc_noprof(const void *p, size_t size, gf= p_t flags) if (WARN(alloced_size < old_size, "vrealloc() has mismatched area vs requested sizes (%p)\n", p)) return NULL; + if (WARN(!IS_ALIGNED((unsigned long)p, align), + "will not reallocate with a bigger alignment (0x%lx)\n", align)) + return NULL; + if (unlikely(flags & __GFP_THISNODE) && nid !=3D NUMA_NO_NODE && + nid !=3D page_to_nid(vmalloc_to_page(p))) + goto need_realloc; } =20 /* @@ -4157,8 +4174,10 @@ void *vrealloc_noprof(const void *p, size_t size, gf= p_t flags) return (void *)p; } =20 +need_realloc: /* TODO: Grow the vm_area, i.e. allocate and map additional pages. */ - n =3D __vmalloc_noprof(size, flags); + n =3D __vmalloc_node_noprof(size, align, flags, nid, __builtin_return_add= ress(0)); + if (!n) return NULL; =20 --=20 2.39.2 From nobody Tue Oct 7 02:03:36 2025 Received: from mailrelay-egress16.pub.mailoutpod3-cph3.one.com (mailrelay-egress16.pub.mailoutpod3-cph3.one.com [46.30.212.3]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7A0DA2E426A for ; Tue, 15 Jul 2025 13:57:36 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=46.30.212.3 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752587859; cv=none; b=jc6qoS0mxgc8O5e0zSgOAm++y5LHRS1zpg4gwePIaKOnZS36PGY5esQdoV1bUBOYPDd481DJA209hS98KSPvUylVor82nHylczYpi7DRGpTBVbrAYpfkspc/jSelE08koPu8VVo712URi9n/Qmjw0wcrWq6vz/HKjQk96bmHqpI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752587859; c=relaxed/simple; bh=b11Dg6R/vlsqaqKkGCKON+PJNYHWYPrTti/tRvvl5Ps=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=hsSvGxIcyG22usk3hb5Qg0gxIh9UcET3EWpEKOc6GI8Kdj+JHUqMQPlcwL5jyWHO2rSKLljblEeW/m8gjTGIsd5dHmdWIp5qWDVeKozIWjs5GqpV+SSGI8BoczDZB/fXcDTHK2n55tA0sV8SyuDZVJOJFFMwQqi9ECkJVJSjwnk= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=konsulko.se; spf=none smtp.mailfrom=konsulko.se; dkim=pass (2048-bit key) header.d=konsulko.se header.i=@konsulko.se header.b=UWbbR/Tc; dkim=permerror (0-bit key) header.d=konsulko.se header.i=@konsulko.se header.b=848oN+c1; arc=none smtp.client-ip=46.30.212.3 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=konsulko.se Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=konsulko.se Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=konsulko.se header.i=@konsulko.se header.b="UWbbR/Tc"; dkim=permerror (0-bit key) header.d=konsulko.se header.i=@konsulko.se header.b="848oN+c1" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; t=1752587854; x=1753192654; d=konsulko.se; s=rsa1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=RvpazCzVxvXnIveADtjt84U407dyEylMhRtOiCnmbUA=; b=UWbbR/Tc5H2QH0ma6nzlJH4+43hHSy99oZBjToIFA9zzlj1OjISXyp4xecXIK6N9OkcJYwvwNzpwO l9yz7zSmi16gfVnlLktA4U9NPTOWUwF+eLhT9UgOtFSKe/RWVfCGGA0Y/YsCOXT7nYaciVlUPejP4o Yro+8h2sN9CB095MAFh/npr33kYvkQlNNiQXbaeyezQPgRce1nHr5k/kMUKmhqhL6PPLJjamOPf1eY xWr357znGIVCHPebSj+fLlxCDGnlBfDAP2TnbdkBPip3d48PzxAsy7lpzN1jDXr6p2aazUaevY2zw7 9GoW9vp0jJTo28BtsbRmVQfnsz5XH4g== DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed; t=1752587854; x=1753192654; d=konsulko.se; s=ed1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=RvpazCzVxvXnIveADtjt84U407dyEylMhRtOiCnmbUA=; b=848oN+c1f8XMdcyZaAPG2ad29Xgmv0OPdHleViT6C1fEdDDsWBsFBdALoKvVJWbzLkfaSMCAhcaIa IIVfucCBA== X-HalOne-ID: a716537d-6183-11f0-9c7f-494313b7f784 Received: from slottsdator.home (host-90-238-19-233.mobileonline.telia.com [90.238.19.233]) by mailrelay6.pub.mailoutpod2-cph3.one.com (Halon) with ESMTPSA id a716537d-6183-11f0-9c7f-494313b7f784; Tue, 15 Jul 2025 13:57:33 +0000 (UTC) From: Vitaly Wool To: linux-mm@kvack.org Cc: akpm@linux-foundation.org, linux-kernel@vger.kernel.org, Uladzislau Rezki , Danilo Krummrich , Alice Ryhl , Vlastimil Babka , rust-for-linux@vger.kernel.org, Lorenzo Stoakes , "Liam R . Howlett" , Kent Overstreet , linux-bcachefs@vger.kernel.org, bpf@vger.kernel.org, Herbert Xu , Jann Horn , Pedro Falcato , Vitaly Wool Subject: [PATCH v13 1/4] :mm/vmalloc: allow to set node and align in vrealloc Date: Tue, 15 Jul 2025 15:57:24 +0200 Message-Id: <20250715135724.2230116-1-vitaly.wool@konsulko.se> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20250715135645.2230065-1-vitaly.wool@konsulko.se> References: <20250715135645.2230065-1-vitaly.wool@konsulko.se> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Reimplement vrealloc() to be able to set node and alignment should a user need to do so. Rename the function to vrealloc_node_align() to better match what it actually does now and introduce macros for vrealloc() and friends for backward compatibility. With that change we also provide the ability for the Rust part of the kernel to set node and alignment in its allocations. Signed-off-by: Vitaly Wool Reviewed-by: Uladzislau Rezki (Sony) Reviewed-by: Vlastimil Babka Acked-by already. --- include/linux/vmalloc.h | 12 +++++++++--- mm/nommu.c | 3 ++- mm/vmalloc.c | 29 ++++++++++++++++++++++++----- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/include/linux/vmalloc.h b/include/linux/vmalloc.h index fdc9aeb74a44..68791f7cb3ba 100644 --- a/include/linux/vmalloc.h +++ b/include/linux/vmalloc.h @@ -197,9 +197,15 @@ extern void *__vcalloc_noprof(size_t n, size_t size, g= fp_t flags) __alloc_size(1 extern void *vcalloc_noprof(size_t n, size_t size) __alloc_size(1, 2); #define vcalloc(...) alloc_hooks(vcalloc_noprof(__VA_ARGS__)) =20 -void * __must_check vrealloc_noprof(const void *p, size_t size, gfp_t flag= s) - __realloc_size(2); -#define vrealloc(...) alloc_hooks(vrealloc_noprof(__VA_ARGS__)) +void *__must_check vrealloc_node_align_noprof(const void *p, size_t size, + unsigned long align, gfp_t flags, int nid) __realloc_size(2); +#define vrealloc_node_noprof(_p, _s, _f, _nid) \ + vrealloc_node_align_noprof(_p, _s, 1, _f, _nid) +#define vrealloc_noprof(_p, _s, _f) \ + vrealloc_node_align_noprof(_p, _s, 1, _f, NUMA_NO_NODE) +#define vrealloc_node_align(...) alloc_hooks(vrealloc_node_align_noprof(_= _VA_ARGS__)) +#define vrealloc_node(...) alloc_hooks(vrealloc_node_noprof(__VA_ARGS__)) +#define vrealloc(...) alloc_hooks(vrealloc_noprof(__VA_ARGS__)) =20 extern void vfree(const void *addr); extern void vfree_atomic(const void *addr); diff --git a/mm/nommu.c b/mm/nommu.c index b624acec6d2e..afde6c626b07 100644 --- a/mm/nommu.c +++ b/mm/nommu.c @@ -119,7 +119,8 @@ void *__vmalloc_noprof(unsigned long size, gfp_t gfp_ma= sk) } EXPORT_SYMBOL(__vmalloc_noprof); =20 -void *vrealloc_noprof(const void *p, size_t size, gfp_t flags) +void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long= align, + gfp_t flags, int node) { return krealloc_noprof(p, size, (flags | __GFP_COMP) & ~__GFP_HIGHMEM); } diff --git a/mm/vmalloc.c b/mm/vmalloc.c index ab986dd09b6a..e0a593651d96 100644 --- a/mm/vmalloc.c +++ b/mm/vmalloc.c @@ -4081,19 +4081,29 @@ void *vzalloc_node_noprof(unsigned long size, int n= ode) EXPORT_SYMBOL(vzalloc_node_noprof); =20 /** - * vrealloc - reallocate virtually contiguous memory; contents remain unch= anged + * vrealloc_node_align_noprof - reallocate virtually contiguous memory; co= ntents + * remain unchanged * @p: object to reallocate memory for * @size: the size to reallocate + * @align: requested alignment * @flags: the flags for the page level allocator + * @nid: node number of the target node + * + * If @p is %NULL, vrealloc_XXX() behaves exactly like vmalloc_XXX(). If @= size + * is 0 and @p is not a %NULL pointer, the object pointed to is freed. * - * If @p is %NULL, vrealloc() behaves exactly like vmalloc(). If @size is = 0 and - * @p is not a %NULL pointer, the object pointed to is freed. + * If the caller wants the new memory to be on specific node *only*, + * __GFP_THISNODE flag should be set, otherwise the function will try to a= void + * reallocation and possibly disregard the specified @nid. * * If __GFP_ZERO logic is requested, callers must ensure that, starting wi= th the * initial memory allocation, every subsequent call to this API for the sa= me * memory allocation is flagged with __GFP_ZERO. Otherwise, it is possible= that * __GFP_ZERO is not fully honored by this API. * + * Requesting an alignment that is bigger than the alignment of the existi= ng + * allocation will fail. + * * In any case, the contents of the object pointed to are preserved up to = the * lesser of the new and old sizes. * @@ -4103,7 +4113,8 @@ EXPORT_SYMBOL(vzalloc_node_noprof); * Return: pointer to the allocated memory; %NULL if @size is zero or in c= ase of * failure */ -void *vrealloc_noprof(const void *p, size_t size, gfp_t flags) +void *vrealloc_node_align_noprof(const void *p, size_t size, unsigned long= align, + gfp_t flags, int nid) { struct vm_struct *vm =3D NULL; size_t alloced_size =3D 0; @@ -4127,6 +4138,12 @@ void *vrealloc_noprof(const void *p, size_t size, gf= p_t flags) if (WARN(alloced_size < old_size, "vrealloc() has mismatched area vs requested sizes (%p)\n", p)) return NULL; + if (WARN(!IS_ALIGNED((unsigned long)p, align), + "will not reallocate with a bigger alignment (0x%lx)\n", align)) + return NULL; + if (unlikely(flags & __GFP_THISNODE) && nid !=3D NUMA_NO_NODE && + nid !=3D page_to_nid(vmalloc_to_page(p))) + goto need_realloc; } =20 /* @@ -4157,8 +4174,10 @@ void *vrealloc_noprof(const void *p, size_t size, gf= p_t flags) return (void *)p; } =20 +need_realloc: /* TODO: Grow the vm_area, i.e. allocate and map additional pages. */ - n =3D __vmalloc_noprof(size, flags); + n =3D __vmalloc_node_noprof(size, align, flags, nid, __builtin_return_add= ress(0)); + if (!n) return NULL; =20 --=20 2.39.2 From nobody Tue Oct 7 02:03:36 2025 Received: from mailrelay-egress16.pub.mailoutpod3-cph3.one.com (mailrelay-egress16.pub.mailoutpod3-cph3.one.com [46.30.212.3]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id BBAF22E973D for ; Tue, 15 Jul 2025 13:58:20 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=46.30.212.3 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752587904; cv=none; b=a5ZI21sXLtrMypwWwtAdfwTxNvQmhtJ5zRDyEU77ouTAu5pSvyGQofx5G9hZUlhJM/Tw+vHMRfw3ni4jelZ3IUUD+c+TaSgEbxJE2u+aQr8dFmEkXHI8gx2AVwZBXwZ+xq3cMHfYOTiofisrJhLvZP0F/2IM0lNFFUEQSnDqhQA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752587904; c=relaxed/simple; bh=uDpABtpi7fABN40wEFRs+KZAVhNwks4LMtg+F/DbG+o=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=IcxspHvLmz1cllTUOcB/ldFrNZkkr/ZRqQdkbQBRNZxEXU4eHFpmpc6tGUj2R4gb3IHazo00/Rxsk8dC7V2QrCJnPnFM5Imr6A6HUXw776gIq7oZqTmDlAhit6vmwwO2yvEhlp707K+sxBh7eWhMoLpJXRJDvJhqdkX39eOOIGM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=konsulko.se; spf=none smtp.mailfrom=konsulko.se; dkim=pass (2048-bit key) header.d=konsulko.se header.i=@konsulko.se header.b=KLpJJGNq; dkim=permerror (0-bit key) header.d=konsulko.se header.i=@konsulko.se header.b=iLK0oDNm; arc=none smtp.client-ip=46.30.212.3 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=konsulko.se Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=konsulko.se Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=konsulko.se header.i=@konsulko.se header.b="KLpJJGNq"; dkim=permerror (0-bit key) header.d=konsulko.se header.i=@konsulko.se header.b="iLK0oDNm" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; t=1752587899; x=1753192699; d=konsulko.se; s=rsa1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=AZxHEJw5HUfz8R1OwaEl9a0TacuZRcBeb83siGw1DWA=; b=KLpJJGNqTOVswfilcT5TmS8mt6rFpCrj055owXe3SKn2C5P1BSUMNO976zldapwV/g95SvIBvfH7M KexRKnCdtkCMKpgwXI82mDPX15LbTs0DDlDZddzhRJc2pz5+3L/3xRVjHYKpPcXbzQe8g2W4WmZrc+ 4LBylbFpuSFn4KfxPb6MXvwA3j5n2GaTeP4tgv+Li2HNh7k8zNLY7geKo6mU1vc3joaLJYC631FNjE bZptN6x5qm3d5z8yJCxPsHAmPeDUY89Qcj1uqRor8Ao9rpVEsr651jePmA3hBF75tERwNdGYqr4AG5 Cr7RvJiXYL88/Hp8s7y9UmMcCpN4W1A== DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed; t=1752587899; x=1753192699; d=konsulko.se; s=ed1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=AZxHEJw5HUfz8R1OwaEl9a0TacuZRcBeb83siGw1DWA=; b=iLK0oDNmHCpRrpGtOOea2z7gedza8/bJXqguzrkHqtA/DnT5Z6V18dAwsmHltIl7W98KH1PUfYRnu 4yZt+3iDw== X-HalOne-ID: c0fddb25-6183-11f0-b805-d510462faafc Received: from slottsdator.home (host-90-238-19-233.mobileonline.telia.com [90.238.19.233]) by mailrelay4.pub.mailoutpod3-cph3.one.com (Halon) with ESMTPSA id c0fddb25-6183-11f0-b805-d510462faafc; Tue, 15 Jul 2025 13:58:18 +0000 (UTC) From: Vitaly Wool To: linux-mm@kvack.org Cc: akpm@linux-foundation.org, linux-kernel@vger.kernel.org, Uladzislau Rezki , Danilo Krummrich , Alice Ryhl , Vlastimil Babka , rust-for-linux@vger.kernel.org, Lorenzo Stoakes , "Liam R . Howlett" , Kent Overstreet , linux-bcachefs@vger.kernel.org, bpf@vger.kernel.org, Herbert Xu , Jann Horn , Pedro Falcato , Vitaly Wool Subject: [PATCH v13 2/4] mm/slub: allow to set node and align in k[v]realloc Date: Tue, 15 Jul 2025 15:58:15 +0200 Message-Id: <20250715135815.2230224-1-vitaly.wool@konsulko.se> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20250715135645.2230065-1-vitaly.wool@konsulko.se> References: <20250715135645.2230065-1-vitaly.wool@konsulko.se> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Reimplement k[v]realloc_node() to be able to set node and alignment should a user need to do so. In order to do that while retaining the maximal backward compatibility, add k[v]realloc_node_align() functions and redefine the rest of API using these new ones. While doing that, we also keep the number of _noprof variants to a minimum, which implies some changes to the existing users of older _noprof functions, that basically being bcachefs. With that change we also provide the ability for the Rust part of the kernel to set node and alignment in its K[v]xxx [re]allocations. Signed-off-by: Vitaly Wool Acked-by already. --- fs/bcachefs/darray.c | 2 +- fs/bcachefs/util.h | 2 +- include/linux/bpfptr.h | 2 +- include/linux/slab.h | 39 +++++++++++++++---------- lib/rhashtable.c | 4 +-- mm/slub.c | 64 +++++++++++++++++++++++++++++------------- 6 files changed, 73 insertions(+), 40 deletions(-) diff --git a/fs/bcachefs/darray.c b/fs/bcachefs/darray.c index e86d36d23e9e..928e83a1ce42 100644 --- a/fs/bcachefs/darray.c +++ b/fs/bcachefs/darray.c @@ -21,7 +21,7 @@ int __bch2_darray_resize_noprof(darray_char *d, size_t el= ement_size, size_t new_ return -ENOMEM; =20 void *data =3D likely(bytes < INT_MAX) - ? kvmalloc_noprof(bytes, gfp) + ? kvmalloc_node_align_noprof(bytes, 1, gfp, NUMA_NO_NODE) : vmalloc_noprof(bytes); if (!data) return -ENOMEM; diff --git a/fs/bcachefs/util.h b/fs/bcachefs/util.h index 0a4b1d433621..2d6d4b547db8 100644 --- a/fs/bcachefs/util.h +++ b/fs/bcachefs/util.h @@ -61,7 +61,7 @@ static inline void *bch2_kvmalloc_noprof(size_t n, gfp_t = flags) { void *p =3D unlikely(n >=3D INT_MAX) ? vmalloc_noprof(n) - : kvmalloc_noprof(n, flags & ~__GFP_ZERO); + : kvmalloc_node_align_noprof(n, 1, flags & ~__GFP_ZERO, NUMA_NO_NODE); if (p && (flags & __GFP_ZERO)) memset(p, 0, n); return p; diff --git a/include/linux/bpfptr.h b/include/linux/bpfptr.h index 1af241525a17..f6e0795db484 100644 --- a/include/linux/bpfptr.h +++ b/include/linux/bpfptr.h @@ -67,7 +67,7 @@ static inline int copy_to_bpfptr_offset(bpfptr_t dst, siz= e_t offset, =20 static inline void *kvmemdup_bpfptr_noprof(bpfptr_t src, size_t len) { - void *p =3D kvmalloc_noprof(len, GFP_USER | __GFP_NOWARN); + void *p =3D kvmalloc_node_align_noprof(len, 1, GFP_USER | __GFP_NOWARN, N= UMA_NO_NODE); =20 if (!p) return ERR_PTR(-ENOMEM); diff --git a/include/linux/slab.h b/include/linux/slab.h index d5a8ab98035c..6dc300bac2a1 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -465,9 +465,13 @@ int kmem_cache_shrink(struct kmem_cache *s); /* * Common kmalloc functions provided by all allocators */ -void * __must_check krealloc_noprof(const void *objp, size_t new_size, - gfp_t flags) __realloc_size(2); -#define krealloc(...) alloc_hooks(krealloc_noprof(__VA_ARGS__)) +void * __must_check krealloc_node_align_noprof(const void *objp, size_t ne= w_size, + unsigned long align, + gfp_t flags, int nid) __realloc_size(2); +#define krealloc_noprof(_o, _s, _f) krealloc_node_align_noprof(_o, _s, 1, = _f, NUMA_NO_NODE) +#define krealloc_node_align(...) alloc_hooks(krealloc_node_align_noprof(__= VA_ARGS__)) +#define krealloc_node(_o, _s, _f, _n) krealloc_node_align(_o, _s, 1, _f, _= n) +#define krealloc(...) krealloc_node(__VA_ARGS__, NUMA_NO_NODE) =20 void kfree(const void *objp); void kfree_sensitive(const void *objp); @@ -1041,18 +1045,20 @@ static inline __alloc_size(1) void *kzalloc_noprof(= size_t size, gfp_t flags) #define kzalloc(...) alloc_hooks(kzalloc_noprof(__VA_ARGS__)) #define kzalloc_node(_size, _flags, _node) kmalloc_node(_size, (_flags)|__= GFP_ZERO, _node) =20 -void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int= node) __alloc_size(1); -#define kvmalloc_node_noprof(size, flags, node) \ - __kvmalloc_node_noprof(PASS_BUCKET_PARAMS(size, NULL), flags, node) -#define kvmalloc_node(...) alloc_hooks(kvmalloc_node_noprof(__VA_ARGS__)) - -#define kvmalloc(_size, _flags) kvmalloc_node(_size, _flags, NUMA_NO_NOD= E) -#define kvmalloc_noprof(_size, _flags) kvmalloc_node_noprof(_size, _flags= , NUMA_NO_NODE) +void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long al= ign, + gfp_t flags, int node) __alloc_size(1); +#define kvmalloc_node_align_noprof(_size, _align, _flags, _node) \ + __kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, NULL), _align, _flags, _= node) +#define kvmalloc_node_align(...) \ + alloc_hooks(kvmalloc_node_align_noprof(__VA_ARGS__)) +#define kvmalloc_node(_s, _f, _n) kvmalloc_node_align(_s, 1, _f, _n) +#define kvmalloc(...) kvmalloc_node(__VA_ARGS__, NUMA_NO_NODE) #define kvzalloc(_size, _flags) kvmalloc(_size, (_flags)|__GFP_ZERO) =20 #define kvzalloc_node(_size, _flags, _node) kvmalloc_node(_size, (_flags)|= __GFP_ZERO, _node) + #define kmem_buckets_valloc(_b, _size, _flags) \ - alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), _flags,= NUMA_NO_NODE)) + alloc_hooks(__kvmalloc_node_noprof(PASS_BUCKET_PARAMS(_size, _b), 1, _fla= gs, NUMA_NO_NODE)) =20 static inline __alloc_size(1, 2) void * kvmalloc_array_node_noprof(size_t n, size_t size, gfp_t flags, int node) @@ -1062,7 +1068,7 @@ kvmalloc_array_node_noprof(size_t n, size_t size, gfp= _t flags, int node) if (unlikely(check_mul_overflow(n, size, &bytes))) return NULL; =20 - return kvmalloc_node_noprof(bytes, flags, node); + return kvmalloc_node_align_noprof(bytes, 1, flags, node); } =20 #define kvmalloc_array_noprof(...) kvmalloc_array_node_noprof(__VA_ARGS__= , NUMA_NO_NODE) @@ -1073,9 +1079,12 @@ kvmalloc_array_node_noprof(size_t n, size_t size, gf= p_t flags, int node) #define kvcalloc_node(...) alloc_hooks(kvcalloc_node_noprof(__VA_ARGS__)) #define kvcalloc(...) alloc_hooks(kvcalloc_noprof(__VA_ARGS__)) =20 -void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags) - __realloc_size(2); -#define kvrealloc(...) alloc_hooks(kvrealloc_noprof(__VA_ARGS__)) +void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned lon= g align, + gfp_t flags, int nid) __realloc_size(2); +#define kvrealloc_node_align(...) \ + alloc_hooks(kvrealloc_node_align_noprof(__VA_ARGS__)) +#define kvrealloc_node(_p, _s, _f, _n) kvrealloc_node_align(_p, _s, 1, _f= , _n) +#define kvrealloc(...) kvrealloc_node(__VA_ARGS__, NUMA_NO_NODE) =20 extern void kvfree(const void *addr); DEFINE_FREE(kvfree, void *, if (!IS_ERR_OR_NULL(_T)) kvfree(_T)) diff --git a/lib/rhashtable.c b/lib/rhashtable.c index 3e555d012ed6..fde0f0e556f8 100644 --- a/lib/rhashtable.c +++ b/lib/rhashtable.c @@ -184,8 +184,8 @@ static struct bucket_table *bucket_table_alloc(struct r= hashtable *ht, static struct lock_class_key __key; =20 tbl =3D alloc_hooks_tag(ht->alloc_tag, - kvmalloc_node_noprof(struct_size(tbl, buckets, nbuckets), - gfp|__GFP_ZERO, NUMA_NO_NODE)); + kvmalloc_node_align_noprof(struct_size(tbl, buckets, nbuckets), + 1, gfp|__GFP_ZERO, NUMA_NO_NODE)); =20 size =3D nbuckets; =20 diff --git a/mm/slub.c b/mm/slub.c index 31e11ef256f9..80f7b609a18d 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -4844,7 +4844,7 @@ void kfree(const void *object) EXPORT_SYMBOL(kfree); =20 static __always_inline __realloc_size(2) void * -__do_krealloc(const void *p, size_t new_size, gfp_t flags) +__do_krealloc(const void *p, size_t new_size, unsigned long align, gfp_t f= lags, int nid) { void *ret; size_t ks =3D 0; @@ -4858,6 +4858,16 @@ __do_krealloc(const void *p, size_t new_size, gfp_t = flags) if (!kasan_check_byte(p)) return NULL; =20 + /* + * If reallocation is not necessary (e. g. the new size is less + * than the current allocated size), the current allocation will be + * preserved unless __GFP_THISNODE is set. In the latter case a new + * allocation on the requested node will be attempted. + */ + if (unlikely(flags & __GFP_THISNODE) && nid !=3D NUMA_NO_NODE && + nid !=3D page_to_nid(virt_to_page(p))) + goto alloc_new; + if (is_kfence_address(p)) { ks =3D orig_size =3D kfence_ksize(p); } else { @@ -4880,6 +4890,10 @@ __do_krealloc(const void *p, size_t new_size, gfp_t = flags) if (new_size > ks) goto alloc_new; =20 + /* If the old object doesn't satisfy the new alignment, allocate a new on= e */ + if (!IS_ALIGNED((unsigned long)p, align)) + goto alloc_new; + /* Zero out spare memory. */ if (want_init_on_alloc(flags)) { kasan_disable_current(); @@ -4902,7 +4916,7 @@ __do_krealloc(const void *p, size_t new_size, gfp_t f= lags) return (void *)p; =20 alloc_new: - ret =3D kmalloc_node_track_caller_noprof(new_size, flags, NUMA_NO_NODE, _= RET_IP_); + ret =3D kmalloc_node_track_caller_noprof(new_size, flags, nid, _RET_IP_); if (ret && p) { /* Disable KASAN checks as the object's redzone is accessed. */ kasan_disable_current(); @@ -4914,10 +4928,12 @@ __do_krealloc(const void *p, size_t new_size, gfp_t= flags) } =20 /** - * krealloc - reallocate memory. The contents will remain unchanged. + * krealloc_node_align - reallocate memory. The contents will remain uncha= nged. * @p: object to reallocate memory for. * @new_size: how many bytes of memory are required. + * @align: desired alignment. * @flags: the type of memory to allocate. + * @nid: NUMA node or NUMA_NO_NODE * * If @p is %NULL, krealloc() behaves exactly like kmalloc(). If @new_size * is 0 and @p is not a %NULL pointer, the object pointed to is freed. @@ -4946,7 +4962,8 @@ __do_krealloc(const void *p, size_t new_size, gfp_t f= lags) * * Return: pointer to the allocated memory or %NULL in case of error */ -void *krealloc_noprof(const void *p, size_t new_size, gfp_t flags) +void *krealloc_node_align_noprof(const void *p, size_t new_size, unsigned = long align, + gfp_t flags, int nid) { void *ret; =20 @@ -4955,13 +4972,13 @@ void *krealloc_noprof(const void *p, size_t new_siz= e, gfp_t flags) return ZERO_SIZE_PTR; } =20 - ret =3D __do_krealloc(p, new_size, flags); + ret =3D __do_krealloc(p, new_size, align, flags, nid); if (ret && kasan_reset_tag(p) !=3D kasan_reset_tag(ret)) kfree(p); =20 return ret; } -EXPORT_SYMBOL(krealloc_noprof); +EXPORT_SYMBOL(krealloc_node_align_noprof); =20 static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t size) { @@ -4992,6 +5009,7 @@ static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t s= ize) * failure, fall back to non-contiguous (vmalloc) allocation. * @size: size of the request. * @b: which set of kmalloc buckets to allocate from. + * @align: desired alignment. * @flags: gfp mask for the allocation - must be compatible (superset) wit= h GFP_KERNEL. * @node: numa node to allocate from * @@ -5004,19 +5022,22 @@ static gfp_t kmalloc_gfp_adjust(gfp_t flags, size_t= size) * * Return: pointer to the allocated memory of %NULL in case of failure */ -void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), gfp_t flags, int= node) +void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size, b), unsigned long al= ign, + gfp_t flags, int node) { void *ret; =20 /* * It doesn't really make sense to fallback to vmalloc for sub page - * requests + * requests and small alignments */ - ret =3D __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), - kmalloc_gfp_adjust(flags, size), - node, _RET_IP_); - if (ret || size <=3D PAGE_SIZE) - return ret; + if (size >=3D align) { + ret =3D __do_kmalloc_node(size, PASS_BUCKET_PARAM(b), + kmalloc_gfp_adjust(flags, size), + node, _RET_IP_); + if (ret || size <=3D PAGE_SIZE) + return ret; + } =20 /* non-sleeping allocations are not supported by vmalloc */ if (!gfpflags_allow_blocking(flags)) @@ -5034,7 +5055,7 @@ void *__kvmalloc_node_noprof(DECL_BUCKET_PARAMS(size,= b), gfp_t flags, int node) * about the resulting pointer, and cannot play * protection games. */ - return __vmalloc_node_range_noprof(size, 1, VMALLOC_START, VMALLOC_END, + return __vmalloc_node_range_noprof(size, align, VMALLOC_START, VMALLOC_EN= D, flags, PAGE_KERNEL, VM_ALLOW_HUGE_VMAP, node, __builtin_return_address(0)); } @@ -5078,10 +5099,12 @@ void kvfree_sensitive(const void *addr, size_t len) EXPORT_SYMBOL(kvfree_sensitive); =20 /** - * kvrealloc - reallocate memory; contents remain unchanged + * kvrealloc_node_align - reallocate memory; contents remain unchanged * @p: object to reallocate memory for * @size: the size to reallocate + * @align: desired alignment * @flags: the flags for the page level allocator + * @nid: NUMA node id * * If @p is %NULL, kvrealloc() behaves exactly like kvmalloc(). If @size i= s 0 * and @p is not a %NULL pointer, the object pointed to is freed. @@ -5099,17 +5122,18 @@ EXPORT_SYMBOL(kvfree_sensitive); * * Return: pointer to the allocated memory or %NULL in case of error */ -void *kvrealloc_noprof(const void *p, size_t size, gfp_t flags) +void *kvrealloc_node_align_noprof(const void *p, size_t size, unsigned lon= g align, + gfp_t flags, int nid) { void *n; =20 if (is_vmalloc_addr(p)) - return vrealloc_noprof(p, size, flags); + return vrealloc_node_align_noprof(p, size, align, flags, nid); =20 - n =3D krealloc_noprof(p, size, kmalloc_gfp_adjust(flags, size)); + n =3D krealloc_node_align_noprof(p, size, align, kmalloc_gfp_adjust(flags= , size), nid); if (!n) { /* We failed to krealloc(), fall back to kvmalloc(). */ - n =3D kvmalloc_noprof(size, flags); + n =3D kvmalloc_node_align_noprof(size, align, flags, nid); if (!n) return NULL; =20 @@ -5125,7 +5149,7 @@ void *kvrealloc_noprof(const void *p, size_t size, gf= p_t flags) =20 return n; } -EXPORT_SYMBOL(kvrealloc_noprof); +EXPORT_SYMBOL(kvrealloc_node_align_noprof); =20 struct detached_freelist { struct slab *slab; --=20 2.39.2 From nobody Tue Oct 7 02:03:36 2025 Received: from mailrelay-egress16.pub.mailoutpod3-cph3.one.com (mailrelay-egress16.pub.mailoutpod3-cph3.one.com [46.30.212.3]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 31F982F2C41 for ; Tue, 15 Jul 2025 13:58:34 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=46.30.212.3 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752587916; cv=none; b=OWTJdpMII0OVoOpjVHSIq9Jn+Z6INrMnMMXY+CYJQ2cpGd3K0/qmx+79zyckM2todit97oE3elHDM7cDVjjzwU00Z6MCR5zx98cxPeEpvQGMtT/8HM80aguI5U8ktGXWa4KJtiC6G50BuUbAqzKcJ/1K+Bxu8h/EDD1lpdb9Jpg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752587916; c=relaxed/simple; bh=VU/RqlzRtvbr/3MFKAfpZmDzC6FYQvNG3CyRKi/ohZg=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=bPnEx1s3nrwBrDPIEPjbZbsuK32/A+PAeNpX92y8GomQ1fmjIjuf2kd0LqusJ64k/g4U6h640sWUjD8LoBQ/xrPCgV4hcviHHUQSEMJBHoLhszTj2isWwCFSKpbTS8CQ7X30kpb1D+sTANvt4pQI6QdP7rPj1gId3SfeWEjiDAo= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=konsulko.se; spf=none smtp.mailfrom=konsulko.se; dkim=pass (2048-bit key) header.d=konsulko.se header.i=@konsulko.se header.b=iCHMZdBi; dkim=permerror (0-bit key) header.d=konsulko.se header.i=@konsulko.se header.b=g9A/b0gP; arc=none smtp.client-ip=46.30.212.3 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=konsulko.se Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=konsulko.se Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=konsulko.se header.i=@konsulko.se header.b="iCHMZdBi"; dkim=permerror (0-bit key) header.d=konsulko.se header.i=@konsulko.se header.b="g9A/b0gP" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; t=1752587912; x=1753192712; d=konsulko.se; s=rsa1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=VXITfa4dJZ2HohHgkDoyGq2EdfXNfNIMJWrj+F386j8=; b=iCHMZdBiTTrC54/BWkvtwXLyCOVgv46NbDeprOslMashb/ctg9RcMhDp3Z1IDxhqJnGvjcu3uP9l7 bl2CqcSGsfBb/PWouq6eUeIp+QpMBEyBrcX6GZEcDPYRvwD4bsSLl7l9Ov993HVoSFfGa4BWSB1ODE La4fYv7sWApgGmDwu37HZ9qWYpiS9gwPDUs7jvilTpxMU6zBSwJ6o1j2Seztf1SDO1yOLStA87v2by exzyxPwH1KwxU3Hgp5WTiefJyr1r7rulUSUCCC9GoMBf8ZbYRfeYGBDW8qkMDJg/7/IPghKBXVVcZq pA7jvjj1bqhV0qGo4LC3ERBBeBthsow== DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed; t=1752587912; x=1753192712; d=konsulko.se; s=ed1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=VXITfa4dJZ2HohHgkDoyGq2EdfXNfNIMJWrj+F386j8=; b=g9A/b0gPAJbU++n5MUSabmzGcgKyd6D/07wW4OE9BKXx7l0YMIMsWH28PLwYyrnF57pZ3nqF22JEU 3aMEwKdDA== X-HalOne-ID: c9bc07ff-6183-11f0-8542-632fe8569f3f Received: from slottsdator.home (host-90-238-19-233.mobileonline.telia.com [90.238.19.233]) by mailrelay2.pub.mailoutpod3-cph3.one.com (Halon) with ESMTPSA id c9bc07ff-6183-11f0-8542-632fe8569f3f; Tue, 15 Jul 2025 13:58:31 +0000 (UTC) From: Vitaly Wool To: linux-mm@kvack.org Cc: akpm@linux-foundation.org, linux-kernel@vger.kernel.org, Uladzislau Rezki , Danilo Krummrich , Alice Ryhl , Vlastimil Babka , rust-for-linux@vger.kernel.org, Lorenzo Stoakes , "Liam R . Howlett" , Kent Overstreet , linux-bcachefs@vger.kernel.org, bpf@vger.kernel.org, Herbert Xu , Jann Horn , Pedro Falcato , Vitaly Wool Subject: [PATCH v13 3/4] rust: add support for NUMA ids in allocations Date: Tue, 15 Jul 2025 15:58:27 +0200 Message-Id: <20250715135827.2230267-1-vitaly.wool@konsulko.se> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20250715135645.2230065-1-vitaly.wool@konsulko.se> References: <20250715135645.2230065-1-vitaly.wool@konsulko.se> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Add a new type to support specifying NUMA identifiers in Rust allocators and extend the allocators to have NUMA id as a parameter. Thus, modify ReallocFunc to use the new extended realloc primitives from the C side of the kernel (i. e. k[v]realloc_node_align/vrealloc_node_align) and add the new function alloc_node to the Allocator trait while keeping the existing one (alloc) for backward compatibility. This will allow to specify node to use for allocation of e. g. {KV}Box, as well as for future NUMA aware users of the API. Signed-off-by: Vitaly Wool Acked-by: Danilo Krummrich Acked-by already. Acked-by: Alice Ryhl --- rust/helpers/slab.c | 8 ++--- rust/helpers/vmalloc.c | 4 +-- rust/kernel/alloc.rs | 54 ++++++++++++++++++++++++++++++---- rust/kernel/alloc/allocator.rs | 35 ++++++++++++++-------- rust/kernel/alloc/kbox.rs | 4 +-- rust/kernel/alloc/kvec.rs | 11 +++++-- 6 files changed, 88 insertions(+), 28 deletions(-) diff --git a/rust/helpers/slab.c b/rust/helpers/slab.c index a842bfbddcba..8472370a4338 100644 --- a/rust/helpers/slab.c +++ b/rust/helpers/slab.c @@ -3,13 +3,13 @@ #include =20 void * __must_check __realloc_size(2) -rust_helper_krealloc(const void *objp, size_t new_size, gfp_t flags) +rust_helper_krealloc_node(const void *objp, size_t new_size, gfp_t flags, = int node) { - return krealloc(objp, new_size, flags); + return krealloc_node(objp, new_size, flags, node); } =20 void * __must_check __realloc_size(2) -rust_helper_kvrealloc(const void *p, size_t size, gfp_t flags) +rust_helper_kvrealloc_node(const void *p, size_t size, gfp_t flags, int no= de) { - return kvrealloc(p, size, flags); + return kvrealloc_node(p, size, flags, node); } diff --git a/rust/helpers/vmalloc.c b/rust/helpers/vmalloc.c index 80d34501bbc0..62d30db9a1a6 100644 --- a/rust/helpers/vmalloc.c +++ b/rust/helpers/vmalloc.c @@ -3,7 +3,7 @@ #include =20 void * __must_check __realloc_size(2) -rust_helper_vrealloc(const void *p, size_t size, gfp_t flags) +rust_helper_vrealloc_node(const void *p, size_t size, gfp_t flags, int nod= e) { - return vrealloc(p, size, flags); + return vrealloc_node(p, size, flags, node); } diff --git a/rust/kernel/alloc.rs b/rust/kernel/alloc.rs index a2c49e5494d3..b39c279236f5 100644 --- a/rust/kernel/alloc.rs +++ b/rust/kernel/alloc.rs @@ -28,6 +28,8 @@ /// Indicates an allocation error. #[derive(Copy, Clone, PartialEq, Eq, Debug)] pub struct AllocError; + +use crate::error::{code::EINVAL, Result}; use core::{alloc::Layout, ptr::NonNull}; =20 /// Flags to be used when allocating memory. @@ -115,6 +117,31 @@ pub mod flags { pub const __GFP_NOWARN: Flags =3D Flags(bindings::__GFP_NOWARN); } =20 +/// Non Uniform Memory Access (NUMA) node identifier. +#[derive(Clone, Copy, PartialEq)] +pub struct NumaNode(i32); + +impl NumaNode { + /// Create a new NUMA node identifier (non-negative integer). + /// + /// Returns [`EINVAL`] if a negative id or an id exceeding [`bindings:= :MAX_NUMNODES`] is + /// specified. + pub fn new(node: i32) -> Result { + // MAX_NUMNODES never exceeds 2**10 because NODES_SHIFT is 0..10. + if node < 0 || node >=3D bindings::MAX_NUMNODES as i32 { + return Err(EINVAL); + } + Ok(Self(node)) + } +} + +/// Specify necessary constant to pass the information to Allocator that t= he caller doesn't care +/// about the NUMA node to allocate memory from. +impl NumaNode { + /// No node preference. + pub const NO_NODE: NumaNode =3D NumaNode(bindings::NUMA_NO_NODE); +} + /// The kernel's [`Allocator`] trait. /// /// An implementation of [`Allocator`] can allocate, re-allocate and free = memory buffers described @@ -137,7 +164,7 @@ pub mod flags { /// - Implementers must ensure that all trait functions abide by the guara= ntees documented in the /// `# Guarantees` sections. pub unsafe trait Allocator { - /// Allocate memory based on `layout` and `flags`. + /// Allocate memory based on `layout`, `flags` and `nid`. /// /// On success, returns a buffer represented as `NonNull<[u8]>` that s= atisfies the layout /// constraints (i.e. minimum size and alignment as specified by `layo= ut`). @@ -153,13 +180,21 @@ pub unsafe trait Allocator { /// /// Additionally, `Flags` are honored as documented in /// . - fn alloc(layout: Layout, flags: Flags) -> Result, AllocE= rror> { + fn alloc(layout: Layout, flags: Flags, nid: NumaNode) -> Result, AllocError> { // SAFETY: Passing `None` to `realloc` is valid by its safety requ= irements and asks for a // new memory allocation. - unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags) } + unsafe { Self::realloc(None, layout, Layout::new::<()>(), flags, n= id) } } =20 - /// Re-allocate an existing memory allocation to satisfy the requested= `layout`. + /// Re-allocate an existing memory allocation to satisfy the requested= `layout` and + /// a specific NUMA node request to allocate the memory for. + /// + /// Systems employing a Non Uniform Memory Access (NUMA) architecture = contain collections of + /// hardware resources including processors, memory, and I/O buses, th= at comprise what is + /// commonly known as a NUMA node. + /// + /// `nid` stands for NUMA id, i. e. NUMA node identifier, which is a n= on-negative integer + /// if a node needs to be specified, or [`NumaNode::NO_NODE`] if the c= aller doesn't care. /// /// If the requested size is zero, `realloc` behaves equivalent to `fr= ee`. /// @@ -196,6 +231,7 @@ unsafe fn realloc( layout: Layout, old_layout: Layout, flags: Flags, + nid: NumaNode, ) -> Result, AllocError>; =20 /// Free an existing memory allocation. @@ -211,7 +247,15 @@ unsafe fn free(ptr: NonNull, layout: Layout) { // SAFETY: The caller guarantees that `ptr` points at a valid allo= cation created by this // allocator. We are passing a `Layout` with the smallest possible= alignment, so it is // smaller than or equal to the alignment previously used with thi= s allocation. - let _ =3D unsafe { Self::realloc(Some(ptr), Layout::new::<()>(), l= ayout, Flags(0)) }; + let _ =3D unsafe { + Self::realloc( + Some(ptr), + Layout::new::<()>(), + layout, + Flags(0), + NumaNode::NO_NODE, + ) + }; } } =20 diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index aa2dfa9dca4c..8af7e04e3cc6 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -13,7 +13,7 @@ use core::ptr; use core::ptr::NonNull; =20 -use crate::alloc::{AllocError, Allocator}; +use crate::alloc::{AllocError, Allocator, NumaNode}; use crate::bindings; use crate::pr_warn; =20 @@ -56,20 +56,25 @@ fn aligned_size(new_layout: Layout) -> usize { =20 /// # Invariants /// -/// One of the following: `krealloc`, `vrealloc`, `kvrealloc`. +/// One of the following: `krealloc_node`, `vrealloc_node`, `kvrealloc_nod= e`. struct ReallocFunc( - unsafe extern "C" fn(*const crate::ffi::c_void, usize, u32) -> *mut cr= ate::ffi::c_void, + unsafe extern "C" fn( + *const crate::ffi::c_void, + usize, + u32, + crate::ffi::c_int, + ) -> *mut crate::ffi::c_void, ); =20 impl ReallocFunc { - // INVARIANT: `krealloc` satisfies the type invariants. - const KREALLOC: Self =3D Self(bindings::krealloc); + // INVARIANT: `krealloc_node` satisfies the type invariants. + const KREALLOC: Self =3D Self(bindings::krealloc_node); =20 - // INVARIANT: `vrealloc` satisfies the type invariants. - const VREALLOC: Self =3D Self(bindings::vrealloc); + // INVARIANT: `vrealloc_node` satisfies the type invariants. + const VREALLOC: Self =3D Self(bindings::vrealloc_node); =20 - // INVARIANT: `kvrealloc` satisfies the type invariants. - const KVREALLOC: Self =3D Self(bindings::kvrealloc); + // INVARIANT: `kvrealloc_node` satisfies the type invariants. + const KVREALLOC: Self =3D Self(bindings::kvrealloc_node); =20 /// # Safety /// @@ -87,6 +92,7 @@ unsafe fn call( layout: Layout, old_layout: Layout, flags: Flags, + nid: NumaNode, ) -> Result, AllocError> { let size =3D aligned_size(layout); let ptr =3D match ptr { @@ -110,7 +116,7 @@ unsafe fn call( // - Those functions provide the guarantees of this function. let raw_ptr =3D unsafe { // If `size =3D=3D 0` and `ptr !=3D NULL` the memory behind th= e pointer is freed. - self.0(ptr.cast(), size, flags.0).cast() + self.0(ptr.cast(), size, flags.0, nid.0).cast() }; =20 let ptr =3D if size =3D=3D 0 { @@ -134,9 +140,10 @@ unsafe fn realloc( layout: Layout, old_layout: Layout, flags: Flags, + nid: NumaNode, ) -> Result, AllocError> { // SAFETY: `ReallocFunc::call` has the same safety requirements as= `Allocator::realloc`. - unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags= ) } + unsafe { ReallocFunc::KREALLOC.call(ptr, layout, old_layout, flags= , nid) } } } =20 @@ -151,6 +158,7 @@ unsafe fn realloc( layout: Layout, old_layout: Layout, flags: Flags, + nid: NumaNode, ) -> Result, AllocError> { // TODO: Support alignments larger than PAGE_SIZE. if layout.align() > bindings::PAGE_SIZE { @@ -160,7 +168,7 @@ unsafe fn realloc( =20 // SAFETY: If not `None`, `ptr` is guaranteed to point to valid me= mory, which was previously // allocated with this `Allocator`. - unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags= ) } + unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags= , nid) } } } =20 @@ -175,6 +183,7 @@ unsafe fn realloc( layout: Layout, old_layout: Layout, flags: Flags, + nid: NumaNode, ) -> Result, AllocError> { // TODO: Support alignments larger than PAGE_SIZE. if layout.align() > bindings::PAGE_SIZE { @@ -184,6 +193,6 @@ unsafe fn realloc( =20 // SAFETY: If not `None`, `ptr` is guaranteed to point to valid me= mory, which was previously // allocated with this `Allocator`. - unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flag= s) } + unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flag= s, nid) } } } diff --git a/rust/kernel/alloc/kbox.rs b/rust/kernel/alloc/kbox.rs index c386ff771d50..5c0b020fb2a4 100644 --- a/rust/kernel/alloc/kbox.rs +++ b/rust/kernel/alloc/kbox.rs @@ -4,7 +4,7 @@ =20 #[allow(unused_imports)] // Used in doc comments. use super::allocator::{KVmalloc, Kmalloc, Vmalloc}; -use super::{AllocError, Allocator, Flags}; +use super::{AllocError, Allocator, Flags, NumaNode}; use core::alloc::Layout; use core::fmt; use core::marker::PhantomData; @@ -271,7 +271,7 @@ pub fn new(x: T, flags: Flags) -> Result { /// ``` pub fn new_uninit(flags: Flags) -> Result, A>, Allo= cError> { let layout =3D Layout::new::>(); - let ptr =3D A::alloc(layout, flags)?; + let ptr =3D A::alloc(layout, flags, NumaNode::NO_NODE)?; =20 // INVARIANT: `ptr` is either a dangling pointer or points to memo= ry allocated with `A`, // which is sufficient in size and alignment for storing a `T`. diff --git a/rust/kernel/alloc/kvec.rs b/rust/kernel/alloc/kvec.rs index 1a0dd852a468..aa5d27176d9c 100644 --- a/rust/kernel/alloc/kvec.rs +++ b/rust/kernel/alloc/kvec.rs @@ -5,7 +5,7 @@ use super::{ allocator::{KVmalloc, Kmalloc, Vmalloc}, layout::ArrayLayout, - AllocError, Allocator, Box, Flags, + AllocError, Allocator, Box, Flags, NumaNode, }; use core::{ fmt, @@ -633,6 +633,7 @@ pub fn reserve(&mut self, additional: usize, flags: Fla= gs) -> Result<(), AllocEr layout.into(), self.layout.into(), flags, + NumaNode::NO_NODE, )? }; =20 @@ -1058,7 +1059,13 @@ pub fn collect(self, flags: Flags) -> Vec { // the type invariant to be smaller than `cap`. Depending on `= realloc` this operation // may shrink the buffer or leave it as it is. ptr =3D match unsafe { - A::realloc(Some(buf.cast()), layout.into(), old_layout.int= o(), flags) + A::realloc( + Some(buf.cast()), + layout.into(), + old_layout.into(), + flags, + NumaNode::NO_NODE, + ) } { // If we fail to shrink, which likely can't even happen, c= ontinue with the existing // buffer. --=20 2.39.2 From nobody Tue Oct 7 02:03:36 2025 Received: from mailrelay-egress16.pub.mailoutpod3-cph3.one.com (mailrelay-egress16.pub.mailoutpod3-cph3.one.com [46.30.212.3]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C37EB2E2F0C for ; Tue, 15 Jul 2025 13:58:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=46.30.212.3 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752587932; cv=none; b=cyWSp3tG+LUnclhHY/jc8W4xJp+xLqirRkQkDoAI4lemE9uWUW1KaMb+kiH/mYqLMT0Xy6Pte4Fe+/BqMtKVtgFcmVQ3HzSxLc06G4AdrMYTBOpdI/xNgRSFKhxVkgt0up4Mspe56b+t5j0zUpqMoRuDLCfOIhxUSNgTk9Z4+tU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1752587932; c=relaxed/simple; bh=jfk2V22ERQUvp6fNK5tGHU3+ExFDd+662b5a1Os5EE0=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=C0JftM+UZbJm+QbnF2S16KegWFGlc1LfrJ35cFUq2AgigOSx+f7n5ebovWk9lJ9btlvnTG+BEWeN8Uq5SmSMXlMXCqQTx2N45A4UQiA9Zzf/t/DqiUOnMGefpWVQR4Bf9D6AXG4VwesArsWDKrc0mGWNzkyu43PnytP4l0Aklb0= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=konsulko.se; spf=none smtp.mailfrom=konsulko.se; dkim=pass (2048-bit key) header.d=konsulko.se header.i=@konsulko.se header.b=ovFk8pli; dkim=permerror (0-bit key) header.d=konsulko.se header.i=@konsulko.se header.b=a6O6XJEF; arc=none smtp.client-ip=46.30.212.3 Authentication-Results: smtp.subspace.kernel.org; dmarc=none (p=none dis=none) header.from=konsulko.se Authentication-Results: smtp.subspace.kernel.org; spf=none smtp.mailfrom=konsulko.se Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=konsulko.se header.i=@konsulko.se header.b="ovFk8pli"; dkim=permerror (0-bit key) header.d=konsulko.se header.i=@konsulko.se header.b="a6O6XJEF" DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; t=1752587929; x=1753192729; d=konsulko.se; s=rsa1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=W9+2URA1qXrl1fhBq8o38MQgoc5ORxkxAU8T1e724LI=; b=ovFk8plinhgq1Pm98F2EKee7NHCl2Ao8cMk6fGyDGd1YoQlEsrBMV6zs55mm+tHUMh35SLSCXMVLF 8gEOdmR33UWgFx1U33SOcQWDl5/J5WaEpjuFi6K2xZ433tQymQ425NCPakQtux0+rRt6BPd6AkBTp7 SdTH9OxYdStN1pJNvrlGMWFSSs5Gnu/jfT7fNIgT2UVjfU5uP8EnqlBrMgZX+fbD+qnahA7+YiAI8b T+9wBc8psR4e0ArRwGbvKVUhAcGcmAN/GzRYDYKyrevQFUP3KmL1gNk0qnHNDSK3wvqC5Ow4/2XAYF QlNnGTwIDwVuxLsEsTtF3WXVii8lNJg== DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed; t=1752587929; x=1753192729; d=konsulko.se; s=ed1; h=content-transfer-encoding:mime-version:references:in-reply-to:message-id:date: subject:cc:to:from:from; bh=W9+2URA1qXrl1fhBq8o38MQgoc5ORxkxAU8T1e724LI=; b=a6O6XJEFJ2Z8BV5TamKbPrPPKpdpDUaatdq2FnN8VN6hmU58hHo+ExcR8mJ7Tp9mcnZct0gQNoiFJ 6XI7xZMCg== X-HalOne-ID: d344b59a-6183-11f0-a951-85eb291bc831 Received: from slottsdator.home (host-90-238-19-233.mobileonline.telia.com [90.238.19.233]) by mailrelay5.pub.mailoutpod2-cph3.one.com (Halon) with ESMTPSA id d344b59a-6183-11f0-a951-85eb291bc831; Tue, 15 Jul 2025 13:58:48 +0000 (UTC) From: Vitaly Wool To: linux-mm@kvack.org Cc: akpm@linux-foundation.org, linux-kernel@vger.kernel.org, Uladzislau Rezki , Danilo Krummrich , Alice Ryhl , Vlastimil Babka , rust-for-linux@vger.kernel.org, Lorenzo Stoakes , "Liam R . Howlett" , Kent Overstreet , linux-bcachefs@vger.kernel.org, bpf@vger.kernel.org, Herbert Xu , Jann Horn , Pedro Falcato , Vitaly Wool Subject: [PATCH v13 4/4] rust: support large alignments in allocations Date: Tue, 15 Jul 2025 15:58:45 +0200 Message-Id: <20250715135845.2230333-1-vitaly.wool@konsulko.se> X-Mailer: git-send-email 2.39.2 In-Reply-To: <20250715135645.2230065-1-vitaly.wool@konsulko.se> References: <20250715135645.2230065-1-vitaly.wool@konsulko.se> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Add support for large (> PAGE_SIZE) alignments in Rust allocators. All the preparations on the C side are already done, we just need to add bindings for _node_align() functions and start using those. Signed-off-by: Vitaly Wool Acked-by: Danilo Krummrich Acked-by already. Acked-by: Alice Ryhl --- rust/helpers/slab.c | 10 ++++++---- rust/helpers/vmalloc.c | 5 +++-- rust/kernel/alloc/allocator.rs | 30 +++++++++--------------------- 3 files changed, 18 insertions(+), 27 deletions(-) diff --git a/rust/helpers/slab.c b/rust/helpers/slab.c index 8472370a4338..7fac958907b0 100644 --- a/rust/helpers/slab.c +++ b/rust/helpers/slab.c @@ -3,13 +3,15 @@ #include =20 void * __must_check __realloc_size(2) -rust_helper_krealloc_node(const void *objp, size_t new_size, gfp_t flags, = int node) +rust_helper_krealloc_node_align(const void *objp, size_t new_size, unsigne= d long align, + gfp_t flags, int node) { - return krealloc_node(objp, new_size, flags, node); + return krealloc_node_align(objp, new_size, align, flags, node); } =20 void * __must_check __realloc_size(2) -rust_helper_kvrealloc_node(const void *p, size_t size, gfp_t flags, int no= de) +rust_helper_kvrealloc_node_align(const void *p, size_t size, unsigned long= align, + gfp_t flags, int node) { - return kvrealloc_node(p, size, flags, node); + return kvrealloc_node_align(p, size, align, flags, node); } diff --git a/rust/helpers/vmalloc.c b/rust/helpers/vmalloc.c index 62d30db9a1a6..7d7f7336b3d2 100644 --- a/rust/helpers/vmalloc.c +++ b/rust/helpers/vmalloc.c @@ -3,7 +3,8 @@ #include =20 void * __must_check __realloc_size(2) -rust_helper_vrealloc_node(const void *p, size_t size, gfp_t flags, int nod= e) +rust_helper_vrealloc_node_align(const void *p, size_t size, unsigned long = align, + gfp_t flags, int node) { - return vrealloc_node(p, size, flags, node); + return vrealloc_node_align(p, size, align, flags, node); } diff --git a/rust/kernel/alloc/allocator.rs b/rust/kernel/alloc/allocator.rs index 8af7e04e3cc6..63f271624428 100644 --- a/rust/kernel/alloc/allocator.rs +++ b/rust/kernel/alloc/allocator.rs @@ -15,7 +15,6 @@ =20 use crate::alloc::{AllocError, Allocator, NumaNode}; use crate::bindings; -use crate::pr_warn; =20 /// The contiguous kernel allocator. /// @@ -56,25 +55,26 @@ fn aligned_size(new_layout: Layout) -> usize { =20 /// # Invariants /// -/// One of the following: `krealloc_node`, `vrealloc_node`, `kvrealloc_nod= e`. +/// One of the following: `krealloc_node_align`, `vrealloc_node_align`, `k= vrealloc_node_align`. struct ReallocFunc( unsafe extern "C" fn( *const crate::ffi::c_void, usize, + crate::ffi::c_ulong, u32, crate::ffi::c_int, ) -> *mut crate::ffi::c_void, ); =20 impl ReallocFunc { - // INVARIANT: `krealloc_node` satisfies the type invariants. - const KREALLOC: Self =3D Self(bindings::krealloc_node); + // INVARIANT: `krealloc_node_align` satisfies the type invariants. + const KREALLOC: Self =3D Self(bindings::krealloc_node_align); =20 - // INVARIANT: `vrealloc_node` satisfies the type invariants. - const VREALLOC: Self =3D Self(bindings::vrealloc_node); + // INVARIANT: `vrealloc_node_align` satisfies the type invariants. + const VREALLOC: Self =3D Self(bindings::vrealloc_node_align); =20 - // INVARIANT: `kvrealloc_node` satisfies the type invariants. - const KVREALLOC: Self =3D Self(bindings::kvrealloc_node); + // INVARIANT: `kvrealloc_node_align` satisfies the type invariants. + const KVREALLOC: Self =3D Self(bindings::kvrealloc_node_align); =20 /// # Safety /// @@ -116,7 +116,7 @@ unsafe fn call( // - Those functions provide the guarantees of this function. let raw_ptr =3D unsafe { // If `size =3D=3D 0` and `ptr !=3D NULL` the memory behind th= e pointer is freed. - self.0(ptr.cast(), size, flags.0, nid.0).cast() + self.0(ptr.cast(), size, layout.align(), flags.0, nid.0).cast() }; =20 let ptr =3D if size =3D=3D 0 { @@ -160,12 +160,6 @@ unsafe fn realloc( flags: Flags, nid: NumaNode, ) -> Result, AllocError> { - // TODO: Support alignments larger than PAGE_SIZE. - if layout.align() > bindings::PAGE_SIZE { - pr_warn!("Vmalloc does not support alignments larger than PAGE= _SIZE yet.\n"); - return Err(AllocError); - } - // SAFETY: If not `None`, `ptr` is guaranteed to point to valid me= mory, which was previously // allocated with this `Allocator`. unsafe { ReallocFunc::VREALLOC.call(ptr, layout, old_layout, flags= , nid) } @@ -185,12 +179,6 @@ unsafe fn realloc( flags: Flags, nid: NumaNode, ) -> Result, AllocError> { - // TODO: Support alignments larger than PAGE_SIZE. - if layout.align() > bindings::PAGE_SIZE { - pr_warn!("KVmalloc does not support alignments larger than PAG= E_SIZE yet.\n"); - return Err(AllocError); - } - // SAFETY: If not `None`, `ptr` is guaranteed to point to valid me= mory, which was previously // allocated with this `Allocator`. unsafe { ReallocFunc::KVREALLOC.call(ptr, layout, old_layout, flag= s, nid) } --=20 2.39.2