From nobody Sat Sep 26 19:35:32 2026 Received: from va-1-114.ptr.blmpb.com (va-1-114.ptr.blmpb.com [209.127.230.114]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7B97C33F5A8 for ; Mon, 31 Aug 2026 11:17:55 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.230.114 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788175078; cv=none; b=a1SotEKePsls3w/+U+P+2r7UJkn4KCAiiBzVKjh0HUzUTMgTBjk5osZsmV20aT8DH+45cLLOlqHpAcawk+XB3jZXUpw9Dm8fQLm/SBey6GZ4PJo61q3G9a/4iMgEspbs1P6vn9hp0IEBTKH6TO/nszKExeASYja4jCIaPN05X2M= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788175078; c=relaxed/simple; bh=1tzC160XRVFTBUvqo0eRGovwFSfOVpQWnrqojoT/RKQ=; h=Date:References:To:Subject:Message-Id:Cc:From:Mime-Version: Content-Type:In-Reply-To; b=LHAUOjGUByLQF7obpueISiYMGnPs66st2tvjKs2tSQYSdS+VR/jAqG6PlU4BHXjLEbAr0cn7PEFHwxMiA5K5kgjnyZLXveNGNv1Z85UJksOR7BCJh07og6/wpzuKicbF5Tr8A9wIsFdpq+S0FSHcCg//NsZ+i+EUlg25oKq2TAI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=cahwJPC8; arc=none smtp.client-ip=209.127.230.114 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="cahwJPC8" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1788175071; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=sk7m5/tft/ydmLxoPchPzp52VQWS3uxvgVEj6+c0Gmk=; b=cahwJPC8yhwEKN/G30w4+TGWKh1rlYTtEwLnyaIiBYtyoJd2zPvhs08amO+XVusVPW+TSO DBVSSJOFyC7uGOeFje2QGOZjUHYc2D7jZcOmJDbrLyi81CcIU+/vrdTXQ1gf8wQrmhnciB nr3km+POkgcMbOog3Qj/CWZFXuIy3n/UJkELyl+tBnRpNBaVAt5ke1rAVDDLD+MCsrMz76 soRy0PoE1c3dO+Zk3y4IgCSsNlBWwmSzsaccRzdIcblt4geR+1KfhI/udj8mXUsGn8Yvyg cJGoiqaI2MtrJA5spGrfakitPfofjDaqxdvoSMox3ZVFwUoJepOXQTWNOISZkA== Date: Mon, 31 Aug 2026 19:16:32 +0800 X-Mailer: git-send-email 2.45.2 References: <20260831111638.76012-1-lizhe.67@bytedance.com> To: , , , , , , , , , , , Subject: [PATCH v11 1/7] mm: fix stale ZONE_DEVICE refcount comment Message-Id: <20260831111638.76012-2-lizhe.67@bytedance.com> Content-Transfer-Encoding: quoted-printable Cc: , , , , , From: "Li Zhe" Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 X-Original-From: Li Zhe X-Lms-Return-Path: In-Reply-To: <20260831111638.76012-1-lizhe.67@bytedance.com> Content-Type: text/plain; charset="utf-8" The comment in __init_zone_device_page() still uses the old MEMORY_TYPE_* names and implies that FS_DAX pages regain a refcount of 1 in the free path. That no longer matches the code. Update the comment to describe the current policy correctly: MEMORY_DEVICE_GENERIC pages regain a refcount of 1 in the free path, while the remaining ZONE_DEVICE types start from 0 here and raise the count again when the allocator or driver hands the page out. No functional change intended. Signed-off-by: Li Zhe Reviewed-by: David Hildenbrand (Arm) Reviewed-by: Alistair Popple Reviewed-by: Muchun Song Reviewed-by: Mike Rapoport (Microsoft) --- mm/mm_init.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 1533aebafb68..8f73d5850db2 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1000,13 +1000,9 @@ static void __ref __init_zone_device_page(struct pag= e *page, unsigned long pfn, page->zone_device_data =3D NULL; =20 /* - * ZONE_DEVICE pages other than MEMORY_TYPE_GENERIC are released - * directly to the driver page allocator which will set the page count - * to 1 when allocating the page. - * - * MEMORY_TYPE_GENERIC and MEMORY_TYPE_FS_DAX pages automatically have - * their refcount reset to one whenever they are freed (ie. after - * their refcount drops to 0). + * MEMORY_DEVICE_GENERIC pages regain a refcount of 1 in the free + * path. The remaining ZONE_DEVICE types start from 0 here and raise + * the count again when the allocator or driver hands the page out. */ switch (pgmap->type) { case MEMORY_DEVICE_FS_DAX: --=20 2.20.1 From nobody Sat Sep 26 19:35:32 2026 Received: from va-1-112.ptr.blmpb.com (va-1-112.ptr.blmpb.com [209.127.230.112]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D3B1C3438B9 for ; Mon, 31 Aug 2026 11:18:26 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.230.112 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788175108; cv=none; b=dM58P8LuHYdqCQTTEtwFJ0qBfHv2SkEMZn3yomBRng4eMMK8/oCFi/BIVyh6l+/NLrPVSa40Z9nD54ZPQLPIFtAiR/0MWdfP5obc1LiQ/A0modPqJvUIgdz54rY59eHLQ+PQYeqZQvwv6i+qAOz6M5oNecmudKOulDi2UJIBQns= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788175108; c=relaxed/simple; bh=TXr+PizXeUinczIJIbNEYY5V0bJAfItxsfEKmy4dDxk=; h=Cc:Mime-Version:To:From:Subject:Message-Id:Date:In-Reply-To: References:Content-Type; b=KenkPUvXu4GMXDN+OCzQJPKzJ+CiwoMBPncqtYiV4rEL7F1HliUP4cqGGVqmTIGM2SXDjK+qM1miXAl1CgdKx4OEDQRZlyd+XVdlPzFvDGU+T9tKYUF+yG8BWfywUIV4fJipdXNKvP5h4m9UBw37EulKBd7W0GHdKtI6j7jAsZY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=e6bQxNRc; arc=none smtp.client-ip=209.127.230.112 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="e6bQxNRc" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1788175102; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=3cbkuNFC4N1fAhS6eNTbPmQJtMCLdt2nXyAfh98ZQIw=; b=e6bQxNRcHzCKsZZJZ+/iFaBsoc07y6qbZ1fxiy0HYE+oyhe2Q6ljMgr4mSeX+/YG1k7O5G 9hx+K2/O8omQJcavD9JIu5pYz7aw5PBDUlO4tHIW0AUbxOtzRlUlvOC+ZduCXnNYkUvqdW nEIVJ7BasMyQfyIkY/lHIXscK9SQSpblfXlL2WSJWaPLg2QOdPrKVvIQgBq6m9+WSIck4o bUZqaMvg9ulHpeP88X/K36TKY1C4Y4ZIO8UaBhaitMRH2PwtwW6/Frph7OTUNoUFBHY3Qo xfnI8gjg36j3IrPDlpjc4W8p44dzoJ9hKt2ioDVS6Oiuz9EPGm9/SqVjw+sRaA== Cc: , , , , , X-Mailer: git-send-email 2.45.2 Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 X-Lms-Return-Path: To: , , , , , , , , , , , From: "Li Zhe" Subject: [PATCH v11 2/7] mm: add a set_page_section_from_pfn() helper Message-Id: <20260831111638.76012-3-lizhe.67@bytedance.com> Content-Transfer-Encoding: quoted-printable X-Original-From: Li Zhe Date: Mon, 31 Aug 2026 19:16:33 +0800 In-Reply-To: <20260831111638.76012-1-lizhe.67@bytedance.com> References: <20260831111638.76012-1-lizhe.67@bytedance.com> Content-Type: text/plain; charset="utf-8" Callers that want to update section bits from a PFN currently need to open-code: set_page_section(page, pfn_to_section_nr(pfn)); and guard that sequence with #ifdef SECTION_IN_PAGE_FLAGS. Add set_page_section_from_pfn() to wrap that update in one place. When section bits are stored in page flags, the helper derives the section number from the PFN and updates the page flags. Otherwise keep it as a no-op so callers can use one helper without open-coding SECTION_IN_PAGE_FLAGS. Convert set_page_links() to use the new helper so later ZONE_DEVICE fast-path patches can also update section bits without open-coding SECTION_IN_PAGE_FLAGS at each callsite. This keeps the PFN-to-section translation local to the configurations that actually store section bits in struct page flags, and avoids exposing that detail to generic callers. No functional change intended. Signed-off-by: Li Zhe Reviewed-by: Mike Rapoport (Microsoft) Acked-by: Muchun Song Reviewed-by: Balbir Singh --- include/linux/mm.h | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index dd09c438fa23..6640c7522822 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2644,12 +2644,23 @@ static inline void set_page_section(struct page *pa= ge, unsigned long section) page->flags.f |=3D (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; } =20 +static inline void set_page_section_from_pfn(struct page *page, + unsigned long pfn) +{ + set_page_section(page, pfn_to_section_nr(pfn)); +} + static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { ASSERT_EXCLUSIVE_BITS(mdf->f, SECTIONS_MASK << SECTIONS_PGSHIFT); return (mdf->f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; } #else /* !SECTION_IN_PAGE_FLAGS */ +static inline void set_page_section_from_pfn(struct page *page, + unsigned long pfn) +{ +} + static inline unsigned long memdesc_section(const memdesc_flags_t *mdf) { return 0; @@ -2872,9 +2883,7 @@ static inline void set_page_links(struct page *page, = enum zone_type zone, { set_page_zone(page, zone); set_page_node(page, node); -#ifdef SECTION_IN_PAGE_FLAGS - set_page_section(page, pfn_to_section_nr(pfn)); -#endif + set_page_section_from_pfn(page, pfn); } =20 /** --=20 2.20.1 From nobody Sat Sep 26 19:35:32 2026 Received: from va-1-111.ptr.blmpb.com (va-1-111.ptr.blmpb.com [209.127.230.111]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 829AA3438B9 for ; Mon, 31 Aug 2026 11:18:50 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.230.111 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788175133; cv=none; b=bsGK2pRagNet/nnULjgsfH7lPFoG04VFqbR3Yzvo9R5sKR2B3FTM60gIIEMWIAYBRR4OmrYy8FXVoD7Y5H+bftc8E+enDkB4jH40Bc9Il4zOuCBH9qNhOw+2Cq1kCIoVM6UObWiEtSK1/Me4RnFPCBmtcFUmV/KQV69rDPFmR10= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788175133; c=relaxed/simple; bh=ynzHxWDVs/gs14/XwDc7LkFCfBtedIs/oV9bwerefj4=; h=In-Reply-To:Content-Type:Cc:Subject:Date:References:To:From: Message-Id:Mime-Version; b=tC7uD3ECoheKqQLYA6ut7HMRfOmsI0yLj7VHloDJQJO7gYadV9kq5wGecp4jrkNrFIDo6LEkXEmH345gh9NLHfvrN/U9kArd2ciKkLxMHXJB7eMluFG4tIESipPg6N2BViJEEL5JQCmPI6g7++Q/JE9DZnULQzpI4VdZ9JNfRfg= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=prEkJkNR; arc=none smtp.client-ip=209.127.230.111 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="prEkJkNR" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1788175124; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=PEniiU9yBVKIk2zvd2yZlOYClo4KAi6kKS5dsV5iO+A=; b=prEkJkNRc94GeiZSlf4W8q32JAwECdSGtcb3NjCxMQDR7pGPv1lEuj0ATTrDle3tKhFBPw C+6lbt5K/u/HB5jSuFr6l2aRsOJO7SicZFWkUk3xjNb5pi25FCqmjElXQL4vh4/Ptmd+GT SuNtGbxJf551ekNGlNCHQQklJuCjFdx9g0wiwQrxKagMXYnUIN04HwiCfRNuI8PKGldaTk ZzaJS8hWjAziEXTUbStoJopTab/LoFtlenjUGBF5WwaGStqLyVPlKfKb3N9I8TaFt3FZxl q6WceI9YhGzwhn3r9BHtSu5kJA9k7WazVm5yO0GkmBaxsuS+AXUGVEHKO6c4Nw== In-Reply-To: <20260831111638.76012-1-lizhe.67@bytedance.com> X-Original-From: Li Zhe Cc: , , , , , Subject: [PATCH v11 3/7] mm: add a template-based fast path for zone-device page init Date: Mon, 31 Aug 2026 19:16:34 +0800 X-Mailer: git-send-email 2.45.2 References: <20260831111638.76012-1-lizhe.67@bytedance.com> To: , , , , , , , , , , , From: "Li Zhe" Message-Id: <20260831111638.76012-4-lizhe.67@bytedance.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 X-Lms-Return-Path: Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" memmap_init_zone_device() repeats nearly identical head-page initialization for each PFN. Initialize the first real ZONE_DEVICE head page through the existing path, copy that final state into a reusable template, refresh the PFN-dependent fields in that template before each copy, and copy it into the remaining destination pages. Use the template path unconditionally. The page_ref_set tracepoint is primarily a debugging aid, while this code is still initializing struct pages before they are handed out. From the perspective of users of those pages, the initialization-time refcount transitions are not part of the observable page lifetime. This means page_ref_set will no longer observe every initialization-time refcount assignment for copied ZONE_DEVICE head pages. The impact is controlled because the final initialized struct page state is unchanged, and keeping a separate non-template path only for this local tracepoint observability would add complexity to the common path. This patch accelerates head-page initialization. The pfns_per_compound =3D=3D 1 case gets the full benefit here, compound tails are handled in the next patch. Tested in a VM with a 100 GB fsdax namespace device configured with map=3Ddev on Intel Ice Lake server. This test exercises the nd_pmem rebind path (pfns_per_compound =3D=3D 1). Test procedure: Rebind the nd_pmem driver 30 times and collect the memmap initialization time from the pr_debug() output of memmap_init_zone_device(). Base(v7.3-rc1): Average of rebinds for nd_pmem driver: 221.07 ms With this patch and its prerequisites applied: Average of rebinds for nd_pmem driver: 155.00 ms This reduces the average memmap initialization time measured during rebind from 221.07 ms to 155.00 ms, or about 29.9%. Signed-off-by: Li Zhe Reviewed-by: Mike Rapoport (Microsoft) --- mm/mm_init.c | 38 +++++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 8f73d5850db2..ba4148de2b28 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1017,6 +1017,17 @@ static void __ref __init_zone_device_page(struct pag= e *page, unsigned long pfn, } } =20 +static void zone_device_page_init_from_template(struct page *page, + unsigned long pfn, struct page *template) +{ + set_page_section_from_pfn(template, pfn); +#ifdef WANT_PAGE_VIRTUAL + if (!is_highmem_idx(ZONE_DEVICE)) + set_page_address(template, __va(pfn << PAGE_SHIFT)); +#endif + memcpy(page, template, sizeof(*page)); +} + /* * With compound page geometry and when struct pages are stored in ram most * tail pages are reused. Consequently, the amount of unique struct pages = to @@ -1079,6 +1090,8 @@ void __ref memmap_init_zone_device(struct zone *zone, unsigned long zone_idx =3D zone_idx(zone); unsigned long start =3D jiffies; int nid =3D pgdat->node_id; + struct page template; + struct page *page; =20 if (WARN_ON_ONCE(!pgmap || zone_idx !=3D ZONE_DEVICE)) return; @@ -1093,10 +1106,29 @@ void __ref memmap_init_zone_device(struct zone *zon= e, nr_pages =3D end_pfn - start_pfn; } =20 - for (pfn =3D start_pfn; pfn < end_pfn; pfn +=3D pfns_per_compound) { - struct page *page =3D pfn_to_page(pfn); + if (!nr_pages) + return; =20 - __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + /* + * Seed the reusable head-page template from the first real struct + * page. The normal page-init and refcount helpers must operate on + * a real memmap entry rather than a stack object. + */ + pfn =3D start_pfn; + page =3D pfn_to_page(pfn); + __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + memcpy(&template, page, sizeof(*page)); + if (pfns_per_compound !=3D 1) + memmap_init_compound(page, pfn, zone_idx, nid, pgmap, + compound_nr_pages(pfn, altmap, pgmap)); + pfn +=3D pfns_per_compound; + + /* Initialize the remaining head pages from template. */ + for (; pfn < end_pfn; pfn +=3D pfns_per_compound) { + page =3D pfn_to_page(pfn); + + zone_device_page_init_from_template(page, pfn, + &template); =20 if (IS_ALIGNED(pfn, PAGES_PER_SECTION)) cond_resched(); --=20 2.20.1 From nobody Sat Sep 26 19:35:32 2026 Received: from va-1-113.ptr.blmpb.com (va-1-113.ptr.blmpb.com [209.127.230.113]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 911403E009C for ; Mon, 31 Aug 2026 11:19:16 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.230.113 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788175160; cv=none; b=M2yH6MGcc07M738VMzAV3B89OVwAeGiyeQIBEcCPE8QuDmPjEYu6U93/ypMjz9zb3y89rWKupMXM/fQsQ7xssR5QDIePylfQtLUEAT/cWJTz4pSibraPSsGVaYOzZU+tumlGOaVC18gFYG/HFtE4RQE2EooskZZvymeKfrPZwyg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788175160; c=relaxed/simple; bh=/WJdK3uvdOGxF0fYzqjQfsNLnZOExeJLdWudhu02EJU=; h=From:References:To:Subject:In-Reply-To:Date:Message-Id: Content-Type:Cc:Mime-Version; b=OBAFH0ZjCJHPNIQo+1nm/HpMhsHTrGxig71uAN353hThBabTurt54cUD+3ishb9LBdSKRLdEe2vPiERif4sjB0ZSn7laOKfYkTBqk5shCgM1/KM4Hdbr+xgdSfWh5tHdM75lh4tTb0HR3O7QJmN3Ls7hQqq/ivBNHYOGOw4wQX8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=PkYX+kv1; arc=none smtp.client-ip=209.127.230.113 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="PkYX+kv1" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1788175151; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=ob9H4dgSGLsuZnXTYmFdoB7QrJmgRd8FFQ463R9dn04=; b=PkYX+kv1AJTKg57OJLF3Zslaz+IEv+uBxIE8T+c56ZXsN7th1qgAmfcbDN0lrLL0BIV+Oj 07jRz3wRsxlC1STsAvP7WiDg1UscV66kMGTCK19JzLAUzQ735qybqs93PqwEdTKONTd1B5 kYZkuSZApJu86B7JCs5/zUDA+L0hGS6hwqA11S2fGWna/vejPOouxCXobIK7yAneZ07rfS k427sjX5ESuOjcxIxZCBsfJ3Uxz56b+yz2eIiriXfo+sMaRR0UKntxJN+v33vCCDGDSYpc XN6sbqQcvU7jhqoO3OyTKk9Lgb99P6o3uIiQ2N990bqvBn8nJe1noK2sK6HPdw== From: "Li Zhe" References: <20260831111638.76012-1-lizhe.67@bytedance.com> X-Original-From: Li Zhe To: , , , , , , , , , , , Subject: [PATCH v11 4/7] mm: extend the template fast path to zone-device compound tails In-Reply-To: <20260831111638.76012-1-lizhe.67@bytedance.com> Date: Mon, 31 Aug 2026 19:16:35 +0800 Message-Id: <20260831111638.76012-5-lizhe.67@bytedance.com> Content-Transfer-Encoding: quoted-printable X-Mailer: git-send-email 2.45.2 X-Lms-Return-Path: Cc: , , , , , Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 Content-Type: text/plain; charset="utf-8" The template fast path from the previous patch only accelerates head pages. Compound tails in memmap_init_compound() still go through the normal initialization path one by one. Build separate head and tail templates and reuse one prepared tail template across the tail pages in a compound range. Head pages preserve the existing refcount policy, while compound tails always start with a refcount of 0 after prep_compound_tail(). This extends the template-copy fast path to pfns_per_compound > 1. Tail-page PFN-dependent fields are refreshed in the reusable tail template before each copy. Do not keep a separate non-template fallback for compound tails either. These pages are still under memmap initialization, and the initialization-time refcount updates are not part of the observable lifetime of pages handed out later. The impact is controlled for the same reason as for head pages. The first tail page still seeds the reusable tail template through the normal tail initialization sequence, and the copied tail pages have the same final initialized state except for the PFN-dependent fields refreshed before each copy. Tested in a VM with a 100 GB devdax namespace (align=3D2097152) on Intel Ice Lake server. This test exercises the dax_pmem rebind path and measures memmap initialization latency. Test procedure: Unbind and rebind the dax_pmem driver 30 times, collect memmap initialization time from the pr_debug() output of memmap_init_zone_device(). Base(v7.3-rc1): Average of rebinds for dax_pmem driver: 191.20 ms With this patch and its prerequisites applied: Average of rebinds for dax_pmem driver: 176.87 ms This reduces the average memmap initialization time measured during rebind from 191.20 ms to 176.87 ms, or about 7.5%. Signed-off-by: Li Zhe Reviewed-by: Mike Rapoport (Microsoft) --- mm/mm_init.c | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index ba4148de2b28..04ba37d95bfd 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1060,6 +1060,8 @@ static void __ref memmap_init_compound(struct page *h= ead, { unsigned long pfn, end_pfn =3D head_pfn + nr_pages; unsigned int order =3D pgmap->vmemmap_shift; + struct page template; + struct page *page; =20 /* * We have to initialize the pages, including setting up page links. @@ -1068,13 +1070,23 @@ static void __ref memmap_init_compound(struct page = *head, * the pages in the same go. */ __SetPageHead(head); - for (pfn =3D head_pfn + 1; pfn < end_pfn; pfn++) { - struct page *page =3D pfn_to_page(pfn); =20 - __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); - prep_compound_tail(page, head, order); - set_page_count(page, 0); - } + /* + * All tails of the same compound page share the state established by + * prep_compound_tail(). Reuse one tail template for the whole range and + * refresh only the PFN-dependent fields in that template before each cop= y. + */ + pfn =3D head_pfn + 1; + page =3D pfn_to_page(pfn); + __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + prep_compound_tail(page, head, order); + set_page_count(page, 0); + memcpy(&template, page, sizeof(*page)); + + /* Initialize the remaining tail pages from template. */ + for (pfn =3D head_pfn + 2; pfn < end_pfn; pfn++) + zone_device_page_init_from_template(pfn_to_page(pfn), pfn, + &template); prep_compound_head(head, order); } =20 --=20 2.20.1 From nobody Sat Sep 26 19:35:32 2026 Received: from va-1-114.ptr.blmpb.com (va-1-114.ptr.blmpb.com [209.127.230.114]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id AE4C939EF0A for ; Mon, 31 Aug 2026 11:19:42 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.230.114 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788175184; cv=none; b=ksvf3Ruw79LJe8uGFqN60jLDM+4pOHRO+v7VGAYh2B8c+4enYrzUPaht+JnPfB6/ZQh7StB/faIvmpxr6lEk5RLtS2OSENziQ4XFnPDgXn9DzHN9gD17qE/PQxlKMwP4x6d8qliqJd+42WvrQBzBe+hvUpL90M/0zC/82VQgE5k= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788175184; c=relaxed/simple; bh=EfJYEoeIj+mCjp87mw4swqkDiGUflYopMu+3ZYEHKXk=; h=Mime-Version:References:Cc:Content-Type:Subject:Date:Message-Id: To:From:In-Reply-To; b=lzVFAGHJ7uvafy5/NByD3x0NlVfFtFlQ7lGeAQvSesJd5bywGlEuIFEZPiX0QfIvibELOSVqW2BvPQOOpBVQTmSPLfLA+KrC880+UM8WIlZx4VQspTFGzWaDTNwf3icIRBMScCNJShTMyUOY2CcWYCIhibQN8X5gTl9Yn8urSII= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=WKffv7O1; arc=none smtp.client-ip=209.127.230.114 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="WKffv7O1" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1788175177; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=vLcTykUrPpLwId0/keP/ANeuOzuTtpvXBhogtTbcEIQ=; b=WKffv7O1sbRM1osAyUrOPj/LcydRazz9CfJLkbVjpmCjPFMacahd0G5PQSYJHm4qKtCnuH QMuTPfJyQ4UjmUO9r8iaNPmXzxu3JIN19GsSQZR3oqwutp6VWYc4KBM05Cy3PdjipoPAAm 4QCcIqvVZ87FFuju5c2m3mLZTr3nLm2eaYhZKrNYgwl7ZICl5LahpjZEx1RHQNGOfWNLNq 3PNxUtelRsAzeZQyZE6onsENE33a6A47eArJ5+BimgZRRmLVf+x58bN9SiCO1drI9WpNZq dqVb6LspxuL2e9iy1zbvq5owwS/bs2HBwdxvoGj3MtjrZdnrjfSZp+6luKxp2A== Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 X-Mailer: git-send-email 2.45.2 X-Original-From: Li Zhe References: <20260831111638.76012-1-lizhe.67@bytedance.com> Cc: , , , , , X-Lms-Return-Path: Subject: [PATCH v11 5/7] string: introduce memcpy_nontemporal() Date: Mon, 31 Aug 2026 19:16:36 +0800 Message-Id: <20260831111638.76012-6-lizhe.67@bytedance.com> Content-Transfer-Encoding: quoted-printable To: , , , , , , , , , , , From: "Li Zhe" In-Reply-To: <20260831111638.76012-1-lizhe.67@bytedance.com> Content-Type: text/plain; charset="utf-8" Introduce memcpy_nontemporal() for write-once copy sites that want a named non-temporal copy primitive. On x86_64, override the helper in arch/x86/include/asm/string_64.h using the usual self-macro pattern, next to the existing memcpy_flushcache() backend that memcpy_nontemporal() wraps. include/linux/string.h provides the generic memcpy_nontemporal() fallback as #define memcpy_nontemporal(dst, src, len) \ ((void)memcpy(dst, src, len)) instead of an inline wrapper, so architectures without a specialized backend keep the usual memcpy() FORTIFY coverage when the compiler can still see object sizes at the original call site. It also makes the memcpy_nontemporal() API uniformly void, matching memcpy_flushcache() and the x86 backend, so callers cannot accidentally depend on a return value on fallback architectures. memcpy_nontemporal() is only a copy primitive. It does not imply a drain or a publication barrier. Callers that use it before a producer-consumer or device-visible handoff must provide the required ordering at that handoff point. The immediate user is the ZONE_DEVICE template-copy path. It populates struct page descriptors in a write-once pattern, so a regular cached memcpy() can incur avoidable write-allocate traffic and cache pollution for data with little near-term reuse. Signed-off-by: Li Zhe --- arch/x86/include/asm/string_64.h | 12 ++++++++++++ include/linux/string.h | 13 +++++++++++++ 2 files changed, 25 insertions(+) diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string= _64.h index 4635616863f5..21ae515ae35a 100644 --- a/arch/x86/include/asm/string_64.h +++ b/arch/x86/include/asm/string_64.h @@ -100,6 +100,18 @@ static __always_inline void memcpy_flushcache(void *ds= t, const void *src, size_t } __memcpy_flushcache(dst, src, cnt); } + +#define memcpy_nontemporal memcpy_nontemporal +/* + * Reuse the existing x86 flushcache backend as the non-temporal copy + * primitive. + */ +static __always_inline void memcpy_nontemporal(void *dst, const void *src, + size_t cnt) +{ + memcpy_flushcache(dst, src, cnt); +} + #endif =20 #endif /* __KERNEL__ */ diff --git a/include/linux/string.h b/include/linux/string.h index 5702daca4326..6cb5cdd01158 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -278,6 +278,19 @@ static inline void memcpy_flushcache(void *dst, const = void *src, size_t cnt) } #endif =20 +#ifndef memcpy_nontemporal +/* + * memcpy_nontemporal() requests a non-temporal copy when the + * architecture has a suitable backend. Architectures without a + * specialized backend fall back to memcpy(). Keep this as a + * function-like macro so the compiler can still see the original + * memcpy() call site and preserve the usual FORTIFY coverage when + * object sizes remain visible there, while keeping the API void. + */ +#define memcpy_nontemporal(dst, src, len) \ + ((void)memcpy(dst, src, len)) +#endif + void *memchr_inv(const void *s, int c, size_t n); char *strreplace(char *str, char old, char new); =20 --=20 2.20.1 From nobody Sat Sep 26 19:35:32 2026 Received: from va-1-113.ptr.blmpb.com (va-1-113.ptr.blmpb.com [209.127.230.113]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C75E9305666 for ; Mon, 31 Aug 2026 11:20:06 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.230.113 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788175208; cv=none; b=WftsRevW0+KlOOV8tD0Nhvnyb3ZfnTi0IvC/xrH7Ta0VNfVC8FFKAatSP/SesQY2XAyrD1YfMQsdOzpoBS/6fjzwKD0WsmhpeukIXvEw7CKG+Z1hho1AEl/l1/UbOwizOPosJlUxUp5RyKkC2cvYvlz2PN5AvB1Vi2qUwaFeVBw= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788175208; c=relaxed/simple; bh=qi09e+OoIqKkSrQPZgn/hyumFnoCGX1dsiMnT5m2bNY=; h=From:Subject:Message-Id:Mime-Version:To:Cc:In-Reply-To: Content-Type:Date:References; b=UuWGG4PycYWd3h3dF0WCt7v3ZMx49MekeRzNkqBtLpDveepmuAjyza6Vk1J4TKQK+DVHP9Xn+0JVNsH0Wdl6PMzKZmdwZY9tkx7wedP2zgyCstSmombmTQ+l61KCCj0s+d3n+x2Fj1/pBx10SwYUNNi/ZRqikSVwDxBcowtDhEc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=GE/I5DYu; arc=none smtp.client-ip=209.127.230.113 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="GE/I5DYu" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1788175202; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=dRKkwMKAA6/qu2BI43qh+QT7VmhddrQx/gvVs4FuAwg=; b=GE/I5DYuIJwsNpKjzL4KSnd/3U8/WCKrjNNyEuq54QJBOp5HKmt5DTwX532iEWrORgwfQL X+GVZNqEqfdgrYAqN/sxfYWeopLzKRDN8CNn/uJud6PrH4z0D7zmZpJuq3nu4ca3EI2pNW dkCFmVukal3IG2/+uePazxedJrM0ytiIYPfYeycX0g/RvzuP0ND4npslowncqaNYRy8zli vaVX7vozrn2nsCbwUTkimB3e25jrJwxmktm5dcC/ImV5PmOj3wQaL+RN7V9HFt+M105oVl FYJJDP0orGg1DShX4X3vXExw9NlfPQHtyJ2tzXRgCqA81Dxr3v/rV6SPq8iUrg== From: "Li Zhe" Content-Transfer-Encoding: quoted-printable X-Original-From: Li Zhe Subject: [PATCH v11 6/7] mm: use memcpy_nontemporal() in zone-device template copies Message-Id: <20260831111638.76012-7-lizhe.67@bytedance.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 X-Lms-Return-Path: To: , , , , , , , , , , , Cc: , , , , , In-Reply-To: <20260831111638.76012-1-lizhe.67@bytedance.com> Date: Mon, 31 Aug 2026 19:16:37 +0800 References: <20260831111638.76012-1-lizhe.67@bytedance.com> X-Mailer: git-send-email 2.45.2 Content-Type: text/plain; charset="utf-8" The template fast path currently uses memcpy() for the actual struct page copy. Switch zone_device_page_init_from_template() to memcpy_nontemporal(). ZONE_DEVICE memmap initialization is largely write-once: each struct page is populated once, and most destination cachelines are not expected to be reused immediately afterwards. On x86, a regular cached memcpy() can therefore incur write-allocate traffic by pulling destination cachelines into the cache before writeback, and can populate the cache with data that has little near-term reuse. Using memcpy_nontemporal() lets this path request nontemporal stores for that copy pattern, which can reduce cache pollution and avoid part of the associated write-allocate overhead, while architectures without a specialized backend still fall back to memcpy(). Do not add a KASAN/KMSAN-specific fallback around this call site. As Muchun pointed out, special KASAN handling for memcpy_flushcache() or memcpy_nontemporal(), if needed, belongs in the low-level helper rather than in this ZONE_DEVICE caller. No separate drain is added here. memcpy_nontemporal() is used only as the copy primitive while memmap_init_zone_device() is still initializing the struct page array. The ordinary stores that follow in this path, such as compound-page setup, are part of the same CPU's initialization sequence; they are not used as a publication store that tells another CPU or device to consume data written by the non-temporal copy. Therefore this call site does not need a helper-level drain for correctness. Callers that use memcpy_nontemporal() as part of a producer-consumer or device-visible handoff must add the required ordering themselves. Tested in a VM with a 100 GB fsdax namespace device configured with map=3Ddev and a 100 GB devdax namespace (align=3D2097152) on Intel Ice Lake server. Test procedure: Rebind the nd_pmem and dax_pmem driver 30 times and collect the memmap initialization time from the pr_debug() output of memmap_init_zone_device(). Base(v7.3-rc1): Average of rebinds for nd_pmem driver: 221.07 ms Average of rebinds for dax_pmem driver: 191.20 ms With this patch and its prerequisites applied: Average of rebinds for nd_pmem driver: 150.40 ms Average of rebinds for dax_pmem driver: 161.83 ms This reduces the average memmap initialization time measured during rebind by about 32.0% for nd_pmem and 15.4% for dax_pmem. Signed-off-by: Li Zhe --- mm/mm_init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 04ba37d95bfd..5a61c0c83fa7 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1025,7 +1025,7 @@ static void zone_device_page_init_from_template(struc= t page *page, if (!is_highmem_idx(ZONE_DEVICE)) set_page_address(template, __va(pfn << PAGE_SHIFT)); #endif - memcpy(page, template, sizeof(*page)); + memcpy_nontemporal(page, template, sizeof(*page)); } =20 /* --=20 2.20.1 From nobody Sat Sep 26 19:35:32 2026 Received: from va-2-111.ptr.blmpb.com (va-2-111.ptr.blmpb.com [209.127.231.111]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id CE6D5282F16 for ; Mon, 31 Aug 2026 11:20:29 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.231.111 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788175232; cv=none; b=mbhpuFdLQX1m70wVBUnyP8gh4i86fZB55DlMLwpy2ozF6TzZQSueu4cw531QH2kPixsV4GuHqho95M18hxGUlybDJWXlvfnU9Z+nCI4UsGzpf3zjLHPRu0PviECo0sL4wK+adJjYOrjTFqsIR2kDqyp8Tu3XPGwD0iPux1qv5bk= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788175232; c=relaxed/simple; bh=xKWhxymsxv6nIhmLcK0gws/TUia/Ic5UXr9PVYBY7/s=; h=In-Reply-To:Content-Type:To:From:Subject:Mime-Version:Cc: Message-Id:Date:References; b=DoO1GDFWW75wNFWFtiT6Js5nGEk/gQax0JwxfGZBp+IWkVOSMSMgFlIKSxcRPbzAHmeAQT0hWZZ0pXagfp297lpZtPfmdEEVamQzS2eynfkjCvY8IkZOnUTtsb1NPdcyujruYOSJ2N317q0G3ThHD/PPh1WofQZXZaklmRgZNTw= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=qWbZE25+; arc=none smtp.client-ip=209.127.231.111 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="qWbZE25+" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1788175224; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=W3mTk0uO3fJ8loj8Vy9eCiD36YUSNCrm55bF0bJb8Z0=; b=qWbZE25+Fl/NO3nw9RCvnXO4dihmAdjyq8obIx9OzGhqChQoZCJxhpFItIo7yZKOHm8Dqp hn5MsrQa6e1Dn1qbScw3plQ6afOWl3995LkplFaD32gJ7kaBZP3vXXaiw3dlOaoRboNJot zzZxOH9zgKiUO5RU86qHZ/d0jaA/FyR3j1FiEfl/9qcZ7u9F3tmuCmTpttEONG6ftWx2y1 /MjVpCSQg5dKcXUTuyIkDuf/vHItBM08K5AofmSqCAD3M+T9ZLIKUu1GJogzSztN3H/aMZ UnehlEqzfbyOLbimAK44vg9maKpRQXiJbhTLCLVG/srG/BloRX725Ad9XXfyxw== In-Reply-To: <20260831111638.76012-1-lizhe.67@bytedance.com> To: , , , , , , , , , , , From: "Li Zhe" Subject: [PATCH v11 7/7] x86/string: extend memcpy_flushcache() fixed-size fastpaths X-Original-From: Li Zhe Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 X-Mailer: git-send-email 2.45.2 Cc: , , , , , Message-Id: <20260831111638.76012-8-lizhe.67@bytedance.com> Content-Transfer-Encoding: quoted-printable Date: Mon, 31 Aug 2026 19:16:38 +0800 References: <20260831111638.76012-1-lizhe.67@bytedance.com> X-Lms-Return-Path: Content-Type: text/plain; charset="utf-8" The x86 memcpy_nontemporal() helper maps to memcpy_flushcache(), and the ZONE_DEVICE template-copy path uses it to copy one struct page at a time. The relevant copy size is sizeof(struct page). On x86_64, the base struct page layout is 64 bytes. Adding either the KMSAN metadata pointers or an out-of-flags last_cpupid field can make it 80 bytes after alignment, and enabling both can make it 96 bytes. memcpy_flushcache() currently only has inline fixed-size cases for 4, 8, and 16 bytes. As a result, these constant-sized struct page copies fall through to __memcpy_flushcache() even though the compiler knows the copy size at the call site. Add fixed-size MOVNTI cases up to 96 bytes so the ZONE_DEVICE template-copy path can keep these struct page copies in the inline memcpy_flushcache() path. This matters for ZONE_DEVICE memmap initialization because the copy happens once per initialized struct page. For a 100 GB fsdax namespace with map=3Ddev, this is about 25 million struct page copies during nd_pmem binding or rebinding. Tested in a VM with a 100 GB fsdax namespace device configured with map=3Ddev and a 100 GB devdax namespace (align=3D2097152) on Intel Ice Lake server. Test procedure: Rebind the nd_pmem and dax_pmem drivers 30 times and collect the memmap initialization time from the pr_debug() output of memmap_init_zone_device(). With memcpy_nontemporal() used by the ZONE_DEVICE template-copy path: Average of rebinds for nd_pmem driver: 150.40 ms Average of rebinds for dax_pmem driver: 161.83 ms With this x86 fixed-size fastpath patch applied: Average of rebinds for nd_pmem driver: 71.93 ms Average of rebinds for dax_pmem driver: 87.37 ms This further reduces the average memmap initialization time measured during rebind by about 52.2% for nd_pmem and 46.0% for dax_pmem. Suggested-by: Borislav Petkov Signed-off-by: Li Zhe Acked-by: Borislav Petkov (AMD) --- arch/x86/include/asm/string_64.h | 71 +++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string= _64.h index 21ae515ae35a..831d3dda3b38 100644 --- a/arch/x86/include/asm/string_64.h +++ b/arch/x86/include/asm/string_64.h @@ -82,23 +82,64 @@ int strcmp(const char *cs, const char *ct); #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE #define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1 void __memcpy_flushcache(void *dst, const void *src, size_t cnt); -static __always_inline void memcpy_flushcache(void *dst, const void *src, = size_t cnt) + +static __always_inline void movnti_4(void *dst, const void *src) +{ + asm volatile("movntil %1, %0" + : "=3Dm"(*(u32 *)dst) + : "r"(*(const u32 *)src) + : "memory"); +} + +static __always_inline void movnti_8(void *dst, const void *src) +{ + asm volatile("movntiq %1, %0" + : "=3Dm"(*(u64 *)dst) + : "r"(*(const u64 *)src) + : "memory"); +} + +static __always_inline void movnti_16(void *dst, const void *src) +{ + movnti_8(dst, src); + movnti_8(dst + 8, src + 8); +} + +static __always_inline void movnti_32(void *dst, const void *src) +{ + movnti_16(dst, src); + movnti_16(dst + 16, src + 16); +} + +static __always_inline void movnti_64(void *dst, const void *src) +{ + movnti_32(dst, src); + movnti_32(dst + 32, src + 32); +} + +static __always_inline void memcpy_flushcache(void *dst, const void *src, + size_t cnt) { - if (__builtin_constant_p(cnt)) { - switch (cnt) { - case 4: - asm ("movntil %1, %0" : "=3Dm"(*(u32 *)dst) : "r"(*(u32 *)src)); - return; - case 8: - asm ("movntiq %1, %0" : "=3Dm"(*(u64 *)dst) : "r"(*(u64 *)src)); - return; - case 16: - asm ("movntiq %1, %0" : "=3Dm"(*(u64 *)dst) : "r"(*(u64 *)src)); - asm ("movntiq %1, %0" : "=3Dm"(*(u64 *)(dst + 8)) : "r"(*(u64 *)(src += 8))); - return; - } + if (!__builtin_constant_p(cnt)) + return __memcpy_flushcache(dst, src, cnt); + + /* + * The relevant fixed-size copies here are the x86_64 struct page sizes: + * 64, 80, and 96 bytes. Keep 32-byte and 48-byte copies inline as well + * instead of sending those nearby fixed-size cases back to + * __memcpy_flushcache(). + */ + switch (cnt) { + case 4: movnti_4(dst, src); break; + case 8: movnti_8(dst, src); break; + case 16: movnti_16(dst, src); break; + case 32: movnti_32(dst, src); break; + case 48: movnti_32(dst, src); movnti_16(dst + 32, src + 32); break; + case 64: movnti_64(dst, src); break; + case 80: movnti_64(dst, src); movnti_16(dst + 64, src + 64); break; + case 96: movnti_64(dst, src); movnti_32(dst + 64, src + 64); break; + default: __memcpy_flushcache(dst, src, cnt); break; } - __memcpy_flushcache(dst, src, cnt); } =20 #define memcpy_nontemporal memcpy_nontemporal --=20 2.20.1