From nobody Sat Jul 25 02:35:06 2026 Received: from va-1-111.ptr.blmpb.com (va-1-111.ptr.blmpb.com [209.127.230.111]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C951E409297 for ; Mon, 20 Jul 2026 12:04:39 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.230.111 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549081; cv=none; b=a7IBPyBayf77IF7keegxJl2axQ7N0RSz0GyqGeHCcbx8R5CTmKz/BR3njhI0BC/KnuQN9FuLBhQ6uFdN6/Rtw0zUpSkNDfSIrnVdYuVIl2leTJdGvvfcrOi6lCRPMx3VOlozouKwtVLCRfFlHOEUOr7GdHwN2BFc7ldfwT3d14c= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549081; c=relaxed/simple; bh=CSS9gwks0lv/fJG4J+/DCdPNhjgQ2fB/8r3a7lOVaFM=; h=To:Subject:Content-Type:Mime-Version:In-Reply-To:Message-Id:Cc: From:Date:References; b=dwvC2VvI1R/GnRrz5SWHgKElRpqTMeX8VOK9V7AtLP0GbX3GnH+00PJP1FEUO8yvqJ68huUZyaXkOASQY1ANuk06Eme8Wz+fAViaA858xVLto4xGflxskaTYzcrijNgNtQQu37qhxvaPXYO+hMYc5bXjDGdDdTKqUF7aBar7k68= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=A5NekaIJ; arc=none smtp.client-ip=209.127.230.111 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="A5NekaIJ" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1784549068; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=6LDfmUcelxmT9nJB/bHRwgrk2ZzY+G3Dy0t3Kxw/TPQ=; b=A5NekaIJyAFlECrGgRgRqQaxnKMRsmqIKIteayYHk3n8ScAbPaqT+ZHSc/k8phL4tbIAW6 mjHo6VyuJYqG2nS2pM3cX/xVLzO7Y6ASrHr0/m4LSgOCrMY0SbZAYdwG6+ddQeRhySNk45 JDk92v9+HVl7r2c/Pw9h+D5WJ7FAjleAVBtTq7NzKUNxRu/SmvT+KUjNYPOHBW7oT+Wof2 EcfP8xw5a/9vssmr/lKe5PjaqqQuGNnoYSuMVxWqLXfshDS4xTa5M6lN5TYj/NA2h834mY 7G82HmFecyFcencJqW5uKj/cugYQe9ldqPxU3DddMZpt9aOJt0sl3XDlX/oyyA== To: , , , , , , , , , , , Subject: [PATCH v7 1/9] mm: fix stale ZONE_DEVICE refcount comment X-Lms-Return-Path: Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 X-Original-From: Li Zhe X-Mailer: git-send-email 2.45.2 In-Reply-To: <20260720120259.1545-1-lizhe.67@bytedance.com> Message-Id: <20260720120259.1545-3-lizhe.67@bytedance.com> Cc: , , , , , From: "Li Zhe" Date: Mon, 20 Jul 2026 20:02:51 +0800 Content-Transfer-Encoding: quoted-printable References: <20260720120259.1545-1-lizhe.67@bytedance.com> Content-Type: text/plain; charset="utf-8" The comment in __init_zone_device_page() still uses the old MEMORY_TYPE_* names and implies that FS_DAX pages regain a refcount of 1 in the free path. That no longer matches the code. Update the comment to describe the current policy correctly: MEMORY_DEVICE_GENERIC pages regain a refcount of 1 in the free path, while the remaining ZONE_DEVICE types start from 0 here and raise the count again when the allocator or driver hands the page out. No functional change intended. Signed-off-by: Li Zhe Reviewed-by: David Hildenbrand (Arm) Reviewed-by: Alistair Popple Reviewed-by: Muchun Song --- mm/mm_init.c | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 0f64909e8d20..95808ab5cfdb 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1030,13 +1030,9 @@ static void __ref __init_zone_device_page(struct pag= e *page, unsigned long pfn, page->zone_device_data =3D NULL; =20 /* - * ZONE_DEVICE pages other than MEMORY_TYPE_GENERIC are released - * directly to the driver page allocator which will set the page count - * to 1 when allocating the page. - * - * MEMORY_TYPE_GENERIC and MEMORY_TYPE_FS_DAX pages automatically have - * their refcount reset to one whenever they are freed (ie. after - * their refcount drops to 0). + * MEMORY_DEVICE_GENERIC pages regain a refcount of 1 in the free + * path. The remaining ZONE_DEVICE types start from 0 here and raise + * the count again when the allocator or driver hands the page out. */ switch (pgmap->type) { case MEMORY_DEVICE_FS_DAX: --=20 2.20.1 From nobody Sat Jul 25 02:35:06 2026 Received: from va-1-111.ptr.blmpb.com (va-1-111.ptr.blmpb.com [209.127.230.111]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 055DF40B37D for ; Mon, 20 Jul 2026 12:04:53 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.230.111 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549095; cv=none; b=B6Tr9VGcxDt32axL9AuLAuH7BnXvTBEgVhCJDHgLDjGoc4Ro4AThOsasIth8sIf9WI18ysYKPD9TkDPYfTydWJYJZ3nd4C+VhFUj3zLr16NEigHJhiL2iFQBoWi9u+VA7iYqfpLvAGB/JROIFolfWDLBJ2rd92CQRqCXWjucy7I= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549095; c=relaxed/simple; bh=DevAn1r9WS6pPdFWehOAQ0KmRVHSptKuRZyUe2bUpqI=; h=From:Message-Id:References:Subject:Date:Mime-Version:Content-Type: To:Cc:In-Reply-To; b=p2mO0vK+CsdKsOiUu+iv62kfADJ6Pgv1QVOZCuK4nhIg061lgqngWm58kNQIFhRaw7ZT7XYeyYE7axGW+0Cd5BgEqccIjgAjiRTpOXeXrJIPNNXJ+H3hvYwqUsxsQGz4tyO6amqS72jfKA8gSEyTJTNyylCqzbBPJ4z4TnJPy94= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=NMnw1rsK; arc=none smtp.client-ip=209.127.230.111 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="NMnw1rsK" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1784549088; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=z2FZl2/YzaMjC0aIAZHsTKk9OBnNm9ulSHwBzV0fwO8=; b=NMnw1rsKv6jttCH3pLngs8UyEN0T8K7VWO6P4ZeW67iWna9zp1Xw0hNdexmdQd9eQolqR8 j+WwwGeLkkvSfLqkxQMuCMK3tYStchVS+TvzedUI4EBnzXXu9wQVw3kMCOH3QhDGWlhmoz gKqF5kp8/W4W2N0/eG5cIK5jJZe6woks75HERXRHUOyAhrgYJqK1oic6loDQchsFzKaL6e Jf29xPKvg7R8EkLC+sR7jKhQdUrkz+5pGnSaum8a9BTsHYGsEWxC5LyG8tcQn9S1eRRy2Z UbvxHfE36Zxkd+q6s5InHtqVMuq3wjRILpeQUhSk0abD172FPQ5dyp483SDq/Q== From: "Li Zhe" Message-Id: <20260720120259.1545-4-lizhe.67@bytedance.com> X-Lms-Return-Path: References: <20260720120259.1545-1-lizhe.67@bytedance.com> Subject: [PATCH v7 2/9] mm: factor zone-device page init helpers out of __init_zone_device_page Date: Mon, 20 Jul 2026 20:02:52 +0800 Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 X-Mailer: git-send-email 2.45.2 X-Original-From: Li Zhe Content-Transfer-Encoding: quoted-printable To: , , , , , , , , , , , Cc: , , , , , In-Reply-To: <20260720120259.1545-1-lizhe.67@bytedance.com> Content-Type: text/plain; charset="utf-8" memmap_init_zone_device() currently mixes refcount policy and core ZONE_DEVICE page setup in a single helper. Factor the refcount-reset predicate into pagemap_requires_refcount_reset(), move the common page initialization into __zone_device_page_init(), and wrap the existing slow path in zone_device_page_init_slow(). This keeps the slow-path behaviour unchanged and gives later patches reusable helper boundaries. No functional change intended. Signed-off-by: Li Zhe Reviewed-by: Mike Rapoport (Microsoft) --- mm/mm_init.c | 56 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 95808ab5cfdb..a70acb7431a6 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1005,11 +1005,37 @@ static void __init memmap_init(void) } =20 #ifdef CONFIG_ZONE_DEVICE -static void __ref __init_zone_device_page(struct page *page, unsigned long= pfn, +/* + * Return true when memmap_init_zone_device() must initialize the page + * refcount to 0. MEMORY_DEVICE_GENERIC pages regain a refcount of 1 in + * the free path, while the remaining ZONE_DEVICE types start from 0 here + * and raise the count again when the allocator or driver hands the page + * out. + */ +static inline bool pagemap_requires_refcount_reset(const struct dev_pagema= p *pgmap) +{ + /* + * MEMORY_DEVICE_GENERIC pages regain a refcount of 1 in the free + * path. The remaining ZONE_DEVICE types start from 0 here and raise + * the count again when the allocator or driver hands the page out. + */ + switch (pgmap->type) { + case MEMORY_DEVICE_FS_DAX: + case MEMORY_DEVICE_PRIVATE: + case MEMORY_DEVICE_COHERENT: + case MEMORY_DEVICE_PCI_P2PDMA: + return true; + case MEMORY_DEVICE_GENERIC: + return false; + } + + return false; +} + +static void __ref __zone_device_page_init(struct page *page, unsigned long= pfn, unsigned long zone_idx, int nid, struct dev_pagemap *pgmap) { - __init_single_page(page, pfn, zone_idx, nid); =20 /* @@ -1028,23 +1054,15 @@ static void __ref __init_zone_device_page(struct pa= ge *page, unsigned long pfn, */ page_folio(page)->pgmap =3D pgmap; page->zone_device_data =3D NULL; +} =20 - /* - * MEMORY_DEVICE_GENERIC pages regain a refcount of 1 in the free - * path. The remaining ZONE_DEVICE types start from 0 here and raise - * the count again when the allocator or driver hands the page out. - */ - switch (pgmap->type) { - case MEMORY_DEVICE_FS_DAX: - case MEMORY_DEVICE_PRIVATE: - case MEMORY_DEVICE_COHERENT: - case MEMORY_DEVICE_PCI_P2PDMA: +static void __ref zone_device_page_init_slow(struct page *page, + unsigned long pfn, unsigned long zone_idx, int nid, + struct dev_pagemap *pgmap) +{ + __zone_device_page_init(page, pfn, zone_idx, nid, pgmap); + if (pagemap_requires_refcount_reset(pgmap)) set_page_count(page, 0); - break; - - case MEMORY_DEVICE_GENERIC: - break; - } } =20 /* @@ -1090,7 +1108,7 @@ static void __ref memmap_init_compound(struct page *h= ead, for (pfn =3D head_pfn + 1; pfn < end_pfn; pfn++) { struct page *page =3D pfn_to_page(pfn); =20 - __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + zone_device_page_init_slow(page, pfn, zone_idx, nid, pgmap); prep_compound_tail(page, head, order); set_page_count(page, 0); } @@ -1126,7 +1144,7 @@ void __ref memmap_init_zone_device(struct zone *zone, for (pfn =3D start_pfn; pfn < end_pfn; pfn +=3D pfns_per_compound) { struct page *page =3D pfn_to_page(pfn); =20 - __init_zone_device_page(page, pfn, zone_idx, nid, pgmap); + zone_device_page_init_slow(page, pfn, zone_idx, nid, pgmap); =20 if (IS_ALIGNED(pfn, PAGES_PER_SECTION)) cond_resched(); --=20 2.20.1 From nobody Sat Jul 25 02:35:06 2026 Received: from va-1-111.ptr.blmpb.com (va-1-111.ptr.blmpb.com [209.127.230.111]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 5D3F040863E for ; Mon, 20 Jul 2026 12:05:15 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.230.111 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549116; cv=none; b=Ey8ifsM6XcuHqnz+BeilxiRX9wdo19pRD3lb6T5Bkdkfw/88oEti4duZvPbuQCgCqmpG+wK57J69JJn6peWHlDPJ5w6J8Iyh6zfTjONkZFJddmfap34pBEZaZWEeIpgEaqWpxL/Q/GV4CDW/383GhvA8/rsZm0JCBgjx9oG1Oic= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549116; c=relaxed/simple; bh=BIVDw/5zLYeH4lTLhdwm8OUozWu2BrvT0BiGbt//W2c=; h=From:Date:Content-Type:Mime-Version:In-Reply-To:To:Message-Id:Cc: Subject:References; b=Q/V4rEwTODyej1Bt5+si/bli3tj6oDa/+o56jDjGXrvXTpTtuNUdxa7TToagcukA2crBgSIhjavFYsgzBThGgsJhImaMLvudR8lSNXZcpk3jjW8S2PSN7NDZdhyvaai4cRbtOeq1XlLiPi3UoPcTDicL6FKu51EmrOIg4uBPeO4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=WbucRlui; arc=none smtp.client-ip=209.127.230.111 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="WbucRlui" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1784549110; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=HX3xWYPWMGyEZ3/iAdp9cHSDCjeSCNmVjV7Tq4qpRvQ=; b=WbucRlui/MyrjjrMbSp6s1Vv/PSyx+GSABeLjgyHjAdxnak8LtYcugyrj5rug01r/izl0j 70M7+h9A38dFsJKY6z0YRt/K6aXSn8FNXUMUjRFL5pfDOAbufuOtF0A5zev1LPPXJy/cqP r9tXmL9IarjQhv34dl5zCq4brT+OyteIv3a7FKaoozd2VpQetcDDLyrg2VgJDfRXs36w4u 3fVJeESMVCJ6KqymByHlJm/gQoNdrUr3ckKHrHy0jDSLeVVnupad7vYpKE6YhHkwQvk5Rh 3Z9afiGHbNq8PWGP2/IPV3g1lKijrYdWHTKr/RHJGTCduheUxx/hAPOkob9Ayw== From: "Li Zhe" Date: Mon, 20 Jul 2026 20:02:53 +0800 X-Mailer: git-send-email 2.45.2 Content-Transfer-Encoding: quoted-printable Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 In-Reply-To: <20260720120259.1545-1-lizhe.67@bytedance.com> X-Original-From: Li Zhe X-Lms-Return-Path: To: , , , , , , , , , , , Message-Id: <20260720120259.1545-5-lizhe.67@bytedance.com> Cc: , , , , , Subject: [PATCH v7 3/9] mm: add a set_page_section_from_pfn() helper References: <20260720120259.1545-1-lizhe.67@bytedance.com> Content-Type: text/plain; charset="utf-8" Callers that want to update section bits from a PFN currently need to open-code: set_page_section(page, pfn_to_section_nr(pfn)); and guard that sequence with #ifdef SECTION_IN_PAGE_FLAGS. Add set_page_section_from_pfn() to wrap that update in one place. When section bits are stored in page flags, the helper derives the section number from the PFN and updates the page flags. Otherwise keep it as a no-op so callers can use one helper without open-coding SECTION_IN_PAGE_FLAGS. Convert set_page_links() to use the new helper so later ZONE_DEVICE fast-path patches can also update section bits without open-coding SECTION_IN_PAGE_FLAGS at each callsite. This keeps the PFN-to-section translation local to the configurations that actually store section bits in struct page flags, and avoids exposing that detail to generic callers. No functional change intended. Signed-off-by: Li Zhe Reviewed-by: Mike Rapoport (Microsoft) Acked-by: Muchun Song Reviewed-by: Balbir Singh --- include/linux/mm.h | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/include/linux/mm.h b/include/linux/mm.h index 485df9c2dbdd..43343bfce493 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -2541,11 +2541,22 @@ static inline void set_page_section(struct page *pa= ge, unsigned long section) page->flags.f |=3D (section & SECTIONS_MASK) << SECTIONS_PGSHIFT; } =20 +static inline void set_page_section_from_pfn(struct page *page, + unsigned long pfn) +{ + set_page_section(page, pfn_to_section_nr(pfn)); +} + static inline unsigned long memdesc_section(memdesc_flags_t mdf) { return (mdf.f >> SECTIONS_PGSHIFT) & SECTIONS_MASK; } #else /* !SECTION_IN_PAGE_FLAGS */ +static inline void set_page_section_from_pfn(struct page *page, + unsigned long pfn) +{ +} + static inline unsigned long memdesc_section(memdesc_flags_t mdf) { return 0; @@ -2768,9 +2779,7 @@ static inline void set_page_links(struct page *page, = enum zone_type zone, { set_page_zone(page, zone); set_page_node(page, node); -#ifdef SECTION_IN_PAGE_FLAGS - set_page_section(page, pfn_to_section_nr(pfn)); -#endif + set_page_section_from_pfn(page, pfn); } =20 /** --=20 2.20.1 From nobody Sat Jul 25 02:35:06 2026 Received: from va-1-113.ptr.blmpb.com (va-1-113.ptr.blmpb.com [209.127.230.113]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E101540B39B for ; Mon, 20 Jul 2026 12:06:14 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.230.113 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549176; cv=none; b=QaDHsSj+DnSqcunWVpYk1TPC37JMVFM6cERXD3zTk+NKBB1EHpFOnCL1YqY+qjnlENaqOmM+HX+zkLkLMliEotAiYzSP51NozWnEmN4V5aE4H5gucp3Ps64OFGe6cO6IE7LBQcRa09F1SrUToPp2DNFU4FUhbfLCl4bLyMkyqB4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549176; c=relaxed/simple; bh=qgr93Zs18EPBcOvB82j2q2F+PNjlHHhMDXF5BEM6q6Q=; h=To:Cc:In-Reply-To:From:Date:Mime-Version:Content-Type:Subject: Message-Id:References; b=UE9ve5fR6UFz5y9lAh2LH2uExXA4db0qI99VOyzWlGe7kEeHs0TbRe5aLcz58gHI5J1/wQFxIJso31p/aSY9UhMAda8kJ5kcpz/zQ7Dl67d/IQEFaz+agYHkZbFXEEF08CkK2PPbYJS7VtBNrgZA2TztG3nW7Z53AEqs6K93MyU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=g3AQwYTz; arc=none smtp.client-ip=209.127.230.113 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="g3AQwYTz" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1784549170; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=1OXSJ03+XyIqi02GqEj3ajlM+g5iP3d6qhTy7H8GWxg=; b=g3AQwYTzMCsotReS2bPVjKGI9UDq/bp5hW2wjSfUJ7bdtGu4/08SsgOOVL1Mp6Pdj6vbqr 6xQzE3M84/J5OwlFCQWy2XWio88dc2cotCm727QtOysazZnH/MI3oplNrzApAWLGZ8NnUQ R/pq4jeyARcLNOrEX1SQ1zzDV0SGxUyAtAYhNVXFq1uQ1fSGn4ZDK4pppw5TI8xrtWruIj xoVXr2xPLKjP0alNZ4U6Ak+/VR+8vBmbZILX8YKnqoXOAuYZ8MwgBnDXG2OgnXD0GBdW1N GXUqRCiOLeKiuRHAQz5pa/ywmHbVNk/NbJs1qGogTaOsCafuXeITa8/Z2rA8iw== X-Original-From: Li Zhe Content-Transfer-Encoding: quoted-printable To: , , , , , , , , , , , Cc: , , , , , In-Reply-To: <20260720120259.1545-1-lizhe.67@bytedance.com> From: "Li Zhe" Date: Mon, 20 Jul 2026 20:02:54 +0800 Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 X-Mailer: git-send-email 2.45.2 Subject: [PATCH v7 4/9] mm: add a template-based fast path for zone-device page init Message-Id: <20260720120259.1545-6-lizhe.67@bytedance.com> X-Lms-Return-Path: References: <20260720120259.1545-1-lizhe.67@bytedance.com> Content-Type: text/plain; charset="utf-8" memmap_init_zone_device() repeats nearly identical head-page initialization for each PFN. Prepare one reusable ZONE_DEVICE head-page template through the existing slow path, refresh the PFN-dependent fields in that template before each copy, and memcpy it into each destination page. The optimized path assigns _refcount through the copied template, so keep it disabled when the page_ref_set tracepoint is enabled. This patch accelerates head-page initialization. The pfns_per_compound =3D=3D 1 case gets the full benefit here, compound tails are handled in the next patch. Tested in a VM with a 100 GB fsdax namespace device configured with map=3Ddev on Intel Ice Lake server. This test exercises the nd_pmem rebind path (pfns_per_compound =3D=3D 1). Test procedure: Rebind the nd_pmem driver 30 times and collect the memmap initialization time from the pr_debug() output of memmap_init_zone_device(). Base(v7.2-rc1): First binding: 1456 ms Average of subsequent rebinds: 244.28 ms With this patch and its prerequisites applied: First binding: 1440 ms Average of subsequent rebinds: 217.19 ms This reduces the average memmap initialization time measured during rebind from 244.28 ms to 217.19 ms, or about 11%. Signed-off-by: Li Zhe --- mm/mm_init.c | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 58 insertions(+), 1 deletion(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index a70acb7431a6..c2645b9bcef3 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1065,6 +1065,44 @@ static void __ref zone_device_page_init_slow(struct = page *page, set_page_count(page, 0); } =20 +static inline bool zone_device_page_init_optimization_enabled(void) +{ + /* + * The template fast path copies a preinitialized struct page image. + * Skip it when the page_ref_set tracepoint is enabled. + */ + return !page_ref_tracepoint_active(page_ref_set); +} + +/* + * 'template' is a reusable page prototype rather than a strictly immutable + * object. Most ZONE_DEVICE fields stay constant across the pages covered = by + * the current template, but section bits and page->virtual may still depe= nd + * on the PFN. Refresh those PFN-dependent fields in the template before + * copying it into @page. + */ +static inline void zone_device_page_update_template(struct page *template, + unsigned long pfn) +{ + set_page_section_from_pfn(template, pfn); +#ifdef WANT_PAGE_VIRTUAL + if (!is_highmem_idx(ZONE_DEVICE)) + set_page_address(template, __va(pfn << PAGE_SHIFT)); +#endif +} + +static void zone_device_page_init_from_template(struct page *page, + unsigned long pfn, struct page *template) +{ + /* + * 'template' carries the invariant portion of a ZONE_DEVICE struct + * page. Update the PFN-dependent fields in place before copying it + * to the destination page. + */ + zone_device_page_update_template(template, pfn); + memcpy(page, template, sizeof(*page)); +} + /* * With compound page geometry and when struct pages are stored in ram most * tail pages are reused. Consequently, the amount of unique struct pages = to @@ -1120,6 +1158,7 @@ void __ref memmap_init_zone_device(struct zone *zone, unsigned long nr_pages, struct dev_pagemap *pgmap) { + bool use_template =3D zone_device_page_init_optimization_enabled(); unsigned long pfn, end_pfn =3D start_pfn + nr_pages; struct pglist_data *pgdat =3D zone->zone_pgdat; struct vmem_altmap *altmap =3D pgmap_altmap(pgmap); @@ -1127,6 +1166,7 @@ void __ref memmap_init_zone_device(struct zone *zone, unsigned long zone_idx =3D zone_idx(zone); unsigned long start =3D jiffies; int nid =3D pgdat->node_id; + struct page template; =20 if (WARN_ON_ONCE(!pgmap || zone_idx !=3D ZONE_DEVICE)) return; @@ -1144,7 +1184,24 @@ void __ref memmap_init_zone_device(struct zone *zone, for (pfn =3D start_pfn; pfn < end_pfn; pfn +=3D pfns_per_compound) { struct page *page =3D pfn_to_page(pfn); =20 - zone_device_page_init_slow(page, pfn, zone_idx, nid, pgmap); + if (!use_template) { + zone_device_page_init_slow(page, pfn, zone_idx, + nid, pgmap); + } else if (pfn =3D=3D start_pfn) { + /* + * Seed the reusable head-page template from the + * first real struct page, because the existing + * page-init and pageblock helpers expect a real + * memmap entry rather than a stack object. + */ + zone_device_page_init_slow(page, pfn, zone_idx, + nid, pgmap); + /* init template page */ + memcpy(&template, page, sizeof(*page)); + } else { + zone_device_page_init_from_template(page, pfn, + &template); + } =20 if (IS_ALIGNED(pfn, PAGES_PER_SECTION)) cond_resched(); --=20 2.20.1 From nobody Sat Jul 25 02:35:06 2026 Received: from va-1-111.ptr.blmpb.com (va-1-111.ptr.blmpb.com [209.127.230.111]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 70E95397AE5 for ; Mon, 20 Jul 2026 12:06:43 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.230.111 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549205; cv=none; b=RmggByobDT9GeAMbrKbvu5cJStoJitzRvXF64MqsM3dBsT7tMEUHr+fxxHiPpbLGiuyOAgagKd/YWfIAQlnkGGmdyZ56Oxsn2DxFf0cZRCtbrceR317SHA5Yu4Umhmt9wEf6Az+D2Ck8WQbrIBgxhK3Fo/UxXMlhxdlBGioVlVQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549205; c=relaxed/simple; bh=xQcXvcyM7Mv+ZqJO2OPVPW/XAtgr8BtonMyNybIklzI=; h=Date:References:In-Reply-To:Mime-Version:Content-Type:To:Cc:From: Subject:Message-Id; b=CL7W7dOdGRb5J4lvCsaL995YbTskob7HGzhgzsCsjTavgOqwz/pQrxqwUD7cI+bVXJscgxgwIDvs1DPsO8t0SHVBCApSdK1Jk0aq0049yevW5q9XnlsWm1ZLBqDVNMfgrUVck58zkWwfqFIPneJF3s37SKTb/uMlSWYFEk9KEXE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=EmVRSfU3; arc=none smtp.client-ip=209.127.230.111 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="EmVRSfU3" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1784549198; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=jxmgrDMAVpE6lcX28qNeYhL42ykWrhIjdDLod/Kz2Jg=; b=EmVRSfU3v31CQbYY/+rjRmt8s+ij584q4javaeE/ysNOMme5O0J/tB/pqT/xsML1+mcrbc 577PUoH1joOV7Yc3tkhN2zTTEetqpZ6oLISrxKQ9HgAP5t7yBBdPkdmqJDAYFGcyc6M7UV W7d5e/zrsc8y8/CtB04WnRFG1/tl4eVMcxv4Q0G9nQscoCPIM8qP/6qZ1rtGYrRnus2BEx SUJolg4B6Oj0poeLoYdIaQQ9Ii/kbBC1gGm9PCZrHXeDsy2CBqYKDgtvV8blCixs6lq6p8 +ZyUF/PEsJdRdR+b8+zweD6j/lZqqHUJ5FNEHAmhsaPkE7g75F3homnFf1iqtQ== X-Original-From: Li Zhe Date: Mon, 20 Jul 2026 20:02:55 +0800 References: <20260720120259.1545-1-lizhe.67@bytedance.com> Content-Transfer-Encoding: quoted-printable In-Reply-To: <20260720120259.1545-1-lizhe.67@bytedance.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 X-Lms-Return-Path: To: , , , , , , , , , , , Cc: , , , , , From: "Li Zhe" Subject: [PATCH v7 5/9] mm: extend the template fast path to zone-device compound tails Message-Id: <20260720120259.1545-7-lizhe.67@bytedance.com> X-Mailer: git-send-email 2.45.2 Content-Type: text/plain; charset="utf-8" The template fast path from the previous patch only accelerates head pages. Compound tails in memmap_init_compound() still go through the zone_device_page_init_slow() one by one. Build separate head and tail templates and reuse one prepared tail template across the tail pages in a compound range. Head pages preserve the existing refcount policy, while compound tails always start with a refcount of 0 after prep_compound_tail(). This extends the template-copy fast path to pfns_per_compound > 1 without changing the existing zone_device_page_init_slow() helper. Tail-page PFN-dependent fields are refreshed in the reusable tail template before each copy. Tested in a VM with a 100 GB devdax namespace (align=3D2097152) on Intel Ice Lake server. This test exercises the dax_pmem rebind path and measures memmap initialization latency. Test procedure: Unbind and rebind the dax_pmem driver 30 times, collect memmap initialization time from the pr_debug() output of memmap_init_zone_device(). Base(v7.2-rc1): First binding: 1462 ms Average of subsequent rebinds: 273.31 ms With this patch and its prerequisites applied: First binding: 1403 ms Average of subsequent rebinds: 244.37 ms This reduces the average memmap initialization time measured during rebind from 273.31 ms to 244.37 ms, or about 10.6%. Signed-off-by: Li Zhe --- mm/mm_init.c | 40 +++++++++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index c2645b9bcef3..2668acef4fe6 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1074,6 +1074,16 @@ static inline bool zone_device_page_init_optimizatio= n_enabled(void) return !page_ref_tracepoint_active(page_ref_set); } =20 +static inline void zone_device_tail_page_init(struct page *page, + unsigned long pfn, unsigned long zone_idx, int nid, + struct dev_pagemap *pgmap, const struct page *head, + unsigned int order) +{ + zone_device_page_init_slow(page, pfn, zone_idx, nid, pgmap); + prep_compound_tail(page, head, order); + set_page_count(page, 0); +} + /* * 'template' is a reusable page prototype rather than a strictly immutable * object. Most ZONE_DEVICE fields stay constant across the pages covered = by @@ -1131,10 +1141,12 @@ static void __ref memmap_init_compound(struct page = *head, unsigned long head_pfn, unsigned long zone_idx, int nid, struct dev_pagemap *pgmap, - unsigned long nr_pages) + unsigned long nr_pages, + bool use_template) { unsigned long pfn, end_pfn =3D head_pfn + nr_pages; unsigned int order =3D pgmap->vmemmap_shift; + struct page template; =20 /* * We have to initialize the pages, including setting up page links. @@ -1143,12 +1155,29 @@ static void __ref memmap_init_compound(struct page = *head, * the pages in the same go. */ __SetPageHead(head); + for (pfn =3D head_pfn + 1; pfn < end_pfn; pfn++) { struct page *page =3D pfn_to_page(pfn); =20 - zone_device_page_init_slow(page, pfn, zone_idx, nid, pgmap); - prep_compound_tail(page, head, order); - set_page_count(page, 0); + if (!use_template) { + zone_device_tail_page_init(page, pfn, zone_idx, nid, + pgmap, head, order); + } else if (pfn =3D=3D head_pfn + 1) { + /* + * All tails of the same compound page share the + * state established by prep_compound_tail(). Reuse + * one tail template for the whole range and + * refresh only the PFN-dependent fields in that + * template before each copy. + */ + zone_device_tail_page_init(page, pfn, zone_idx, nid, + pgmap, head, order); + /* init template page */ + memcpy(&template, page, sizeof(*page)); + } else { + zone_device_page_init_from_template(page, pfn, + &template); + } } prep_compound_head(head, order); } @@ -1210,7 +1239,8 @@ void __ref memmap_init_zone_device(struct zone *zone, continue; =20 memmap_init_compound(page, pfn, zone_idx, nid, pgmap, - compound_nr_pages(pfn, altmap, pgmap)); + compound_nr_pages(pfn, altmap, pgmap), + use_template); } =20 pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE); --=20 2.20.1 From nobody Sat Jul 25 02:35:06 2026 Received: from va-1-114.ptr.blmpb.com (va-1-114.ptr.blmpb.com [209.127.230.114]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 2DFCE403B00 for ; Mon, 20 Jul 2026 12:07:22 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.230.114 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549248; cv=none; b=GaegvRi+ZEi9W53M2pFYvZfIh5NN/0+fspQU19bHwQYvtc3BXFrlQvATgLurHcOxAxvMeEL0iS5uwBfBJVe7w0Vn+bmAPjDLGKGEqTTTb+0ZyeyLo+7P6BWP3kaMze7uxMIg+ia2ZoXRUn2//lWCND5NFKB+z/4lb9fRmHCad2o= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549248; c=relaxed/simple; bh=LI2u6cYOjnFYacZ6NJqKaWuuHjSmz5yGa1W1BpxWZAk=; h=Date:In-Reply-To:To:From:Cc:Message-Id:Mime-Version:References: Subject:Content-Type; b=N7fkhTLIR+sW1WVMcfIfxlLf/CAA/yXTgrHKYsNavQHluHZVqs67F4ZlnEidlELJ/71NtnNxUTKKvlXdqgNzuRCyrlpndNHNsN35pUytLMmW7WOpnisuzF2axgqQkU1eNFJem6UBSJcBOszV6ui6qzY3cUy6znPKGmpfDkfOVzU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=UVkkQAHM; arc=none smtp.client-ip=209.127.230.114 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="UVkkQAHM" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1784549233; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=D0OG9QWc77q/zAL8FPrnNLiG/3kvsdhyOefV1kn8AAY=; b=UVkkQAHMLAeDzlYg/0cLIGmCA4LcXbccTSAL29whzLZDcvB/FJuYzSRR2qZXK/c9qfX1J4 xDzH5LR6rEgEsGl/xwJV27XpLvQNi/pZxds54Bhtd5aXLoVNOZ2cz4KrpdzKiKJ3v6vr0k V/G2fxSK1gRST59EGJ/RTkGZIW69NCrmwqB3QImNdqjcJlN1y8De/nR8pLtWVRnxvUUuTX LOrkHBpkcLe8fcvtvO35hX6omqToioenJV6oAnKv4pOQf5cnjbp6nXE2/n+9G/WK5RFi7G fEskuZoGK8lyaB8uBWmCI5iFVg41C83OCONbajDf5gWH2EEoJD1x6uAfgpXcvw== Date: Mon, 20 Jul 2026 20:02:56 +0800 In-Reply-To: <20260720120259.1545-1-lizhe.67@bytedance.com> To: , , , , , , , , , , , From: "Li Zhe" Cc: , , , , , Message-Id: <20260720120259.1545-8-lizhe.67@bytedance.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 Content-Transfer-Encoding: quoted-printable X-Lms-Return-Path: References: <20260720120259.1545-1-lizhe.67@bytedance.com> X-Mailer: git-send-email 2.45.2 Subject: [PATCH v7 6/9] string: introduce memcpy_nontemporal() X-Original-From: Li Zhe Content-Type: text/plain; charset="utf-8" Introduce memcpy_nontemporal() for write-once copy sites that want a named non-temporal copy primitive. On x86_64, override the helper in arch/x86/include/asm/string_64.h using the usual self-macro pattern, next to the existing memcpy_flushcache() backend that memcpy_nontemporal() wraps. include/linux/string.h provides the generic memcpy_nontemporal() fallback as #define memcpy_nontemporal memcpy instead of an inline wrapper, so architectures without a specialized backend keep the usual memcpy() FORTIFY coverage when the compiler can still see object sizes at the original call site. The immediate user is the ZONE_DEVICE template-copy path. It populates struct page descriptors in a write-once pattern, so a regular cached memcpy() can incur avoidable write-allocate traffic and cache pollution for data with little near-term reuse. Signed-off-by: Li Zhe --- arch/x86/include/asm/string_64.h | 12 ++++++++++++ include/linux/string.h | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string= _64.h index 4635616863f5..21ae515ae35a 100644 --- a/arch/x86/include/asm/string_64.h +++ b/arch/x86/include/asm/string_64.h @@ -100,6 +100,18 @@ static __always_inline void memcpy_flushcache(void *ds= t, const void *src, size_t } __memcpy_flushcache(dst, src, cnt); } + +#define memcpy_nontemporal memcpy_nontemporal +/* + * Reuse the existing x86 flushcache backend as the non-temporal copy + * primitive. + */ +static __always_inline void memcpy_nontemporal(void *dst, const void *src, + size_t cnt) +{ + memcpy_flushcache(dst, src, cnt); +} + #endif =20 #endif /* __KERNEL__ */ diff --git a/include/linux/string.h b/include/linux/string.h index 5702daca4326..9815c08867d7 100644 --- a/include/linux/string.h +++ b/include/linux/string.h @@ -278,6 +278,18 @@ static inline void memcpy_flushcache(void *dst, const = void *src, size_t cnt) } #endif =20 +#ifndef memcpy_nontemporal +/* + * memcpy_nontemporal() requests a non-temporal copy when the + * architecture has a suitable backend. Architectures without a + * specialized backend fall back to memcpy(). Keep this as a macro alias + * so the compiler can still see the original memcpy() call site and + * preserve the usual FORTIFY coverage when object sizes remain visible + * there. + */ +#define memcpy_nontemporal memcpy +#endif + void *memchr_inv(const void *s, int c, size_t n); char *strreplace(char *str, char old, char new); =20 --=20 2.20.1 From nobody Sat Jul 25 02:35:06 2026 Received: from va-2-112.ptr.blmpb.com (va-2-112.ptr.blmpb.com [209.127.231.112]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id C92B040D572 for ; Mon, 20 Jul 2026 12:07:38 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.231.112 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549264; cv=none; b=A6SvKske+HBxODkjQDw75/N739SmeC5e/Cpt/eNhXanV1TMA1tq/6BVs1gaG+GCMD7GpPc/DQS+oeu2EIsLpXs64s+fn/QEtJKpsQ94GExQALjy86BvPGcNOH2mUKH6VIG/LdlYfwxo4G3aAjSiqhSIZ41BHE/1ywYsFFsiE4NM= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549264; c=relaxed/simple; bh=lOfsESTU3A3Vrg05a/Gf2lU/W74DqePtJvn0GwFEsjc=; h=Mime-Version:References:Content-Type:To:Date:From:Message-Id: In-Reply-To:Cc:Subject; b=NvxVQku7BNygKG43fYVDxjWe/6FFxsakV2yvEWkejYK6G5LmJXlHW7rAtRYPnsL/+JKnDorgR+j+RCR2dQ4JUW8YzASm5tXqp8rzqPg2kY1N93kHlSCxaaIH3PjOBVArnjcPCg1tjuRRbRCDdbQ+WJb6GgGw7Olyir5cXrxJ9RI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=A47XnSxm; arc=none smtp.client-ip=209.127.231.112 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="A47XnSxm" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1784549253; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=03h9QJRQ38dlV/nJWkLzpVtuC5RS5+G1+uFcq5bFKQ8=; b=A47XnSxmMgnDWlpq6lEWGeKigMZtiy1HPFSkL7TsMrpwJHSnEL1NjNzZ4bwPTN9gJr0qtM njA03pcbtMdfSKWcNHGoo9+Fm7l7WHw/HsrKxFodFB6SQdzO1Pjv/pcvjmBJgNfN3bqvv7 Qzep3Hy/ALtENiYIvEOFoNa/p0TxEkJQJ+eEX5pLPVcURRPVgkbRPWhPp3T1YMHRNYj+bk NHWSfQ0/XofclPbpe5uBr+0GQBJymieBhUQQCn4/EPQFCtwcv+9Hq3EZqq/IZjyVT9W6G/ bPcv+687Azb/+IiH3A6Sjitx/UewCeNyHKg3TOCFuZkbz4tzX5iceVHSc6J+vg== Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 References: <20260720120259.1545-1-lizhe.67@bytedance.com> Content-Transfer-Encoding: quoted-printable To: , , , , , , , , , , , Date: Mon, 20 Jul 2026 20:02:57 +0800 X-Lms-Return-Path: X-Mailer: git-send-email 2.45.2 From: "Li Zhe" Message-Id: <20260720120259.1545-9-lizhe.67@bytedance.com> X-Original-From: Li Zhe In-Reply-To: <20260720120259.1545-1-lizhe.67@bytedance.com> Cc: , , , , , Subject: [PATCH v7 7/9] x86/string: extend memcpy_flushcache() fixed-size fastpaths Content-Type: text/plain; charset="utf-8" The new x86 memcpy_nontemporal() helper in this series maps to memcpy_flushcache(), and the ZONE_DEVICE fast path uses that primitive for constant-sized struct page template copies. The immediately relevant x86_64 sizeof(struct page) values here are 64, 80, and 96 bytes. memcpy_flushcache() currently inlines only the 4, 8, and 16-byte cases, so even those constant-sized struct page copies fall through to __memcpy_flushcache(). Add 32, 48, 64, 80, and 96-byte MOVNTI sequences to the main fixed-size switch so those constant-sized copies can stay on the inline path. Factor the larger sequences into movnti_8()/16()/32()/64() helpers so the switch can reuse them without duplicating the inline assembly. While the ZONE_DEVICE template-copy path only needs 64/80/96-byte copies, keep 32-byte and 48-byte copies inline as well rather than sending those nearby fixed-size cases back to __memcpy_flushcache(). A microbenchmark compared two memcpy_flushcache()-style helpers that shared the same generic body and differed only in whether 32/48/64/80/96-byte copies stayed in the fixed-size switch or were redirected to __memcpy_flushcache() when the destination was not 8-byte aligned. The timed interval covered the copy loop only; wmb() was used only to separate rounds. In pseudo-code, the two variants were: variant A: switch (len) { case 32: case 48: case 64: case 80: case 96: do fixed-size MOVNTI stores; return; default: generic __memcpy_flushcache()-style body; } variant B: switch (len) { case 32: case 48: case 64: case 80: case 96: if (!IS_ALIGNED(dst, 8)) goto generic_path; do fixed-size MOVNTI stores; return; default: generic_path: generic __memcpy_flushcache()-style body; } For offsets 1..7 averaged, keeping those sizes in the fixed-size switch took about 0.410/0.411/0.426/0.426/0.428 us per 32/48/64/80/96-byte copy, while redirecting the same cases to __memcpy_flushcache() took about 0.962/0.956/0.923/0.998/1.001 us. Based on that measurement, keep these new cases in the main memcpy_flushcache() fixed-size switch without a separate alignment check. Signed-off-by: Li Zhe --- arch/x86/include/asm/string_64.h | 56 +++++++++++++++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/arch/x86/include/asm/string_64.h b/arch/x86/include/asm/string= _64.h index 21ae515ae35a..bc6a9f34b346 100644 --- a/arch/x86/include/asm/string_64.h +++ b/arch/x86/include/asm/string_64.h @@ -82,7 +82,35 @@ int strcmp(const char *cs, const char *ct); #ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE #define __HAVE_ARCH_MEMCPY_FLUSHCACHE 1 void __memcpy_flushcache(void *dst, const void *src, size_t cnt); -static __always_inline void memcpy_flushcache(void *dst, const void *src, = size_t cnt) + +static __always_inline void movnti_8(void *dst, const void *src) +{ + asm volatile("movntiq %1, %0" + : "=3Dm"(*(u64 *)dst) + : "r"(*(const u64 *)src) + : "memory"); +} + +static __always_inline void movnti_16(void *dst, const void *src) +{ + movnti_8(dst, src); + movnti_8(dst + 8, src + 8); +} + +static __always_inline void movnti_32(void *dst, const void *src) +{ + movnti_16(dst, src); + movnti_16(dst + 16, src + 16); +} + +static __always_inline void movnti_64(void *dst, const void *src) +{ + movnti_32(dst, src); + movnti_32(dst + 32, src + 32); +} + +static __always_inline void memcpy_flushcache(void *dst, const void *src, + size_t cnt) { if (__builtin_constant_p(cnt)) { switch (cnt) { @@ -96,8 +124,34 @@ static __always_inline void memcpy_flushcache(void *dst= , const void *src, size_t asm ("movntiq %1, %0" : "=3Dm"(*(u64 *)dst) : "r"(*(u64 *)src)); asm ("movntiq %1, %0" : "=3Dm"(*(u64 *)(dst + 8)) : "r"(*(u64 *)(src += 8))); return; + /* + * The relevant fixed-size copies here are the + * x86_64 struct page sizes: 64, 80, and 96 bytes. + * Keep 32-byte and 48-byte copies inline as well + * instead of sending those nearby fixed-size + * cases back to __memcpy_flushcache(). + */ + case 32: + movnti_32(dst, src); + return; + case 48: + movnti_32(dst, src); + movnti_16(dst + 32, src + 32); + return; + case 64: + movnti_64(dst, src); + return; + case 80: + movnti_64(dst, src); + movnti_16(dst + 64, src + 64); + return; + case 96: + movnti_64(dst, src); + movnti_32(dst + 64, src + 64); + return; } } + __memcpy_flushcache(dst, src, cnt); } =20 --=20 2.20.1 From nobody Sat Jul 25 02:35:06 2026 Received: from va-2-115.ptr.blmpb.com (va-2-115.ptr.blmpb.com [209.127.231.115]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id D452440D57D for ; Mon, 20 Jul 2026 12:08:05 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.231.115 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549290; cv=none; b=EQbPF2HvH7EummhWUf3H8JzioBM1/thZWRR/wUDf8xO6u/KyNtZ3Un0ZbFaP6og6O/+me4TU/7ZoRO8JcL755u8aDVA6CyYjwMjqJXBrqsuNFequG6rNHksRlViOq18e0uUHxFuFjo6gY94FTi4M5lFz4J0P2ANNdwONXKS/yBs= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549290; c=relaxed/simple; bh=NVdkAhbl9bllswTnkFTRAaSUOKQuVIYHxSNZ4ZJ9tPs=; h=In-Reply-To:To:From:Date:References:Content-Type:Cc:Message-Id: Subject:Mime-Version; b=Twr9FKt+zL+XXWqleU+SUeiuAcVwrA64f6SYgqoljfbuvf38AtIBrBFsKI244ffz1hfMLkxCXbCZBkfDQabPL86oRQAg33c40oR/crzQAQa3qapDUXkCS4W3+8MTxRsypTDy+PcWFMiE2tEclkiJTKX84A7eLG/M8RBtvsCcnco= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=VWU3ZUE8; arc=none smtp.client-ip=209.127.231.115 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="VWU3ZUE8" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1784549278; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=avcd0GvpHsHr7mxbfL1p/CzBnaA4uWaToeE7s1gnya0=; b=VWU3ZUE8cWZh+o3D8L2Nl7R/fjaWU/75RGkcWxynmyrWaZ1jAD38piQKSw3e/SMo73XISM sqZ8b5YI3xVakILgJlYrluPosvj7OQJpcjyCIkTxCf8VrNm1A4OsxU5s672uJK6lUMo1ZI h4NMfRx6rkbRypNGi+liJQ0yi9p8YVB7ozKMxjB3sLrEJmnhxQdAehkpPIIJwMR7ZNkgKo 93YYXvgBaSFO6fdchYI/nGgEiLJRdXyi662cQryep4zgucg/gno9CNDe28dMuhUjrMPSHG 8F2jRoylP7SBVlOUlX7xbkRUIj4PX0x07h71fCsSJRPhfYA9vLr7vr36ZGJtaQ== In-Reply-To: <20260720120259.1545-1-lizhe.67@bytedance.com> Content-Transfer-Encoding: quoted-printable X-Mailer: git-send-email 2.45.2 To: , , , , , , , , , , , From: "Li Zhe" Date: Mon, 20 Jul 2026 20:02:58 +0800 References: <20260720120259.1545-1-lizhe.67@bytedance.com> Cc: , , , , , Message-Id: <20260720120259.1545-10-lizhe.67@bytedance.com> X-Original-From: Li Zhe X-Lms-Return-Path: Subject: [PATCH v7 8/9] mm: use memcpy_nontemporal() in zone-device template copies Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 Content-Type: text/plain; charset="utf-8" The template fast path currently uses memcpy() for the actual struct page copy. Switch zone_device_page_init_from_template() to memcpy_nontemporal(). ZONE_DEVICE memmap initialization is largely write-once: each struct page is populated once, and most destination cachelines are not expected to be reused immediately afterwards. On x86, a regular cached memcpy() can therefore incur write-allocate traffic by pulling destination cachelines into the cache before writeback, and can populate the cache with data that has little near-term reuse. Using memcpy_nontemporal() lets this path request nontemporal stores for that copy pattern, which can reduce cache pollution and avoid part of the associated write-allocate overhead, while architectures without a specialized backend still fall back to memcpy(). No separate drain is added here. memcpy_nontemporal() is used only as the copy primitive while memmap_init_zone_device() is still initializing the struct page array. The ordinary stores that follow in this path, such as compound-page setup, are part of the same CPU's initialization sequence; they are not used as a publication store that tells another CPU or device to consume data written by the non-temporal copy. Therefore this call site does not need a helper-level drain for correctness. Callers that use memcpy_nontemporal() as part of a producer-consumer or device-visible handoff must add the required ordering themselves. At this point, sanitized builds still use the slow path so KASAN/KMSAN retain their instrumented stores. The following cleanup removes that local opt-out together with the remaining non-template fallback. Tested in a VM with a 100 GB fsdax namespace device configured with map=3Ddev and a 100 GB devdax namespace (align=3D2097152) on Intel Ice Lake server. Test procedure: Rebind the nd_pmem and dax_pmem driver 30 times and collect the memmap initialization time from the pr_debug() output of memmap_init_zone_device(). Base(v7.2-rc1): First binding for nd_pmem driver: 1456 ms Average of subsequent rebinds: 244.28 ms First binding for dax_pmem driver: 1462 ms Average of subsequent rebinds: 273.31 ms With this series: First binding for nd_pmem driver: 1272 ms Average of subsequent rebinds: 96.79 ms First binding for dax_pmem driver: 1354 ms Average of subsequent rebinds: 119.04 ms This reduces the average memmap initialization time measured during rebind by about 60.4% for nd_pmem and 56.4% for dax_pmem. Signed-off-by: Li Zhe --- mm/mm_init.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 2668acef4fe6..2a723a518f41 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1067,11 +1067,21 @@ static void __ref zone_device_page_init_slow(struct= page *page, =20 static inline bool zone_device_page_init_optimization_enabled(void) { + /* + * Keep sanitized builds on the slow path so their stores stay + * instrumented. + */ + if (IS_ENABLED(CONFIG_KASAN) || IS_ENABLED(CONFIG_KMSAN)) + return false; + /* * The template fast path copies a preinitialized struct page image. * Skip it when the page_ref_set tracepoint is enabled. */ - return !page_ref_tracepoint_active(page_ref_set); + if (page_ref_tracepoint_active(page_ref_set)) + return false; + + return true; } =20 static inline void zone_device_tail_page_init(struct page *page, @@ -1110,7 +1120,7 @@ static void zone_device_page_init_from_template(struc= t page *page, * to the destination page. */ zone_device_page_update_template(template, pfn); - memcpy(page, template, sizeof(*page)); + memcpy_nontemporal(page, template, sizeof(*page)); } =20 /* --=20 2.20.1 From nobody Sat Jul 25 02:35:06 2026 Received: from va-1-112.ptr.blmpb.com (va-1-112.ptr.blmpb.com [209.127.230.112]) (using TLSv1.2 with cipher ECDHE-RSA-AES128-GCM-SHA256 (128/128 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 369CD40D584 for ; Mon, 20 Jul 2026 12:08:42 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=209.127.230.112 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549326; cv=none; b=QquYcjkzqrEh8e9j2VEkUOlzGOFMxXXoVykhAQTDH84GJoDdkdtfc9uG1E4ZXNolcihS6fVveo73z+Q5qIi9KaL0lcvQDHKqnAXQmqq05FaFcSymax1qgrNJjwsqFu8oGpMqZBgfMhBK6w1D1P+kO5kRR/7RDwDX82WpTVUwSeo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1784549326; c=relaxed/simple; bh=B2lFffzW9abBCwheY0aePRqx8cZ0hJLWqOzMVlKY2nk=; h=To:Subject:Date:Message-Id:From:Mime-Version:References: Content-Type:Cc:In-Reply-To; b=fig52UbtKG1yz2tu0YCa54Ud0jq3GlSj1zRB+Fxe1/iaqFoDBzmrMJwSlT9mupkyYtGT+1Sjfrg1pI+xDBmoc9Xo205vA9n2TGX6+nN4q8F3izM2srbeLBBuvYAoIhVDu76SfqqWHS7T0S1eXiLLr97yDdtt5zouDTxL4bMBVkU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com; spf=pass smtp.mailfrom=bytedance.com; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b=Y3ADUwL8; arc=none smtp.client-ip=209.127.230.112 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=quarantine dis=none) header.from=bytedance.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=bytedance.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=bytedance.com header.i=@bytedance.com header.b="Y3ADUwL8" DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; s=2212171451; d=bytedance.com; t=1784549316; h=from:subject: mime-version:from:date:message-id:subject:to:cc:reply-to:content-type: mime-version:in-reply-to:message-id; bh=j8bUTrgrk6ymdv5ZLPK/ZcA1DiVAjVvBeifaSs352+I=; b=Y3ADUwL88UciMimN1WEiiFPPwgiVjrR2a0x2Eot/LtPBCXNdVcXDS3VBhUk/CFGzHOKs31 98vng044HT5FYxQThZvPtq7b3O981oiTiERtxdrogTEtn/cPcFr31izfOCBUTDDkHEssvD TWYGbzgZ2i/W4tcI3zKo/9mFl8sdlEKLwab+kVPUtWKhCh3Xw68bma7N1EBsR1yEsoHxa7 juLhiNqsywhrNiH1bjMi2naKzdBUqNOC90rIcDH0ajpPANuZJnFYe/JRq8SQOPRkyBIfrw 3TSYvNJFg84seXQZXvXg9LFGg+jC9qP1d5S9k7Rg5VhDgBQHcn11ddbugSoPrg== To: , , , , , , , , , , , Subject: [PATCH v7 9/9] mm: always use the zone-device template init path Date: Mon, 20 Jul 2026 20:02:59 +0800 Message-Id: <20260720120259.1545-11-lizhe.67@bytedance.com> X-Mailer: git-send-email 2.45.2 From: "Li Zhe" Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Mime-Version: 1.0 Content-Transfer-Encoding: quoted-printable References: <20260720120259.1545-1-lizhe.67@bytedance.com> X-Lms-Return-Path: Cc: , , , , , X-Original-From: Li Zhe In-Reply-To: <20260720120259.1545-1-lizhe.67@bytedance.com> Content-Type: text/plain; charset="utf-8" The template path still has a local opt-out predicate for the page_ref_set tracepoint and keeps a non-template fallback path. This makes the code more complex without a strong reason to keep the separate path. Use the template path unconditionally after the corresponding seed page has been initialized: the first head page seeds the head template, and the first tail page in each compound range seeds the tail template. This keeps the first-page seeding logic unchanged, but removes the extra non-template fallback from the main loops. This also means sanitized builds no longer force this path back to the per-page initialization stores, and an enabled page_ref_set tracepoint will no longer observe every ZONE_DEVICE initialization-time refcount assignment. That tradeoff is intentional: this code is still initializing struct pages before they are handed out, and keeping those local debug special cases would preserve a separate initialization-only path while making the common code more complex. Suggested-by: Muchun Song Signed-off-by: Li Zhe --- mm/mm_init.c | 36 ++++-------------------------------- 1 file changed, 4 insertions(+), 32 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 2a723a518f41..ae42a1311e1d 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -1065,25 +1065,6 @@ static void __ref zone_device_page_init_slow(struct = page *page, set_page_count(page, 0); } =20 -static inline bool zone_device_page_init_optimization_enabled(void) -{ - /* - * Keep sanitized builds on the slow path so their stores stay - * instrumented. - */ - if (IS_ENABLED(CONFIG_KASAN) || IS_ENABLED(CONFIG_KMSAN)) - return false; - - /* - * The template fast path copies a preinitialized struct page image. - * Skip it when the page_ref_set tracepoint is enabled. - */ - if (page_ref_tracepoint_active(page_ref_set)) - return false; - - return true; -} - static inline void zone_device_tail_page_init(struct page *page, unsigned long pfn, unsigned long zone_idx, int nid, struct dev_pagemap *pgmap, const struct page *head, @@ -1151,8 +1132,7 @@ static void __ref memmap_init_compound(struct page *h= ead, unsigned long head_pfn, unsigned long zone_idx, int nid, struct dev_pagemap *pgmap, - unsigned long nr_pages, - bool use_template) + unsigned long nr_pages) { unsigned long pfn, end_pfn =3D head_pfn + nr_pages; unsigned int order =3D pgmap->vmemmap_shift; @@ -1169,10 +1149,7 @@ static void __ref memmap_init_compound(struct page *= head, for (pfn =3D head_pfn + 1; pfn < end_pfn; pfn++) { struct page *page =3D pfn_to_page(pfn); =20 - if (!use_template) { - zone_device_tail_page_init(page, pfn, zone_idx, nid, - pgmap, head, order); - } else if (pfn =3D=3D head_pfn + 1) { + if (pfn =3D=3D head_pfn + 1) { /* * All tails of the same compound page share the * state established by prep_compound_tail(). Reuse @@ -1197,7 +1174,6 @@ void __ref memmap_init_zone_device(struct zone *zone, unsigned long nr_pages, struct dev_pagemap *pgmap) { - bool use_template =3D zone_device_page_init_optimization_enabled(); unsigned long pfn, end_pfn =3D start_pfn + nr_pages; struct pglist_data *pgdat =3D zone->zone_pgdat; struct vmem_altmap *altmap =3D pgmap_altmap(pgmap); @@ -1223,10 +1199,7 @@ void __ref memmap_init_zone_device(struct zone *zone, for (pfn =3D start_pfn; pfn < end_pfn; pfn +=3D pfns_per_compound) { struct page *page =3D pfn_to_page(pfn); =20 - if (!use_template) { - zone_device_page_init_slow(page, pfn, zone_idx, - nid, pgmap); - } else if (pfn =3D=3D start_pfn) { + if (pfn =3D=3D start_pfn) { /* * Seed the reusable head-page template from the * first real struct page, because the existing @@ -1249,8 +1222,7 @@ void __ref memmap_init_zone_device(struct zone *zone, continue; =20 memmap_init_compound(page, pfn, zone_idx, nid, pgmap, - compound_nr_pages(pfn, altmap, pgmap), - use_template); + compound_nr_pages(pfn, altmap, pgmap)); } =20 pageblock_migratetype_init_range(start_pfn, nr_pages, MIGRATE_MOVABLE); --=20 2.20.1