From nobody Sat Oct 4 12:41:17 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id E6901271457; Mon, 18 Aug 2025 06:46:26 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755499587; cv=none; b=JPMBpZJjdtP986XUsBhd4y8lSfbxhGo8+Gu1/SrC8NzTv7MH/iPDZUoSjd314XmBk7+COQ4fymCKDXFqGikLv+ecQguBQ86V98V9jj6uTp8NBfpZ6Ogf/+G6l4AQX1gmHk08NET98KIoVcnEG+rDVe2He70Oa1AZjS+IEemwh3M= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755499587; c=relaxed/simple; bh=ToPABIT8QK9PCW9+WkqU/491rrTUTTVEcOSU5pzOhyU=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=C1eE71yDpOE1U6l+QuHnNmQn9kqGmIOKsYQg58cdsYe/4+s3gTc1j4/hn+EzpmlORjBA/V2kwNGl3UmQoVVhObjwyXBP96/+Pgo7/0rqyBPB0rCRSOSwwjfukGjG6Dh7LsPsP6xkwn6yU7KNWuCQ9wjjqViXTBCtuZdiNUi+zq8= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=b+XFnarF; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="b+XFnarF" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 672F7C4CEED; Mon, 18 Aug 2025 06:46:23 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1755499586; bh=ToPABIT8QK9PCW9+WkqU/491rrTUTTVEcOSU5pzOhyU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=b+XFnarFgko6oqbL2InH2xpcouOfZmq8dhXw2n1dYcgxlqdklKMK26ttMOjscc/Ii 8SsSFXJbsnFLbOWyXC/dkPDp8N+OUGJjai17davlyrFyQm+sgjfZU06MuPCTkcbGWs ueBHZNPFmLMMyET/B9l1AOkSAk0yF2H6uigdq40qxcQvHhPIQ526VKbKXVy96PS2SZ xkLc6fAOAKMMORb7nzJiijwQpgG8NhDAb7ffVgaL8vDCLNYoU4FbM1BW24vLnYernK 6muOvZmHPwGsDBj/3HtlhJ8OxXbm54tiQ5b8aNRLesdGAEy0jas8VvOhqtcLZcFzv+ SaGKf8uF57bGg== From: Mike Rapoport To: linux-mm@kvack.org Cc: Andrew Morton , Bill Wendling , Daniel Jordan , Justin Stitt , Michael Ellerman , Miguel Ojeda , Mike Rapoport , Nathan Chancellor , Nick Desaulniers , linux-kernel@vger.kernel.org, llvm@lists.linux.dev Subject: [PATCH 1/4] mm/mm_init: use deferred_init_memmap_chunk() in deferred_grow_zone() Date: Mon, 18 Aug 2025 09:46:12 +0300 Message-ID: <20250818064615.505641-2-rppt@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20250818064615.505641-1-rppt@kernel.org> References: <20250818064615.505641-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: "Mike Rapoport (Microsoft)" deferred_grow_zone() initializes one or more sections in the memory map if buddy runs out of initialized struct pages when CONFIG_DEFERRED_STRUCT_PAGE_INIT is enabled. It loops through memblock regions and initializes and frees pages in MAX_ORDER_NR_PAGES chunks. Essentially the same loop is implemented in deferred_init_memmap_chunk(), the only actual difference is that deferred_init_memmap_chunk() does not count initialized pages. Make deferred_init_memmap_chunk() count the initialized pages and return their number, wrap it with deferred_init_memmap_job() for multithreaded initialization with padata_do_multithreaded() and replace open-coded initialization of struct pages in deferred_grow_zone() with a call to deferred_init_memmap_chunk(). Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: David Hildenbrand Reviewed-by: Wei Yang --- mm/mm_init.c | 65 ++++++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 33 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 5c21b3af216b..81809b83814b 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2134,12 +2134,12 @@ deferred_init_maxorder(u64 *i, struct zone *zone, u= nsigned long *start_pfn, return nr_pages; } =20 -static void __init +static unsigned long __init deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, - void *arg) + struct zone *zone) { + unsigned long nr_pages =3D 0; unsigned long spfn, epfn; - struct zone *zone =3D arg; u64 i =3D 0; =20 deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); @@ -2149,9 +2149,20 @@ deferred_init_memmap_chunk(unsigned long start_pfn, = unsigned long end_pfn, * we can avoid introducing any issues with the buddy allocator. */ while (spfn < end_pfn) { - deferred_init_maxorder(&i, zone, &spfn, &epfn); + nr_pages +=3D deferred_init_maxorder(&i, zone, &spfn, &epfn); cond_resched(); } + + return nr_pages; +} + +static void __init +deferred_init_memmap_job(unsigned long start_pfn, unsigned long end_pfn, + void *arg) +{ + struct zone *zone =3D arg; + + deferred_init_memmap_chunk(start_pfn, end_pfn, zone); } =20 static unsigned int __init @@ -2204,7 +2215,7 @@ static int __init deferred_init_memmap(void *data) while (deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, first_= init_pfn)) { first_init_pfn =3D ALIGN(epfn, PAGES_PER_SECTION); struct padata_mt_job job =3D { - .thread_fn =3D deferred_init_memmap_chunk, + .thread_fn =3D deferred_init_memmap_job, .fn_arg =3D zone, .start =3D spfn, .size =3D first_init_pfn - spfn, @@ -2240,12 +2251,11 @@ static int __init deferred_init_memmap(void *data) */ bool __init deferred_grow_zone(struct zone *zone, unsigned int order) { - unsigned long nr_pages_needed =3D ALIGN(1 << order, PAGES_PER_SECTION); + unsigned long nr_pages_needed =3D SECTION_ALIGN_UP(1 << order); pg_data_t *pgdat =3D zone->zone_pgdat; unsigned long first_deferred_pfn =3D pgdat->first_deferred_pfn; unsigned long spfn, epfn, flags; unsigned long nr_pages =3D 0; - u64 i =3D 0; =20 /* Only the last zone may have deferred pages */ if (zone_end_pfn(zone) !=3D pgdat_end_pfn(pgdat)) @@ -2262,37 +2272,26 @@ bool __init deferred_grow_zone(struct zone *zone, u= nsigned int order) return true; } =20 - /* If the zone is empty somebody else may have cleared out the zone */ - if (!deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, - first_deferred_pfn)) { - pgdat->first_deferred_pfn =3D ULONG_MAX; - pgdat_resize_unlock(pgdat, &flags); - /* Retry only once. */ - return first_deferred_pfn !=3D ULONG_MAX; + /* + * Initialize at least nr_pages_needed in section chunks. + * If a section has less free memory than nr_pages_needed, the next + * section will be also initalized. + * Note, that it still does not guarantee that allocation of order can + * be satisfied if the sections are fragmented because of memblock + * allocations. + */ + for (spfn =3D first_deferred_pfn, epfn =3D SECTION_ALIGN_UP(spfn + 1); + nr_pages < nr_pages_needed && spfn < zone_end_pfn(zone); + spfn =3D epfn, epfn +=3D PAGES_PER_SECTION) { + nr_pages +=3D deferred_init_memmap_chunk(spfn, epfn, zone); } =20 /* - * Initialize and free pages in MAX_PAGE_ORDER sized increments so - * that we can avoid introducing any issues with the buddy - * allocator. + * There were no pages to initialize and free which means the zone's + * memory map is completely initialized. */ - while (spfn < epfn) { - /* update our first deferred PFN for this section */ - first_deferred_pfn =3D spfn; - - nr_pages +=3D deferred_init_maxorder(&i, zone, &spfn, &epfn); - touch_nmi_watchdog(); - - /* We should only stop along section boundaries */ - if ((first_deferred_pfn ^ spfn) < PAGES_PER_SECTION) - continue; - - /* If our quota has been met we can stop here */ - if (nr_pages >=3D nr_pages_needed) - break; - } + pgdat->first_deferred_pfn =3D nr_pages ? spfn : ULONG_MAX; =20 - pgdat->first_deferred_pfn =3D spfn; pgdat_resize_unlock(pgdat, &flags); =20 return nr_pages > 0; --=20 2.50.1 From nobody Sat Oct 4 12:41:17 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 858FF271457; Mon, 18 Aug 2025 06:46:30 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755499590; cv=none; b=faEwq1pxovpUTWzUJQw9c2hpbQz699M6jQEvGoRtuwgJlozDuZBbfdGmiqkoq4NNoFWDi0pIgZYkehQEo2cj5aItuk3DTVJcANFLDUeKvtHFzyj+psJO3AAYn7YqQ44JItuEg9wjRziIEVHZMDVwLPnFleZd4E4ybEcjS4G9C8M= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755499590; c=relaxed/simple; bh=5eSfCtdZloXF8ewebFcesyIx+wEpCfgf7oheGJfpmyY=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=TGSUgvwAFv6HdRJ1WIr1vtU08MD2GMBzMGH/CLLNCcksiwLpf16sosbQwgwMqLffsVR4UUcphMrsVdCJwlj3V4JEcbCqwFlc+CQcmG4yd+lB6XBQtsncg6RtHar+RKRrHPZugOBVFF7tbOg3pxMIeO6oRce2K+naOwcuJO1js0M= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=JIZVzUMH; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="JIZVzUMH" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 05165C4CEEB; Mon, 18 Aug 2025 06:46:26 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1755499590; bh=5eSfCtdZloXF8ewebFcesyIx+wEpCfgf7oheGJfpmyY=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=JIZVzUMHBOjxEJIsAsKLJVaP/k5p0NKfh9iQouzhRsaNWKVfLpr4A4Hn4ESMremCr U/eytjBACZepXoOYyenDDEZluSvgj5g3bZI3cInpMUYHz8Db+Y49P7D/ZTFy4Qs2WE 5ds52Bni3LYsuNAv+sqjdE8uhnOuRYwc2BfoL0ynhD2aq2inEhY7zsmwfoHJulgfSk XrnHUrqOUWsUxdMvLqo79rRZbNTN/Gh4ErlL+CtzOrxRLMoj/0F6iVu6aQHSs9Umvh 4erND4l+1nYMW68x2BY2/gudB7atMQNNFslobrMsAUcb+BAG+ljpBaItW/uGLvySMN TSRnHX3GWdUmA== From: Mike Rapoport To: linux-mm@kvack.org Cc: Andrew Morton , Bill Wendling , Daniel Jordan , Justin Stitt , Michael Ellerman , Miguel Ojeda , Mike Rapoport , Nathan Chancellor , Nick Desaulniers , linux-kernel@vger.kernel.org, llvm@lists.linux.dev Subject: [PATCH 2/4] mm/mm_init: deferred_init_memmap: use a job per zone Date: Mon, 18 Aug 2025 09:46:13 +0300 Message-ID: <20250818064615.505641-3-rppt@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20250818064615.505641-1-rppt@kernel.org> References: <20250818064615.505641-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: "Mike Rapoport (Microsoft)" deferred_init_memmap() loops over free memory ranges and creates a padata_mt_job for every free range that intersects with the zone being initialized. padata_do_multithreaded() then splits every such range to several chunks and runs a thread that initializes struct pages in that chunk using deferred_init_memmap_chunk(). The number of threads is limited by amount of the CPUs on the node (or 1 for memoryless nodes). Looping through free memory ranges is then repeated in deferred_init_memmap_chunk() first to find the first range that should be initialized and then to traverse the ranges until the end of the chunk is reached. Remove the loop over free memory regions in deferred_init_memmap() and pass the entire zone to padata_do_multithreaded() so that it will be divided to several chunks by the parallelization code. Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: David Hildenbrand Reviewed-by: Wei Yang --- mm/mm_init.c | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 81809b83814b..1ecfba98ddbe 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2176,12 +2176,10 @@ static int __init deferred_init_memmap(void *data) { pg_data_t *pgdat =3D data; const struct cpumask *cpumask =3D cpumask_of_node(pgdat->node_id); - unsigned long spfn =3D 0, epfn =3D 0; - unsigned long first_init_pfn, flags; + int max_threads =3D deferred_page_init_max_threads(cpumask); + unsigned long first_init_pfn, last_pfn, flags; unsigned long start =3D jiffies; struct zone *zone; - int max_threads; - u64 i =3D 0; =20 /* Bind memory initialisation thread to a local node if possible */ if (!cpumask_empty(cpumask)) @@ -2209,24 +2207,20 @@ static int __init deferred_init_memmap(void *data) =20 /* Only the highest zone is deferred */ zone =3D pgdat->node_zones + pgdat->nr_zones - 1; - - max_threads =3D deferred_page_init_max_threads(cpumask); - - while (deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, first_= init_pfn)) { - first_init_pfn =3D ALIGN(epfn, PAGES_PER_SECTION); - struct padata_mt_job job =3D { - .thread_fn =3D deferred_init_memmap_job, - .fn_arg =3D zone, - .start =3D spfn, - .size =3D first_init_pfn - spfn, - .align =3D PAGES_PER_SECTION, - .min_chunk =3D PAGES_PER_SECTION, - .max_threads =3D max_threads, - .numa_aware =3D false, - }; - - padata_do_multithreaded(&job); - } + last_pfn =3D SECTION_ALIGN_UP(zone_end_pfn(zone)); + + struct padata_mt_job job =3D { + .thread_fn =3D deferred_init_memmap_job, + .fn_arg =3D zone, + .start =3D first_init_pfn, + .size =3D last_pfn - first_init_pfn, + .align =3D PAGES_PER_SECTION, + .min_chunk =3D PAGES_PER_SECTION, + .max_threads =3D max_threads, + .numa_aware =3D false, + }; + + padata_do_multithreaded(&job); =20 /* Sanity check that the next zone really is unpopulated */ WARN_ON(pgdat->nr_zones < MAX_NR_ZONES && populated_zone(++zone)); --=20 2.50.1 From nobody Sat Oct 4 12:41:17 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 0C23C25C83A; Mon, 18 Aug 2025 06:46:33 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755499594; cv=none; b=PXYby5tEh6GN5PViM9kuQifTxi6DL2iI5NPRXivrZE/k3+c4ii3kmjb8njCR3LQxbk07bC1Bfr0N9pHKdr9bQ7bRTWQlZdxh9TE5fEO8DnvpJuPQWSjaiHEpwhEL1Q+h/LBNfk5YJCpM5mw9DUhVqIxT+qU9eROAksxkMS6ANAA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755499594; c=relaxed/simple; bh=DisL+u0A+oDt7cK+lZ4T+ubCByoMEuH39AEZRCnBWcM=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=ACSrboQOBEj6TdfY28cZ2xyUSUotTrEsIMpcp3uDcb12sRwXrvlAtY73+POepzWiMWto0MbuVhfKCMQmA+MXzIwD1f3X/aYRruw+dUjNBN1jOlVeHxdlKjImWymIxG3N6zCHrG/zvYVtd04KE2iUF+SaPM2hizMLaO4ZQ/pLCsI= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=MHwvojO5; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="MHwvojO5" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 96028C4CEED; Mon, 18 Aug 2025 06:46:30 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1755499593; bh=DisL+u0A+oDt7cK+lZ4T+ubCByoMEuH39AEZRCnBWcM=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=MHwvojO5ZNQ/vIbqTX2n+p36PC7eGEFq5CKUCZcbcf+pDLqlU5qIAkSNiv7MO9IzM PQ+YPYOTEmQ6eW0FZFl5PgCJQQZ/kBct1QG9lJweWjnLh238X+HGsXGJpWZaxA+ZAT ysCq4vtiZoiCv5ENtSwG7GMkrUD0nlaTBdMdsKdNF2YkXz4Tt0TIHQXjaqMxIgK6py D3LwtTcQRZpsR7d7rN2my9k+45XHP9FfmM3HzPbEUET+rQG1IBYvvNkkrDWZgchm7q ktz1nk2n5SFzJXxJljWaJUdV6ZUj3gw6WBkd3ccgikDjUGIGuuFIfEkDt8zXNC4ewp B63YizXPZVfYA== From: Mike Rapoport To: linux-mm@kvack.org Cc: Andrew Morton , Bill Wendling , Daniel Jordan , Justin Stitt , Michael Ellerman , Miguel Ojeda , Mike Rapoport , Nathan Chancellor , Nick Desaulniers , linux-kernel@vger.kernel.org, llvm@lists.linux.dev Subject: [PATCH 3/4] mm/mm_init: drop deferred_init_maxorder() Date: Mon, 18 Aug 2025 09:46:14 +0300 Message-ID: <20250818064615.505641-4-rppt@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20250818064615.505641-1-rppt@kernel.org> References: <20250818064615.505641-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: "Mike Rapoport (Microsoft)" deferred_init_memmap_chunk() calls deferred_init_maxorder() to initialize struct pages in MAX_ORDER_NR_PAGES because according to commit 0e56acae4b4d ("mm: initialize MAX_ORDER_NR_PAGES at a time instead of doing larger sections") this provides better cache locality than initializing the memory map in larger sections. The looping through free memory ranges is quite cumbersome in the current implementation as it is divided between deferred_init_memmap_chunk() and deferred_init_maxorder(). Besides, the latter has two loops, one that initializes struct pages and another one that frees them. There is no need in two loops because it is safe to free pages in groups smaller than MAX_ORDER_NR_PAGES. Even if lookup for a buddy page will access a struct page ahead of the pages being initialized, that page is guaranteed to be initialized either by memmap_init_reserved_pages() or by init_unavailable_range(). Simplify the code by moving initialization and freeing of the pages into deferred_init_memmap_chunk() and dropping deferred_init_maxorder(). Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: David Hildenbrand Reviewed-by: Wei Yang --- mm/mm_init.c | 122 ++++++++++++--------------------------------------- 1 file changed, 29 insertions(+), 93 deletions(-) diff --git a/mm/mm_init.c b/mm/mm_init.c index 1ecfba98ddbe..bca05891cb16 100644 --- a/mm/mm_init.c +++ b/mm/mm_init.c @@ -2046,111 +2046,47 @@ static unsigned long __init deferred_init_pages(st= ruct zone *zone, } =20 /* - * This function is meant to pre-load the iterator for the zone init from - * a given point. - * Specifically it walks through the ranges starting with initial index - * passed to it until we are caught up to the first_init_pfn value and - * exits there. If we never encounter the value we return false indicating - * there are no valid ranges left. - */ -static bool __init -deferred_init_mem_pfn_range_in_zone(u64 *i, struct zone *zone, - unsigned long *spfn, unsigned long *epfn, - unsigned long first_init_pfn) -{ - u64 j =3D *i; - - if (j =3D=3D 0) - __next_mem_pfn_range_in_zone(&j, zone, spfn, epfn); - - /* - * Start out by walking through the ranges in this zone that have - * already been initialized. We don't need to do anything with them - * so we just need to flush them out of the system. - */ - for_each_free_mem_pfn_range_in_zone_from(j, zone, spfn, epfn) { - if (*epfn <=3D first_init_pfn) - continue; - if (*spfn < first_init_pfn) - *spfn =3D first_init_pfn; - *i =3D j; - return true; - } - - return false; -} - -/* - * Initialize and free pages. We do it in two loops: first we initialize - * struct page, then free to buddy allocator, because while we are - * freeing pages we can access pages that are ahead (computing buddy - * page in __free_one_page()). + * Initialize and free pages. + * + * At this point reserved pages and struct pages that correspond to holes = in + * memblock.memory are already intialized so every free range has a valid + * memory map around it. + * This ensures that access of pages that are ahead of the range being + * initialized (computing buddy page in __free_one_page()) always reads a = valid + * struct page. * - * In order to try and keep some memory in the cache we have the loop - * broken along max page order boundaries. This way we will not cause - * any issues with the buddy page computation. + * In order to try and improve CPU cache locality we have the loop broken = along + * max page order boundaries. */ static unsigned long __init -deferred_init_maxorder(u64 *i, struct zone *zone, unsigned long *start_pfn, - unsigned long *end_pfn) +deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, + struct zone *zone) { - unsigned long mo_pfn =3D ALIGN(*start_pfn + 1, MAX_ORDER_NR_PAGES); - unsigned long spfn =3D *start_pfn, epfn =3D *end_pfn; + int nid =3D zone_to_nid(zone); unsigned long nr_pages =3D 0; - u64 j =3D *i; - - /* First we loop through and initialize the page values */ - for_each_free_mem_pfn_range_in_zone_from(j, zone, start_pfn, end_pfn) { - unsigned long t; - - if (mo_pfn <=3D *start_pfn) - break; - - t =3D min(mo_pfn, *end_pfn); - nr_pages +=3D deferred_init_pages(zone, *start_pfn, t); - - if (mo_pfn < *end_pfn) { - *start_pfn =3D mo_pfn; - break; - } - } - - /* Reset values and now loop through freeing pages as needed */ - swap(j, *i); - - for_each_free_mem_pfn_range_in_zone_from(j, zone, &spfn, &epfn) { - unsigned long t; - - if (mo_pfn <=3D spfn) - break; + phys_addr_t start, end; + u64 i =3D 0; =20 - t =3D min(mo_pfn, epfn); - deferred_free_pages(spfn, t - spfn); + for_each_free_mem_range(i, nid, 0, &start, &end, NULL) { + unsigned long spfn =3D PFN_UP(start); + unsigned long epfn =3D PFN_DOWN(end); =20 - if (mo_pfn <=3D epfn) + if (spfn >=3D end_pfn) break; - } =20 - return nr_pages; -} + spfn =3D max(spfn, start_pfn); + epfn =3D min(epfn, end_pfn); =20 -static unsigned long __init -deferred_init_memmap_chunk(unsigned long start_pfn, unsigned long end_pfn, - struct zone *zone) -{ - unsigned long nr_pages =3D 0; - unsigned long spfn, epfn; - u64 i =3D 0; + while (spfn < epfn) { + unsigned long mo_pfn =3D ALIGN(spfn + 1, MAX_ORDER_NR_PAGES); + unsigned long chunk_end =3D min(mo_pfn, epfn); =20 - deferred_init_mem_pfn_range_in_zone(&i, zone, &spfn, &epfn, start_pfn); + nr_pages +=3D deferred_init_pages(zone, spfn, chunk_end); + deferred_free_pages(spfn, chunk_end - spfn); =20 - /* - * Initialize and free pages in MAX_PAGE_ORDER sized increments so that - * we can avoid introducing any issues with the buddy allocator. - */ - while (spfn < end_pfn) { - nr_pages +=3D deferred_init_maxorder(&i, zone, &spfn, &epfn); - cond_resched(); + spfn =3D chunk_end; + cond_resched(); + } } =20 return nr_pages; --=20 2.50.1 From nobody Sat Oct 4 12:41:17 2025 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-1.web.codeaurora.org [10.30.226.201]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id BA98625C83A; Mon, 18 Aug 2025 06:46:37 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=10.30.226.201 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755499597; cv=none; b=uneCBYueVItCwGjen+uE1LSZcuNReKX5vOWpSBO4hRUmQf1VNn1i/rJiGAAQT4SjyBqLn0GzQz1+az9mVkAF2AXqarfqTH7+D0q4B/wV+lJaw8cL5x/ydG2HsGutHwkLG8nfywA6JpGgQt4DaqdS/cqhHvsz3V9MfOoMGTqxJBc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1755499597; c=relaxed/simple; bh=mWBw2dxrn3qCb6ZjGXqIStKZeXRZV0QwSHDvWZaNzXg=; h=From:To:Cc:Subject:Date:Message-ID:In-Reply-To:References: MIME-Version; b=XYaTldkY4j4JlPPTGuw/gyuvDu2WWzptplKLuNcyDY+KQ5elhvPzbJv6age3NWHyV+3gvmlK9C6ZwGbgTwRjw3/8hwncMEnZkRLaAJePNntLJBjqm/i4bGCkWyp65uRwxP8/6AwboKCwEj/uLBHYSNXNMfpoeXZvU63UixqAw+s= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=u0Rpyu4y; arc=none smtp.client-ip=10.30.226.201 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="u0Rpyu4y" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 325E2C4CEEB; Mon, 18 Aug 2025 06:46:33 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=kernel.org; s=k20201202; t=1755499597; bh=mWBw2dxrn3qCb6ZjGXqIStKZeXRZV0QwSHDvWZaNzXg=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=u0Rpyu4y8bEUq8EbFwZHD5j3l2gjivkI8HDDJLhyxDRHEzCdCC1nxhMjOdh19vgTq l2Rzc/qXKbDwoWkF8Ma9mBNItHubHiDEcL60X2pyfsp/yozfyASpmtNvkn2ppiP6bB K6wWyhJ/w6QjVyS96IfFo28aq/M+NW61AdgJE1nl989dXsHPbfrR1fQmRp6glOM3xP 5DSgy8XwoD+L7Spl9BPnUEIZbM9qVXYU7mj/i2XxngBbxSUtSsf8JKLy3qvkjxZySg d7lut5t2oHgzynTJjzf0lUc+6ZM4fT2vERwhMokUsHvgPSi31DWsxsDcw2doc3jVyJ qazIie45dnMmQ== From: Mike Rapoport To: linux-mm@kvack.org Cc: Andrew Morton , Bill Wendling , Daniel Jordan , Justin Stitt , Michael Ellerman , Miguel Ojeda , Mike Rapoport , Nathan Chancellor , Nick Desaulniers , linux-kernel@vger.kernel.org, llvm@lists.linux.dev Subject: [PATCH 4/4] memblock: drop for_each_free_mem_pfn_range_in_zone_from() Date: Mon, 18 Aug 2025 09:46:15 +0300 Message-ID: <20250818064615.505641-5-rppt@kernel.org> X-Mailer: git-send-email 2.50.1 In-Reply-To: <20250818064615.505641-1-rppt@kernel.org> References: <20250818064615.505641-1-rppt@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" From: "Mike Rapoport (Microsoft)" for_each_free_mem_pfn_range_in_zone_from() and its "backend" implementation __next_mem_pfn_range_in_zone() were only used by deferred initialization of the memory map. Remove them as they are not used anymore. Signed-off-by: Mike Rapoport (Microsoft) Reviewed-by: Wei Yang --- .clang-format | 1 - include/linux/memblock.h | 22 -------------- mm/memblock.c | 64 ---------------------------------------- 3 files changed, 87 deletions(-) diff --git a/.clang-format b/.clang-format index 48405c54ef27..f371a13b4d19 100644 --- a/.clang-format +++ b/.clang-format @@ -294,7 +294,6 @@ ForEachMacros: - 'for_each_fib6_node_rt_rcu' - 'for_each_fib6_walker_rt' - 'for_each_file_lock' - - 'for_each_free_mem_pfn_range_in_zone_from' - 'for_each_free_mem_range' - 'for_each_free_mem_range_reverse' - 'for_each_func_rsrc' diff --git a/include/linux/memblock.h b/include/linux/memblock.h index b96746376e17..20b61e910f4d 100644 --- a/include/linux/memblock.h +++ b/include/linux/memblock.h @@ -323,28 +323,6 @@ void __next_mem_pfn_range(int *idx, int nid, unsigned = long *out_start_pfn, for (i =3D -1, __next_mem_pfn_range(&i, nid, p_start, p_end, p_nid); \ i >=3D 0; __next_mem_pfn_range(&i, nid, p_start, p_end, p_nid)) =20 -#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT -void __next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone, - unsigned long *out_spfn, - unsigned long *out_epfn); - -/** - * for_each_free_mem_pfn_range_in_zone_from - iterate through zone specific - * free memblock areas from a given point - * @i: u64 used as loop variable - * @zone: zone in which all of the memory blocks reside - * @p_start: ptr to phys_addr_t for start address of the range, can be %NU= LL - * @p_end: ptr to phys_addr_t for end address of the range, can be %NULL - * - * Walks over free (memory && !reserved) areas of memblock in a specific - * zone, continuing from current position. Available as soon as memblock is - * initialized. - */ -#define for_each_free_mem_pfn_range_in_zone_from(i, zone, p_start, p_end) \ - for (; i !=3D U64_MAX; \ - __next_mem_pfn_range_in_zone(&i, zone, p_start, p_end)) - -#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ =20 /** * for_each_free_mem_range - iterate through free memblock areas diff --git a/mm/memblock.c b/mm/memblock.c index 154f1d73b61f..337c025109fa 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -1438,70 +1438,6 @@ int __init_memblock memblock_set_node(phys_addr_t ba= se, phys_addr_t size, return 0; } =20 -#ifdef CONFIG_DEFERRED_STRUCT_PAGE_INIT -/** - * __next_mem_pfn_range_in_zone - iterator for for_each_*_range_in_zone() - * - * @idx: pointer to u64 loop variable - * @zone: zone in which all of the memory blocks reside - * @out_spfn: ptr to ulong for start pfn of the range, can be %NULL - * @out_epfn: ptr to ulong for end pfn of the range, can be %NULL - * - * This function is meant to be a zone/pfn specific wrapper for the - * for_each_mem_range type iterators. Specifically they are used in the - * deferred memory init routines and as such we were duplicating much of - * this logic throughout the code. So instead of having it in multiple - * locations it seemed like it would make more sense to centralize this to - * one new iterator that does everything they need. - */ -void __init_memblock -__next_mem_pfn_range_in_zone(u64 *idx, struct zone *zone, - unsigned long *out_spfn, unsigned long *out_epfn) -{ - int zone_nid =3D zone_to_nid(zone); - phys_addr_t spa, epa; - - __next_mem_range(idx, zone_nid, MEMBLOCK_NONE, - &memblock.memory, &memblock.reserved, - &spa, &epa, NULL); - - while (*idx !=3D U64_MAX) { - unsigned long epfn =3D PFN_DOWN(epa); - unsigned long spfn =3D PFN_UP(spa); - - /* - * Verify the end is at least past the start of the zone and - * that we have at least one PFN to initialize. - */ - if (zone->zone_start_pfn < epfn && spfn < epfn) { - /* if we went too far just stop searching */ - if (zone_end_pfn(zone) <=3D spfn) { - *idx =3D U64_MAX; - break; - } - - if (out_spfn) - *out_spfn =3D max(zone->zone_start_pfn, spfn); - if (out_epfn) - *out_epfn =3D min(zone_end_pfn(zone), epfn); - - return; - } - - __next_mem_range(idx, zone_nid, MEMBLOCK_NONE, - &memblock.memory, &memblock.reserved, - &spa, &epa, NULL); - } - - /* signal end of iteration */ - if (out_spfn) - *out_spfn =3D ULONG_MAX; - if (out_epfn) - *out_epfn =3D 0; -} - -#endif /* CONFIG_DEFERRED_STRUCT_PAGE_INIT */ - /** * memblock_alloc_range_nid - allocate boot memory block * @size: size of memory block to be allocated in bytes --=20 2.50.1