From nobody Sat Sep 26 11:46:51 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 692C646D541 for ; Tue, 1 Sep 2026 23:24:32 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788305073; cv=none; b=Me5JiufaqS7sJAASOm/JozsgOPkLyI+3s27TqxAiENhcM/toIr+QPj6ybvA/bw7aCc7TYvL5hod0PZsuDlFUMnjln/vYDO44eK3QC2VlW0Y0dqUd39Z4nmWwpr6x5LofD7Kh3+v/PBOSx5v1OtRH5tLeUr6p6O3BB5slMUWNEzo= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788305073; c=relaxed/simple; bh=pBddXw9kLtL3WD5gu4P1b2Rk7upxu2Ms30pxfE4ZHgE=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=aCsYs1zQ7ec6YcMemxA2Y3pHCd5wdQSuuL/zPXw8Cc40D+80FBj1Zo11CgcnhSr8AsbUkX1M7LX6aUzs3Mu0JLj2zDifAVRSKxcHNn2z+ZunSfwkr3QMM51s+F7wKBG173JXSb+o94mT5DKlIkg1Hh3G8vrug+PenKtghOz6/1s= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=ONNpfGFz; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="ONNpfGFz" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 44C771F00A3A; Tue, 1 Sep 2026 23:24:28 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1788305072; bh=Z0X7OvF6yz6PTqxJmY35ICxOJAuUHhLkK951+WZcIng=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=ONNpfGFzn7/T6wh+dIHvukw5pagye5MuBBVr0HN8zSArlZBW3NpZi89/Q2VTjz1Xu efTyofK06CMYGUYqzgpM0x/rEsRcvbNkiUNW/IQNcP+0c/Orb1+sTq2p/df5AntkE6 YLaw1PIsQYaYlRqL0xHMxajJPelQ+WPNigtM/EEqAxb2iI108K6c+IJYbjJQy1pE1B QvdoXEq62EqTOXIcMaXdBX0stgWpI5ayA8z2IVg0feyRkgpzHWk1mTSYjgKILTg0HR B/TxgSKyq9pF7pZN2PM77xw3kMhEkbK2BHHpNw+Phfd9xafJWR70FE47gVJhALnIMb Dn1MS+JSVqdbQ== From: "Barry Song (Xiaomi)" To: akpm@linux-foundation.org, linux-mm@kvack.org Cc: axelrasmussen@google.com, baolin.wang@linux.alibaba.com, baoquan.he@linux.dev, chenridong@xiaomi.com, david@kernel.org, hannes@cmpxchg.org, kasong@tencent.com, lianux.mm@gmail.com, linux-kernel@vger.kernel.org, ljs@kernel.org, lyugaofei@xiaomi.com, mhocko@kernel.org, qi.zheng@linux.dev, shakeel.butt@linux.dev, stevensd@chromium.org, wangzicheng@honor.com, weixugc@google.com, yuanchu@google.com, zhangbo56@xiaomi.com, "Barry Song (Xiaomi)" , Xueyuan Chen Subject: [PATCH v3 1/7] mm/mglru: separate folio generation update from LRU accounting Date: Wed, 2 Sep 2026 07:24:15 +0800 Message-Id: <20260901232421.40157-2-baohua@kernel.org> X-Mailer: git-send-email 2.39.3 (Apple Git-146) In-Reply-To: <20260901232421.40157-1-baohua@kernel.org> References: <20260901232421.40157-1-baohua@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" folio_inc_gen() currently updates both the folio's generation and the LRU size accounting. This makes it difficult to batch the LRU size updates when moving multiple folios. Extract the generation update into __folio_inc_gen(), which only updates the folio's generation and reports whether the generation was actually increased. Keep folio_inc_gen() as the wrapper that performs the LRU size accounting when needed. This separates the per-folio generation update from LRU accounting and allows the latter to be batched by subsequent changes. Signed-off-by: Barry Song (Xiaomi) Tested-by: Xueyuan Chen Reviewed-by: Lian Wang Reviewed-by: Baolin Wang --- mm/vmscan.c | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index f11491ee9ed5..54bce2f608ef 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3295,21 +3295,21 @@ static int folio_update_gen(struct folio *folio, in= t gen, const vma_flags_t *vma return ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; } =20 -/* protect pages accessed multiple times through file descriptors */ -static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) +static int __folio_inc_gen(struct folio *folio, int old_gen, bool *increas= ed) { - int type =3D folio_is_file_lru(folio); - struct lru_gen_folio *lrugen =3D &lruvec->lrugen; - int new_gen, old_gen =3D lru_gen_from_seq(lrugen->min_seq[type]); unsigned long new_flags, old_flags =3D READ_ONCE(folio->flags.f); + int new_gen; =20 VM_WARN_ON_ONCE_FOLIO(!(old_flags & LRU_GEN_MASK), folio); =20 do { new_gen =3D ((old_flags & LRU_GEN_MASK) >> LRU_GEN_PGOFF) - 1; /* folio_update_gen() has promoted this page? */ - if (new_gen >=3D 0 && new_gen !=3D old_gen) + if (new_gen >=3D 0 && new_gen !=3D old_gen) { + if (increased) + *increased =3D false; return new_gen; + } =20 new_gen =3D (old_gen + 1) % MAX_NR_GENS; =20 @@ -3317,8 +3317,22 @@ static int folio_inc_gen(struct lruvec *lruvec, stru= ct folio *folio) new_flags |=3D (new_gen + 1UL) << LRU_GEN_PGOFF; } while (!try_cmpxchg(&folio->flags.f, &old_flags, new_flags)); =20 - lru_gen_update_size(lruvec, folio, old_gen, new_gen); + if (increased) + *increased =3D true; + return new_gen; +} + +/* protect pages accessed multiple times through file descriptors */ +static int folio_inc_gen(struct lruvec *lruvec, struct folio *folio) +{ + int type =3D folio_is_file_lru(folio); + struct lru_gen_folio *lrugen =3D &lruvec->lrugen; + int new_gen, old_gen =3D lru_gen_from_seq(lrugen->min_seq[type]); + bool gen_increased; =20 + new_gen =3D __folio_inc_gen(folio, old_gen, &gen_increased); + if (gen_increased) + lru_gen_update_size(lruvec, folio, old_gen, new_gen); return new_gen; } =20 --=20 2.34.1 From nobody Sat Sep 26 11:46:51 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id DF11646D541 for ; Tue, 1 Sep 2026 23:24:36 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788305078; cv=none; b=iFObfUKkliovtnf1RdK3Vz+3VvNaDlB3vanlNenUzeVvMyVinWPYdH4RknPHU8nAPEA17nWkgA8MRhIo9OqvHgVGsnr4TTbHm9UhHoceWnE3+d8QqGVZ3/mLgeqeu4eKP56koxItSKXoBv0ZZK+EnM1O0Nqjc6sCzWF0tXbrCJ8= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788305078; c=relaxed/simple; bh=NP8ebDdt8O3z8l/Sq5Z2hu6aATnHI2jjkvuT1DrtszY=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=SV+rQwFaGlUY0lN1XkZb5rtPNdr+j0kjLk13z6qS+vC95mfRND0xMXAau4j+BPhewQ8+WWBtj2YK3sq6eYRJLvt3LMC6IeDS7DwSqbYxitnIVS8uMEEbgf0zNhKC+drJDv9X949FEzPPgNRLQoOekQnFptozoY+J0+9ScllBbhc= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=TXAQ3QSm; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="TXAQ3QSm" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 73A611F000E9; Tue, 1 Sep 2026 23:24:32 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1788305076; bh=3mrvoCwf3JhHo6/B0HvRbUVwX76oJvSOJSq8DmmzItY=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=TXAQ3QSmW1kL0/7w7FaPW8RnsDkCuqPXbtBoqB0y2WXus7vg1zrqEl/L0Q9l4bFNl ri6q7uhCpuIhvRBdIjis6rb2sE9cN1rdK//O9NnxouQgBFBkHYVcliV0NisEKwbKed O6H0SpjWaCY9ptsqg0RI9Hgkpm3K0n/9OjVK7PigCLKdp0HbR3EtbEjHziSrN9wFqb YahoOIrYqne51/guURoNXtmcDMcZ/7j++r1/CM3q8wHS+CU9JTsWkCrbH69xgLaWVq v5TROxmYjEv9+W9pIxioG92t0dslVeEDrNNUXZ4ibzKWFR2n1zTLfcWd3SUJpMVblx C25N/pdk72h3g== From: "Barry Song (Xiaomi)" To: akpm@linux-foundation.org, linux-mm@kvack.org Cc: axelrasmussen@google.com, baolin.wang@linux.alibaba.com, baoquan.he@linux.dev, chenridong@xiaomi.com, david@kernel.org, hannes@cmpxchg.org, kasong@tencent.com, lianux.mm@gmail.com, linux-kernel@vger.kernel.org, ljs@kernel.org, lyugaofei@xiaomi.com, mhocko@kernel.org, qi.zheng@linux.dev, shakeel.butt@linux.dev, stevensd@chromium.org, wangzicheng@honor.com, weixugc@google.com, yuanchu@google.com, zhangbo56@xiaomi.com, "Barry Song (Xiaomi)" , Xueyuan Chen , Kunwu Chan Subject: [PATCH v3 2/7] mm/mglru: batch update lrugen->nr_pages in inc_min_seq() Date: Wed, 2 Sep 2026 07:24:16 +0800 Message-Id: <20260901232421.40157-3-baohua@kernel.org> X-Mailer: git-send-email 2.39.3 (Apple Git-146) In-Reply-To: <20260901232421.40157-1-baohua@kernel.org> References: <20260901232421.40157-1-baohua@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Currently, folio_inc_gen() updates lrugen->nr_pages for every folio as it advances generations. Instead, accumulate the size changes and update lrugen->nr_pages in a batch after scanning the entire oldest generation, or when the scan stops because remaining reaches zero. Since we only move folios from the oldest generation to the second oldest generation, the active/inactive state cannot change. We can therefore skip __lru_update_size(). Signed-off-by: Barry Song (Xiaomi) Tested-by: Xueyuan Chen Reviewed-by: Lian Wang Reviewed-by: Kunwu Chan Reviewed-by: Baolin Wang --- mm/vmscan.c | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 54bce2f608ef..27494505cecc 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3918,6 +3918,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int ty= pe, int swappiness) struct lru_gen_folio *lrugen =3D &lruvec->lrugen; int hist =3D lru_hist_from_seq(lrugen->min_seq[type]); int new_gen, old_gen =3D lru_gen_from_seq(lrugen->min_seq[type]); + int target_gen =3D (old_gen + 1) % MAX_NR_GENS; =20 /* For file type, skip the check if swappiness is anon only */ if (type && (swappiness =3D=3D SWAPPINESS_ANON_ONLY)) @@ -3927,35 +3928,47 @@ static bool inc_min_seq(struct lruvec *lruvec, int = type, int swappiness) if (!type && !swappiness) goto done; =20 + VM_WARN_ON_ONCE(get_nr_gens(lruvec, type) !=3D MAX_NR_GENS); + VM_WARN_ON_ONCE(lru_gen_is_active(lruvec, old_gen) !=3D + lru_gen_is_active(lruvec, target_gen)); /* prevent cold/hot inversion if the type is evictable */ for (zone =3D 0; zone < MAX_NR_ZONES; zone++) { struct list_head *head =3D &lrugen->folios[old_gen][type][zone]; + long delta =3D 0; =20 while (!list_empty(head)) { struct folio *folio =3D lru_to_folio(head); + long nr_pages =3D folio_nr_pages(folio); int refs =3D folio_lru_refs(folio); bool workingset =3D folio_test_workingset(folio); + bool gen_increased; =20 VM_WARN_ON_ONCE_FOLIO(folio_test_unevictable(folio), folio); VM_WARN_ON_ONCE_FOLIO(folio_test_active(folio), folio); VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) !=3D type, folio); VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) !=3D zone, folio); =20 - new_gen =3D folio_inc_gen(lruvec, folio); + new_gen =3D __folio_inc_gen(folio, old_gen, &gen_increased); list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); - + if (gen_increased) + delta +=3D nr_pages; /* don't count the workingset being lazily promoted */ if (refs + workingset !=3D BIT(LRU_REFS_WIDTH) + 1) { int tier =3D lru_tier_from_refs(refs, workingset); - int delta =3D folio_nr_pages(folio); =20 WRITE_ONCE(lrugen->protected[hist][type][tier], - lrugen->protected[hist][type][tier] + delta); + lrugen->protected[hist][type][tier] + nr_pages); } =20 if (!--remaining) - return false; + break; } + WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], + lrugen->nr_pages[old_gen][type][zone] - delta); + WRITE_ONCE(lrugen->nr_pages[target_gen][type][zone], + lrugen->nr_pages[target_gen][type][zone] + delta); + if (!remaining) + return false; } done: reset_ctrl_pos(lruvec, type, true); --=20 2.34.1 From nobody Sat Sep 26 11:46:51 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 26FA646D541 for ; Tue, 1 Sep 2026 23:24:40 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788305082; cv=none; b=A+19bEMXRpJBQwCMJ+G60WKu9kUI1WQM6Zwj/yr0FcxFa5RJTsBhXadSbwXpZzFLFOH5qDTKBZMPZTXKgGzJWc1Lb7ZhuGgMogMlDabLwdRYZ7lUm5K90a+ylvDQNHQNDYsePNwd2LiZwuchn6O1UVwW1MYqkHGhmanv1vz2ThQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788305082; c=relaxed/simple; bh=FsPLwYxEQmT+DPaufObk4MU47CT8l7EHpXts93nyAy0=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=KKdhCQYZLHlom/iEVEK6oEwKF4wK1480ukjqQOB9VWtC7LF/uwZgZ5Nk9dlYg8ioP4UKbGE5c0xJOXkNkZ+dKOWzLCuxrIxqdJuYprKKHkBKrOTzpBLhkq2BOcT1SKTx6R/FDrOE99r3EGjF9QRYnYBrkUMJSu3i6aa95G20S3o= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=IdybSuZf; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="IdybSuZf" Received: by smtp.kernel.org (Postfix) with ESMTPSA id E2FCB1F00A3A; Tue, 1 Sep 2026 23:24:36 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1788305080; bh=UYsVp4S12kJxQZ9xleV3/HWxsgB9SxfAwwi/YHWmcuM=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=IdybSuZfHV9uKjVOOovxDICpVc/WEhejni2FnCnnPNQzYP8WPabII3oOraAPBAg2m Hi9nWpk0osyKzn8ysNfhM1PSdNcE7IqE5g5BCwV3dTFojCEvwMwpOG+MjrzqA6IWiu r61/wNV4mmw/dbwHpdlC3IWkevs360ET6min6myLHM9agWsf5yaSGa22nrfchdmbEm wgnhubNrL5OVHjsd/XNRj9fpjRykACMpCperraUl4q93M8zImsMCB0N4NyJRmXTHAl sjj1ST+7TR+AcXNzJWsc51J44tLbKP37O0omMiD8JtZmpeA9sIcD9NEk12ppAVA+gw rMmGZA5QppUVQ== From: "Barry Song (Xiaomi)" To: akpm@linux-foundation.org, linux-mm@kvack.org Cc: axelrasmussen@google.com, baolin.wang@linux.alibaba.com, baoquan.he@linux.dev, chenridong@xiaomi.com, david@kernel.org, hannes@cmpxchg.org, kasong@tencent.com, lianux.mm@gmail.com, linux-kernel@vger.kernel.org, ljs@kernel.org, lyugaofei@xiaomi.com, mhocko@kernel.org, qi.zheng@linux.dev, shakeel.butt@linux.dev, stevensd@chromium.org, wangzicheng@honor.com, weixugc@google.com, yuanchu@google.com, zhangbo56@xiaomi.com, "Barry Song (Xiaomi)" , Xueyuan Chen , Ridong Chen Subject: [PATCH v3 3/7] mm/mglru: enhance cold/hot inversion handling in inc_min_seq() Date: Wed, 2 Sep 2026 07:24:17 +0800 Message-Id: <20260901232421.40157-4-baohua@kernel.org> X-Mailer: git-send-email 2.39.3 (Apple Git-146) In-Reply-To: <20260901232421.40157-1-baohua@kernel.org> References: <20260901232421.40157-1-baohua@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" During aging, a folio's generation may already have been updated by folio_update_gen(), even though it has not yet been moved to the corresponding generation list. Such folios are hotter than those already in that generation. It makes sense for inc_min_seq() to increment the generation of folios that were never promoted during aging and move them to the tail of the new oldest generation. However, folios that were already promoted should instead be moved to the head of their updated generation, just as sort_folio() does in scan_folios(). Otherwise, promoted folios could end up behind folios that were never promoted, effectively inverting their hot/cold ordering. Signed-off-by: Barry Song (Xiaomi) Reviewed-by: Kairui Song Tested-by: Xueyuan Chen Reviewed-by: Ridong Chen Reviewed-by: Lian Wang Reviewed-by: Baolin Wang --- mm/vmscan.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 27494505cecc..95d48f9efc09 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3949,9 +3949,17 @@ static bool inc_min_seq(struct lruvec *lruvec, int t= ype, int swappiness) VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) !=3D zone, folio); =20 new_gen =3D __folio_inc_gen(folio, old_gen, &gen_increased); - list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); - if (gen_increased) + /* + * If gen_increased is false, this is a promotion. Put folios + * at the head of the promoted gen. Otherwise, put them at + * the tail of the second-oldest gen. + */ + if (gen_increased) { delta +=3D nr_pages; + list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); + } else { + list_move(&folio->lru, &lrugen->folios[new_gen][type][zone]); + } /* don't count the workingset being lazily promoted */ if (refs + workingset !=3D BIT(LRU_REFS_WIDTH) + 1) { int tier =3D lru_tier_from_refs(refs, workingset); --=20 2.34.1 From nobody Sat Sep 26 11:46:51 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7AC644734D3 for ; Tue, 1 Sep 2026 23:24:45 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788305086; cv=none; b=m4pckQQeiYFqyrt/uWFay2UhjahU6uLNVeVFElchjqfNE5ayly25wvPQdt0QOlp/sCl1MJSj3+JU9kNMxT1lDJrh8C7IgOEp9Fsd1+mz1/hrC7yKfH2wsosrUrrdlzzaWM5k9vKl4xHEQkTgJL2r/BMVaXLBKhcn5DlZ+kuZifA= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788305086; c=relaxed/simple; bh=ftgkjq6TXNX4FIULhVflBvdBaWOKXrO9O9lbOJaduH0=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=nLK24JaoWWjH+asBGBLZ6fc3l5b4rXI3Ig8zk5F85AJxhbgryJ+DfWzqlmfq7ykeoGY3YhIZzLwh6P+AIHUZQlN3MwegMk2DQcTDUFJB1I8anbsQrNyQ0keo+u8vC/WMXhPDf6SdW1/NbMPBa6LiRWhU/blm6VtKo4v4NCkxsuY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=AxdpiHeb; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="AxdpiHeb" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 3FDC31F00A3D; Tue, 1 Sep 2026 23:24:41 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1788305085; bh=PgOmxkUUjVNV1Eh5GMR0aIwGNf9xgs0cOpQXYVxNorE=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=AxdpiHeb7kJvNzMGM7D5vEhXi7aegklfIJNMxOX683Do1AQIZNYHwN6ILMk2lUsko cvujbCse6W3W8ZTMaEckADtZ1pGV2z3BmYmf6cqB9k+7oXeukTVEtTpS9X8Ofkjun4 qGnuB77ySAUsIEipMxEHjiJO8xwpPnjSm2BKrcwFxzwOr/nLkwe1KTFzIP0B++Iyva Yxk2e3ZHHRs4mVY4BWaaLD6VscDJJiHGyPAOTmCUpAeCWhOjaTsccmAnobtiZIBaEM 4RH3KZqA7379HOf0RhZ/+/hHi9X1HFovC5ei3N5Y8MZ5Jja2V8a9BfAaevKBNkLB32 ai2Lhi1XrkGrQ== From: "Barry Song (Xiaomi)" To: akpm@linux-foundation.org, linux-mm@kvack.org Cc: axelrasmussen@google.com, baolin.wang@linux.alibaba.com, baoquan.he@linux.dev, chenridong@xiaomi.com, david@kernel.org, hannes@cmpxchg.org, kasong@tencent.com, lianux.mm@gmail.com, linux-kernel@vger.kernel.org, ljs@kernel.org, lyugaofei@xiaomi.com, mhocko@kernel.org, qi.zheng@linux.dev, shakeel.butt@linux.dev, stevensd@chromium.org, wangzicheng@honor.com, weixugc@google.com, yuanchu@google.com, zhangbo56@xiaomi.com, "Barry Song (Xiaomi)" , Xueyuan Chen , Ridong Chen Subject: [PATCH v3 4/7] mm/mglru: exclude folios promoted by aging from protected in inc_min_seq() Date: Wed, 2 Sep 2026 07:24:18 +0800 Message-Id: <20260901232421.40157-5-baohua@kernel.org> X-Mailer: git-send-email 2.39.3 (Apple Git-146) In-Reply-To: <20260901232421.40157-1-baohua@kernel.org> References: <20260901232421.40157-1-baohua@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Some folios may have been promoted during aging, so don't count them as protected, similar to sort_folio(). Signed-off-by: Barry Song (Xiaomi) Reviewed-by: Baoquan He Tested-by: Xueyuan Chen Reviewed-by: Ridong Chen Reviewed-by: Lian Wang --- mm/vmscan.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 95d48f9efc09..ca57f6095b37 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3957,17 +3957,17 @@ static bool inc_min_seq(struct lruvec *lruvec, int = type, int swappiness) if (gen_increased) { delta +=3D nr_pages; list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); + + /* don't count the workingset being lazily promoted */ + if (refs + workingset !=3D BIT(LRU_REFS_WIDTH) + 1) { + int tier =3D lru_tier_from_refs(refs, workingset); + + WRITE_ONCE(lrugen->protected[hist][type][tier], + lrugen->protected[hist][type][tier] + nr_pages); + } } else { list_move(&folio->lru, &lrugen->folios[new_gen][type][zone]); } - /* don't count the workingset being lazily promoted */ - if (refs + workingset !=3D BIT(LRU_REFS_WIDTH) + 1) { - int tier =3D lru_tier_from_refs(refs, workingset); - - WRITE_ONCE(lrugen->protected[hist][type][tier], - lrugen->protected[hist][type][tier] + nr_pages); - } - if (!--remaining) break; } --=20 2.34.1 From nobody Sat Sep 26 11:46:51 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 7656A4734D3 for ; Tue, 1 Sep 2026 23:24:49 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788305090; cv=none; b=M39iBZmgdthGZ3L18FEaxlthp/RY2bTRA8aAZuWbfoA+8eU5VRC1qwRR62Ad2YAJLcRF3GlBhfoAuOIfnsBYbzAATb5pEf4jEpbVy9bALcaQaW1zZQOflt6ckrsQ2UB7YdRJxRN2tDJNQYanIkL3m3dBV1IrYYdAKN9dE7cD44w= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788305090; c=relaxed/simple; bh=oGBCxZ/W1ePKeKfKC+WTKK7+1MAv4ztkQG0NEUx6DE0=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=NeA7fVbNCLsKh4E/nNh5LbSMOo4PJns2peXBjncXM7CD3OvSCOvmM9Ckx+IK8MS7wAt/bILrh6EMEW8QqckP6ld1vZaRXod2tL+iH4PUeP3F2eRXRtcqx/ph/NX7Y6CvUl9+RmhlnkyCG524sFCASHFnUofC29h6S3K93rZD8UM= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=ibSekCGB; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="ibSekCGB" Received: by smtp.kernel.org (Postfix) with ESMTPSA id 90F941F000E9; Tue, 1 Sep 2026 23:24:45 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1788305089; bh=dD8NdLX7k458iQXtwaaXdMdVjxLHPxreylsAIMI2VEQ=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=ibSekCGBXiuyZPZB+TrUcrTvmAVz4J6blUZQSkuuhDcLDR9ixnqf+520nUiUdNc6T /cpaWjlXga/eXeMvWhdiJdjIq5EufHp0OkMW012RFkqkkeR+Lr7Z+U8P3LKYiPRRyR dOBRbhWd8KtLI007ns+Kd4ST8tbyD0yo/O5kzHGxZGFpRpOga/W8TMY/QblqcaC8Ny mchLUVU7GQlymWA6adU+ZBXG1OEHfMZU6ex7wEs3LpwhePv5+aAmVavwglaYKnv+d2 HAf4VC18aG9u8u4oldXleMhgNffeppkJBDl3iii0bdGyOBwLfUkW25gCsqXguV4Y3l Ar+vE10FgAfpg== From: "Barry Song (Xiaomi)" To: akpm@linux-foundation.org, linux-mm@kvack.org Cc: axelrasmussen@google.com, baolin.wang@linux.alibaba.com, baoquan.he@linux.dev, chenridong@xiaomi.com, david@kernel.org, hannes@cmpxchg.org, kasong@tencent.com, lianux.mm@gmail.com, linux-kernel@vger.kernel.org, ljs@kernel.org, lyugaofei@xiaomi.com, mhocko@kernel.org, qi.zheng@linux.dev, shakeel.butt@linux.dev, stevensd@chromium.org, wangzicheng@honor.com, weixugc@google.com, yuanchu@google.com, zhangbo56@xiaomi.com, "Barry Song (Xiaomi)" , Xueyuan Chen Subject: [PATCH v3 5/7] mm/mglru: make LRU folio prefetch helper an inline function Date: Wed, 2 Sep 2026 07:24:19 +0800 Message-Id: <20260901232421.40157-6-baohua@kernel.org> X-Mailer: git-send-email 2.39.3 (Apple Git-146) In-Reply-To: <20260901232421.40157-1-baohua@kernel.org> References: <20260901232421.40157-1-baohua@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" `prefetchw_prev_lru_folio()` is currently implemented as a macro with a potentially unused argument. This makes the helper harder to read and can also trigger checkpatch warnings about unused macro arguments. Make it a `static inline` function and remove the unnecessary `_field` argument. The helper always prefetches the previous folio's `flags`, so there is no need to make the field configurable. Signed-off-by: Barry Song (Xiaomi) Tested-by: Xueyuan Chen Reviewed-by: Lian Wang Reviewed-by: Baolin Wang --- mm/vmscan.c | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index ca57f6095b37..1907a946840d 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -182,17 +182,21 @@ struct scan_control { }; =20 #ifdef ARCH_HAS_PREFETCHW -#define prefetchw_prev_lru_folio(_folio, _base, _field) \ - do { \ - if ((_folio)->lru.prev !=3D _base) { \ - struct folio *prev; \ - \ - prev =3D lru_to_folio(&(_folio->lru)); \ - prefetchw(&prev->_field); \ - } \ - } while (0) +static inline void prefetchw_prev_lru_folio(struct folio *folio, + struct list_head *base) +{ + if (folio->lru.prev !=3D base) { + struct folio *prev; + + prev =3D lru_to_folio(&folio->lru); + prefetchw(&prev->flags); + } +} #else -#define prefetchw_prev_lru_folio(_folio, _base, _field) do { } while (0) +static inline void prefetchw_prev_lru_folio(struct folio *folio, + struct list_head *base) +{ +} #endif =20 /* @@ -1695,7 +1699,7 @@ static unsigned long isolate_lru_folios(unsigned long= nr_to_scan, struct folio *folio; =20 folio =3D lru_to_folio(src); - prefetchw_prev_lru_folio(folio, src, flags); + prefetchw_prev_lru_folio(folio, src); =20 nr_pages =3D folio_nr_pages(folio); total_scan +=3D nr_pages; --=20 2.34.1 From nobody Sat Sep 26 11:46:51 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id A0F4F46AA68 for ; Tue, 1 Sep 2026 23:24:53 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788305094; cv=none; b=duP2j3YIrUtnS1erRsC5+do1JIYMkBqQOFHDmyP7B4MDx/2kWsg6CauH6qNn0f4eDc2PDXSKVcevI514LkA20Qww8dvEQJoMxbdUSOdYeGMeYcQ3Js+sU7rlxnJZjTtHI7vzyZ+gA9feRlkoq7jP5mxpJdmWh+83jNRCPFpWaAg= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788305094; c=relaxed/simple; bh=eudb/UeG4VBqSbJ/WswZzZvM69bQ+YvvRyHz19z5I8M=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=cOSF5WBRhVZBCVKclbvncLOUo/wuk/dSRu3RV8z4u9vZrKxIsrhs0caxWaGwN/0V/aevzKQVWQ3LKOVuZktx7R7rqoys5qyKxZpY0v6vhlc8j+64PRDRqtAhK3XkjhWpEJFhvNmBbbL2+NL09TK/NQ/X9rm0pPbXxhRFTAlEt10= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=VvGWg+Kr; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="VvGWg+Kr" Received: by smtp.kernel.org (Postfix) with ESMTPSA id BCF101F00A3A; Tue, 1 Sep 2026 23:24:49 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1788305093; bh=rFw5zgkGnAPMxb67e0R/o6CVVTXZaTWLLe5xe6qm9Ko=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=VvGWg+Kr1z5qUex3V5KxKtgW0/4PU1vYOVO5+DIRjj24ddVqGhl+RYfdUxxHg+cqG EApYK3aeTVVouHEmKSMSC0eWou5bakMYTzeWh+g0ef6Z8ugDcA0mjzvy1iYtQ3T+Pl uot1sQMFB01SqK3M5lURJJ/hRhJdKGvsFhcc7rsRDeACQMgz2mCwOwbEFX1L7C7zv0 zBSICyyYlu9rcCLwYr40KPXuKB7YNW34TqGG1XPVqOWav2lIe6vNck5ZEEorrTrAYr XPFnyvdwp/cXodcjxVG4G1pZFIkkLOEBICc+UaRVEAopyOZFFm7LbyLjAqGhGa929s 11w/MTT2VlsZA== From: "Barry Song (Xiaomi)" To: akpm@linux-foundation.org, linux-mm@kvack.org Cc: axelrasmussen@google.com, baolin.wang@linux.alibaba.com, baoquan.he@linux.dev, chenridong@xiaomi.com, david@kernel.org, hannes@cmpxchg.org, kasong@tencent.com, lianux.mm@gmail.com, linux-kernel@vger.kernel.org, ljs@kernel.org, lyugaofei@xiaomi.com, mhocko@kernel.org, qi.zheng@linux.dev, shakeel.butt@linux.dev, stevensd@chromium.org, wangzicheng@honor.com, weixugc@google.com, yuanchu@google.com, zhangbo56@xiaomi.com, "Barry Song (Xiaomi)" , Xueyuan Chen Subject: [PATCH v3 6/7] mm/mglru: move folios from oldest gen to second-oldest gen from head to tail Date: Wed, 2 Sep 2026 07:24:20 +0800 Message-Id: <20260901232421.40157-7-baohua@kernel.org> X-Mailer: git-send-email 2.39.3 (Apple Git-146) In-Reply-To: <20260901232421.40157-1-baohua@kernel.org> References: <20260901232421.40157-1-baohua@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" For reclamation, it makes sense to reclaim folios from tail to head, as folios near the head are relatively hot. However, when moving folios from the oldest generation to the second-oldest generation, using the tail-to-head order would effectively cause a cold/hot inversion. Signed-off-by: Barry Song (Xiaomi) Reviewed-by: Baoquan He Tested-by: Xueyuan Chen Reviewed-by: Lian Wang --- mm/vmscan.c | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 1907a946840d..76dfa9575852 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -192,11 +192,27 @@ static inline void prefetchw_prev_lru_folio(struct fo= lio *folio, prefetchw(&prev->flags); } } + +static inline void prefetchw_next_lru_folio(struct folio *folio, + struct list_head *base) +{ + if (folio->lru.next !=3D base) { + struct folio *next; + + next =3D list_entry(folio->lru.next, struct folio, lru); + prefetchw(&next->flags); + } +} #else static inline void prefetchw_prev_lru_folio(struct folio *folio, struct list_head *base) { } + +static inline void prefetchw_next_lru_folio(struct folio *folio, + struct list_head *base) +{ +} #endif =20 /* @@ -3938,10 +3954,11 @@ static bool inc_min_seq(struct lruvec *lruvec, int = type, int swappiness) /* prevent cold/hot inversion if the type is evictable */ for (zone =3D 0; zone < MAX_NR_ZONES; zone++) { struct list_head *head =3D &lrugen->folios[old_gen][type][zone]; + struct list_head *pos =3D head->next; long delta =3D 0; =20 - while (!list_empty(head)) { - struct folio *folio =3D lru_to_folio(head); + while (pos !=3D head) { + struct folio *folio =3D list_entry(pos, struct folio, lru); long nr_pages =3D folio_nr_pages(folio); int refs =3D folio_lru_refs(folio); bool workingset =3D folio_test_workingset(folio); @@ -3952,6 +3969,8 @@ static bool inc_min_seq(struct lruvec *lruvec, int ty= pe, int swappiness) VM_WARN_ON_ONCE_FOLIO(folio_is_file_lru(folio) !=3D type, folio); VM_WARN_ON_ONCE_FOLIO(folio_zonenum(folio) !=3D zone, folio); =20 + prefetchw_next_lru_folio(folio, head); + pos =3D pos->next; new_gen =3D __folio_inc_gen(folio, old_gen, &gen_increased); /* * If gen_increased is false, this is a promotion. Put folios --=20 2.34.1 From nobody Sat Sep 26 11:46:51 2026 Received: from smtp.kernel.org (aws-us-west-2-korg-mail-alma10-1.taild15c8.ts.net [100.103.45.18]) (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) (No client certificate requested) by smtp.subspace.kernel.org (Postfix) with ESMTPS id 1BB2330C617 for ; Tue, 1 Sep 2026 23:24:57 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=100.103.45.18 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788305099; cv=none; b=iUKkrbK1v6RipKAK9iOnGKkooMqJsKlboRIKHDap5iz4C0Erglbk94gsZJ13wNgLFOE6dAgMlWCAW37jwBmAIg6XOF46V3Vdmfdr3K15RLOQdV4AxpYQ9hu0ohNx4BErIjiqmhq2/hmZyHsN8uKFi+JWmKLdO7pxVl6ZuZrQBDU= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1788305099; c=relaxed/simple; bh=5ClfFsrwU1Rz8dszqqRa6GmGLdKTuqyTjhmr7GJo+Tw=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=D6COZiJxaMLzEi8jh+PA5m6R6XutiNnwzO7pT9WRYU+7PV4KeEgG79taxIIspjYQ8SttbenZWIzfj0Xu0hYZhqg2+L3oVh1fmqrbse0q3GqgQdGNwmQdQKqHGKy0NAuBYllwXOwo4VRXQJpsd9TOAPs2IQEqVcdxB2pOHR7U1tY= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b=UgcS4yZ0; arc=none smtp.client-ip=100.103.45.18 Authentication-Results: smtp.subspace.kernel.org; dkim=pass (2048-bit key) header.d=kernel.org header.i=@kernel.org header.b="UgcS4yZ0" Received: by smtp.kernel.org (Postfix) with ESMTPSA id E8E0D1F000E9; Tue, 1 Sep 2026 23:24:53 +0000 (UTC) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=kernel.org; s=k20260515; t=1788305097; bh=jk2RFe/XVKJiusQmuo8ChyovasaBl76wwu4zIL3Vsv4=; h=From:To:Cc:Subject:Date:In-Reply-To:References; b=UgcS4yZ06pssXbFHHgG3+Ba+Bbr3x3lwd8ymNYfhY8aNnW+fzhwjTE19IIdLf3Sbt /m4VP+4uYyt8LOV1zRCYv/cYeVi4Vek0sUaMdYHVc2IlCjkRp3vS1Cjgyau25mxUY8 bwh7uq2MFhTrAQR+r+Zj0aO6vgdKbxMPBMc3qERLkgpyUZJytfp07A4i95Il+ODhqe CM3yE2H6QZVkEGDb25z8NtTRBm9YdvWbnecC2zBCWXX7LzNZtlXn+71xRLFJPMIs1g 9RK51c7UNMO8stCGfu1jVeMzO9TY8zHr4C1miE8AZT/dVutp1osI9/cTZheaIrSSrj tbhlffqX8dzVw== From: "Barry Song (Xiaomi)" To: akpm@linux-foundation.org, linux-mm@kvack.org Cc: axelrasmussen@google.com, baolin.wang@linux.alibaba.com, baoquan.he@linux.dev, chenridong@xiaomi.com, david@kernel.org, hannes@cmpxchg.org, kasong@tencent.com, lianux.mm@gmail.com, linux-kernel@vger.kernel.org, ljs@kernel.org, lyugaofei@xiaomi.com, mhocko@kernel.org, qi.zheng@linux.dev, shakeel.butt@linux.dev, stevensd@chromium.org, wangzicheng@honor.com, weixugc@google.com, yuanchu@google.com, zhangbo56@xiaomi.com, "Barry Song (Xiaomi)" , Xueyuan Chen Subject: [PATCH v3 7/7] mm/mglru: batch move folios to the second-oldest gen's LRU Date: Wed, 2 Sep 2026 07:24:21 +0800 Message-Id: <20260901232421.40157-8-baohua@kernel.org> X-Mailer: git-send-email 2.39.3 (Apple Git-146) In-Reply-To: <20260901232421.40157-1-baohua@kernel.org> References: <20260901232421.40157-1-baohua@kernel.org> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Detect folios that need to move from the oldest generation to the second-oldest generation, and batch-move them together. This can significantly reduce the sys time of inc_min_seq(), especially when the other type is significantly behind the preferred type. Assisted-by: gemini:gemini-3.6-flash Signed-off-by: Barry Song (Xiaomi) Reviewed-by: Baoquan He Tested-by: Xueyuan Chen Reviewed-by: Lian Wang Reviewed-by: Baolin Wang --- mm/vmscan.c | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/mm/vmscan.c b/mm/vmscan.c index 76dfa9575852..79defbb44c6e 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -3931,6 +3931,19 @@ static void clear_mm_walk(void) kfree(walk); } =20 +static inline void flush_lru_batch(struct list_head *head, struct list_hea= d **batch_end, + struct list_head *dst) +{ + LIST_HEAD(movable); + + if (!*batch_end) + return; + + list_cut_position(&movable, head, *batch_end); + list_splice_tail_init(&movable, dst); + *batch_end =3D NULL; +} + static bool inc_min_seq(struct lruvec *lruvec, int type, int swappiness) { int zone; @@ -3953,8 +3966,10 @@ static bool inc_min_seq(struct lruvec *lruvec, int t= ype, int swappiness) lru_gen_is_active(lruvec, target_gen)); /* prevent cold/hot inversion if the type is evictable */ for (zone =3D 0; zone < MAX_NR_ZONES; zone++) { + struct list_head *target_list =3D &lrugen->folios[target_gen][type][zone= ]; struct list_head *head =3D &lrugen->folios[old_gen][type][zone]; struct list_head *pos =3D head->next; + struct list_head *batch_end =3D NULL; long delta =3D 0; =20 while (pos !=3D head) { @@ -3979,7 +3994,7 @@ static bool inc_min_seq(struct lruvec *lruvec, int ty= pe, int swappiness) */ if (gen_increased) { delta +=3D nr_pages; - list_move_tail(&folio->lru, &lrugen->folios[new_gen][type][zone]); + batch_end =3D &folio->lru; =20 /* don't count the workingset being lazily promoted */ if (refs + workingset !=3D BIT(LRU_REFS_WIDTH) + 1) { @@ -3989,11 +4004,14 @@ static bool inc_min_seq(struct lruvec *lruvec, int = type, int swappiness) lrugen->protected[hist][type][tier] + nr_pages); } } else { + flush_lru_batch(head, &batch_end, target_list); list_move(&folio->lru, &lrugen->folios[new_gen][type][zone]); } if (!--remaining) break; } + flush_lru_batch(head, &batch_end, target_list); + WRITE_ONCE(lrugen->nr_pages[old_gen][type][zone], lrugen->nr_pages[old_gen][type][zone] - delta); WRITE_ONCE(lrugen->nr_pages[target_gen][type][zone], --=20 2.34.1