remap(), folio_ref_unfreeze(), lru_add_split_folio() are not relevant to
splitting unmapped folio operations. Move them out to __folio_split() so
that __split_unmapped_folio() only handles unmapped folio splits. This
makes __split_unmapped_folio() reusable.
Remove the swapcache folio split check code before __split_unmapped_folio()
call, since it is already checked at the beginning of __folio_split() in
uniform_split_supported() and non_uniform_split_supported().
Along with the code move, there are some variable renames:
1. release is renamed to new_folio,
2. origin_folio is now folio, since __folio_split() has folio pointing to
the original folio already.
Signed-off-by: Zi Yan <ziy@nvidia.com>
---
mm/huge_memory.c | 270 +++++++++++++++++++++++------------------------
1 file changed, 133 insertions(+), 137 deletions(-)
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index ce130225a8e5..63eebca07628 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -3385,10 +3385,6 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
* order - 1 to new_order).
* @split_at: in buddy allocator like split, the folio containing @split_at
* will be split until its order becomes @new_order.
- * @lock_at: the folio containing @lock_at is left locked for caller.
- * @list: the after split folios will be added to @list if it is not NULL,
- * otherwise to LRU lists.
- * @end: the end of the file @folio maps to. -1 if @folio is anonymous memory.
* @xas: xa_state pointing to folio->mapping->i_pages and locked by caller
* @mapping: @folio->mapping
* @uniform_split: if the split is uniform or not (buddy allocator like split)
@@ -3414,52 +3410,26 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
* @page, which is split in next for loop.
*
* After splitting, the caller's folio reference will be transferred to the
- * folio containing @page. The other folios may be freed if they are not mapped.
- *
- * In terms of locking, after splitting,
- * 1. uniform split leaves @page (or the folio contains it) locked;
- * 2. buddy allocator like (non-uniform) split leaves @folio locked.
- *
+ * folio containing @page. The caller needs to unlock and/or free after-split
+ * folios if necessary.
*
* For !uniform_split, when -ENOMEM is returned, the original folio might be
* split. The caller needs to check the input folio.
*/
static int __split_unmapped_folio(struct folio *folio, int new_order,
- struct page *split_at, struct page *lock_at,
- struct list_head *list, pgoff_t end,
- struct xa_state *xas, struct address_space *mapping,
- bool uniform_split)
+ struct page *split_at, struct xa_state *xas,
+ struct address_space *mapping, bool uniform_split)
{
- struct lruvec *lruvec;
- struct address_space *swap_cache = NULL;
- struct folio *origin_folio = folio;
- struct folio *next_folio = folio_next(folio);
- struct folio *new_folio;
- struct folio *next;
int order = folio_order(folio);
- int split_order;
int start_order = uniform_split ? new_order : order - 1;
- int nr_dropped = 0;
- int ret = 0;
bool stop_split = false;
-
- if (folio_test_swapcache(folio)) {
- VM_BUG_ON(mapping);
-
- /* a swapcache folio can only be uniformly split to order-0 */
- if (!uniform_split || new_order != 0)
- return -EINVAL;
-
- swap_cache = swap_address_space(folio->swap);
- xa_lock(&swap_cache->i_pages);
- }
+ struct folio *next;
+ int split_order;
+ int ret = 0;
if (folio_test_anon(folio))
mod_mthp_stat(order, MTHP_STAT_NR_ANON, -1);
- /* lock lru list/PageCompound, ref frozen by page_ref_freeze */
- lruvec = folio_lruvec_lock(folio);
-
folio_clear_has_hwpoisoned(folio);
/*
@@ -3469,9 +3439,9 @@ static int __split_unmapped_folio(struct folio *folio, int new_order,
for (split_order = start_order;
split_order >= new_order && !stop_split;
split_order--) {
- int old_order = folio_order(folio);
- struct folio *release;
struct folio *end_folio = folio_next(folio);
+ int old_order = folio_order(folio);
+ struct folio *new_folio;
/* order-1 anonymous folio is not supported */
if (folio_test_anon(folio) && split_order == 1)
@@ -3506,113 +3476,32 @@ static int __split_unmapped_folio(struct folio *folio, int new_order,
after_split:
/*
- * Iterate through after-split folios and perform related
- * operations. But in buddy allocator like split, the folio
+ * Iterate through after-split folios and update folio stats.
+ * But in buddy allocator like split, the folio
* containing the specified page is skipped until its order
* is new_order, since the folio will be worked on in next
* iteration.
*/
- for (release = folio; release != end_folio; release = next) {
- next = folio_next(release);
+ for (new_folio = folio; new_folio != end_folio; new_folio = next) {
+ next = folio_next(new_folio);
/*
- * for buddy allocator like split, the folio containing
- * page will be split next and should not be released,
- * until the folio's order is new_order or stop_split
- * is set to true by the above xas_split() failure.
+ * for buddy allocator like split, new_folio containing
+ * @split_at page could be split again, thus do not
+ * change stats yet. Wait until new_folio's order is
+ * @new_order or stop_split is set to true by the above
+ * xas_split() failure.
*/
- if (release == page_folio(split_at)) {
- folio = release;
+ if (new_folio == page_folio(split_at)) {
+ folio = new_folio;
if (split_order != new_order && !stop_split)
continue;
}
- if (folio_test_anon(release)) {
- mod_mthp_stat(folio_order(release),
- MTHP_STAT_NR_ANON, 1);
- }
-
- /*
- * origin_folio should be kept frozon until page cache
- * entries are updated with all the other after-split
- * folios to prevent others seeing stale page cache
- * entries.
- */
- if (release == origin_folio)
- continue;
-
- folio_ref_unfreeze(release, 1 +
- ((mapping || swap_cache) ?
- folio_nr_pages(release) : 0));
-
- lru_add_split_folio(origin_folio, release, lruvec,
- list);
-
- /* Some pages can be beyond EOF: drop them from cache */
- if (release->index >= end) {
- if (shmem_mapping(mapping))
- nr_dropped += folio_nr_pages(release);
- else if (folio_test_clear_dirty(release))
- folio_account_cleaned(release,
- inode_to_wb(mapping->host));
- __filemap_remove_folio(release, NULL);
- folio_put_refs(release, folio_nr_pages(release));
- } else if (mapping) {
- __xa_store(&mapping->i_pages,
- release->index, release, 0);
- } else if (swap_cache) {
- __xa_store(&swap_cache->i_pages,
- swap_cache_index(release->swap),
- release, 0);
- }
+ if (folio_test_anon(new_folio))
+ mod_mthp_stat(folio_order(new_folio),
+ MTHP_STAT_NR_ANON, 1);
}
}
- /*
- * Unfreeze origin_folio only after all page cache entries, which used
- * to point to it, have been updated with new folios. Otherwise,
- * a parallel folio_try_get() can grab origin_folio and its caller can
- * see stale page cache entries.
- */
- folio_ref_unfreeze(origin_folio, 1 +
- ((mapping || swap_cache) ? folio_nr_pages(origin_folio) : 0));
-
- unlock_page_lruvec(lruvec);
-
- if (swap_cache)
- xa_unlock(&swap_cache->i_pages);
- if (mapping)
- xa_unlock(&mapping->i_pages);
-
- /* Caller disabled irqs, so they are still disabled here */
- local_irq_enable();
-
- if (nr_dropped)
- shmem_uncharge(mapping->host, nr_dropped);
-
- remap_page(origin_folio, 1 << order,
- folio_test_anon(origin_folio) ?
- RMP_USE_SHARED_ZEROPAGE : 0);
-
- /*
- * At this point, folio should contain the specified page.
- * For uniform split, it is left for caller to unlock.
- * For buddy allocator like split, the first after-split folio is left
- * for caller to unlock.
- */
- for (new_folio = origin_folio; new_folio != next_folio; new_folio = next) {
- next = folio_next(new_folio);
- if (new_folio == page_folio(lock_at))
- continue;
-
- folio_unlock(new_folio);
- /*
- * Subpages may be freed if there wasn't any mapping
- * like if add_to_swap() is running on a lru page that
- * had its mapping zapped. And freeing these pages
- * requires taking the lru_lock so we do the put_page
- * of the tail pages after the split is complete.
- */
- free_folio_and_swap_cache(new_folio);
- }
return ret;
}
@@ -3686,6 +3575,11 @@ bool uniform_split_supported(struct folio *folio, unsigned int new_order,
* It is in charge of checking whether the split is supported or not and
* preparing @folio for __split_unmapped_folio().
*
+ * After splitting, the after-split folio containing @lock_at remains locked
+ * and others are unlocked:
+ * 1. for uniform split, @lock_at points to one of @folio's subpages;
+ * 2. for buddy allocator like (non-uniform) split, @lock_at points to @folio.
+ *
* return: 0: successful, <0 failed (if -ENOMEM is returned, @folio might be
* split but not to @new_order, the caller needs to check)
*/
@@ -3695,10 +3589,12 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
{
struct deferred_split *ds_queue = get_deferred_split_queue(folio);
XA_STATE(xas, &folio->mapping->i_pages, folio->index);
+ struct folio *end_folio = folio_next(folio);
bool is_anon = folio_test_anon(folio);
struct address_space *mapping = NULL;
struct anon_vma *anon_vma = NULL;
int order = folio_order(folio);
+ struct folio *new_folio, *next;
int extra_pins, ret;
pgoff_t end;
bool is_hzp;
@@ -3829,6 +3725,10 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
/* Prevent deferred_split_scan() touching ->_refcount */
spin_lock(&ds_queue->split_queue_lock);
if (folio_ref_freeze(folio, 1 + extra_pins)) {
+ struct address_space *swap_cache = NULL;
+ int nr_dropped = 0;
+ struct lruvec *lruvec;
+
if (folio_order(folio) > 1 &&
!list_empty(&folio->_deferred_list)) {
ds_queue->split_queue_len--;
@@ -3862,9 +3762,105 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
}
}
- ret = __split_unmapped_folio(folio, new_order,
- split_at, lock_at, list, end, &xas, mapping,
- uniform_split);
+ if (folio_test_swapcache(folio)) {
+ VM_BUG_ON(mapping);
+
+ swap_cache = swap_address_space(folio->swap);
+ xa_lock(&swap_cache->i_pages);
+ }
+
+ /* lock lru list/PageCompound, ref frozen by page_ref_freeze */
+ lruvec = folio_lruvec_lock(folio);
+
+ ret = __split_unmapped_folio(folio, new_order, split_at, &xas,
+ mapping, uniform_split);
+
+ /*
+ * Unfreeze after-split folios and put them back to the right
+ * list. @folio should be kept frozon until page cache
+ * entries are updated with all the other after-split folios
+ * to prevent others seeing stale page cache entries.
+ * As a result, new_folio starts from the next folio of
+ * @folio.
+ */
+ for (new_folio = folio_next(folio); new_folio != end_folio;
+ new_folio = next) {
+ next = folio_next(new_folio);
+
+ folio_ref_unfreeze(
+ new_folio,
+ 1 + ((mapping || swap_cache) ?
+ folio_nr_pages(new_folio) :
+ 0));
+
+ lru_add_split_folio(folio, new_folio, lruvec, list);
+
+ /* Some pages can be beyond EOF: drop them from cache */
+ if (new_folio->index >= end) {
+ if (shmem_mapping(mapping))
+ nr_dropped += folio_nr_pages(new_folio);
+ else if (folio_test_clear_dirty(new_folio))
+ folio_account_cleaned(
+ new_folio,
+ inode_to_wb(mapping->host));
+ __filemap_remove_folio(new_folio, NULL);
+ folio_put_refs(new_folio,
+ folio_nr_pages(new_folio));
+ } else if (mapping) {
+ __xa_store(&mapping->i_pages, new_folio->index,
+ new_folio, 0);
+ } else if (swap_cache) {
+ __xa_store(&swap_cache->i_pages,
+ swap_cache_index(new_folio->swap),
+ new_folio, 0);
+ }
+ }
+ /*
+ * Unfreeze @folio only after all page cache entries, which
+ * used to point to it, have been updated with new folios.
+ * Otherwise, a parallel folio_try_get() can grab @folio
+ * and its caller can see stale page cache entries.
+ */
+ folio_ref_unfreeze(folio, 1 +
+ ((mapping || swap_cache) ? folio_nr_pages(folio) : 0));
+
+ unlock_page_lruvec(lruvec);
+
+ if (swap_cache)
+ xa_unlock(&swap_cache->i_pages);
+ if (mapping)
+ xas_unlock(&xas);
+
+ local_irq_enable();
+
+ if (nr_dropped)
+ shmem_uncharge(mapping->host, nr_dropped);
+
+ remap_page(folio, 1 << order,
+ !ret && folio_test_anon(folio) ?
+ RMP_USE_SHARED_ZEROPAGE :
+ 0);
+
+ /*
+ * Unlock all after-split folios except the one containing
+ * @lock_at page. If @folio is not split, it will be kept locked.
+ */
+ for (new_folio = folio; new_folio != end_folio;
+ new_folio = next) {
+ next = folio_next(new_folio);
+ if (new_folio == page_folio(lock_at))
+ continue;
+
+ folio_unlock(new_folio);
+ /*
+ * Subpages may be freed if there wasn't any mapping
+ * like if add_to_swap() is running on a lru page that
+ * had its mapping zapped. And freeing these pages
+ * requires taking the lru_lock so we do the put_page
+ * of the tail pages after the split is complete.
+ */
+ free_folio_and_swap_cache(new_folio);
+ }
} else {
spin_unlock(&ds_queue->split_queue_lock);
fail:
--
2.47.2On Thu, Jul 17, 2025 at 10:29:55PM -0400, Zi Yan wrote:
> remap(), folio_ref_unfreeze(), lru_add_split_folio() are not relevant to
> splitting unmapped folio operations. Move them out to __folio_split() so
> that __split_unmapped_folio() only handles unmapped folio splits. This
> makes __split_unmapped_folio() reusable.
>
> Remove the swapcache folio split check code before __split_unmapped_folio()
> call, since it is already checked at the beginning of __folio_split() in
> uniform_split_supported() and non_uniform_split_supported().
>
> Along with the code move, there are some variable renames:
>
> 1. release is renamed to new_folio,
> 2. origin_folio is now folio, since __folio_split() has folio pointing to
> the original folio already.
>
> Signed-off-by: Zi Yan <ziy@nvidia.com>
OK, have had a careful look through and LGTM. Thanks very much for address
prior review! So:
Reviewed-by: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
> ---
> mm/huge_memory.c | 270 +++++++++++++++++++++++------------------------
> 1 file changed, 133 insertions(+), 137 deletions(-)
>
> diff --git a/mm/huge_memory.c b/mm/huge_memory.c
> index ce130225a8e5..63eebca07628 100644
> --- a/mm/huge_memory.c
> +++ b/mm/huge_memory.c
> @@ -3385,10 +3385,6 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
> * order - 1 to new_order).
> * @split_at: in buddy allocator like split, the folio containing @split_at
> * will be split until its order becomes @new_order.
> - * @lock_at: the folio containing @lock_at is left locked for caller.
> - * @list: the after split folios will be added to @list if it is not NULL,
> - * otherwise to LRU lists.
> - * @end: the end of the file @folio maps to. -1 if @folio is anonymous memory.
> * @xas: xa_state pointing to folio->mapping->i_pages and locked by caller
> * @mapping: @folio->mapping
> * @uniform_split: if the split is uniform or not (buddy allocator like split)
> @@ -3414,52 +3410,26 @@ static void __split_folio_to_order(struct folio *folio, int old_order,
> * @page, which is split in next for loop.
> *
> * After splitting, the caller's folio reference will be transferred to the
> - * folio containing @page. The other folios may be freed if they are not mapped.
> - *
> - * In terms of locking, after splitting,
> - * 1. uniform split leaves @page (or the folio contains it) locked;
> - * 2. buddy allocator like (non-uniform) split leaves @folio locked.
> - *
> + * folio containing @page. The caller needs to unlock and/or free after-split
> + * folios if necessary.
> *
> * For !uniform_split, when -ENOMEM is returned, the original folio might be
> * split. The caller needs to check the input folio.
> */
> static int __split_unmapped_folio(struct folio *folio, int new_order,
> - struct page *split_at, struct page *lock_at,
> - struct list_head *list, pgoff_t end,
> - struct xa_state *xas, struct address_space *mapping,
> - bool uniform_split)
> + struct page *split_at, struct xa_state *xas,
> + struct address_space *mapping, bool uniform_split)
> {
> - struct lruvec *lruvec;
> - struct address_space *swap_cache = NULL;
> - struct folio *origin_folio = folio;
> - struct folio *next_folio = folio_next(folio);
> - struct folio *new_folio;
> - struct folio *next;
> int order = folio_order(folio);
> - int split_order;
> int start_order = uniform_split ? new_order : order - 1;
> - int nr_dropped = 0;
> - int ret = 0;
> bool stop_split = false;
> -
> - if (folio_test_swapcache(folio)) {
> - VM_BUG_ON(mapping);
> -
> - /* a swapcache folio can only be uniformly split to order-0 */
> - if (!uniform_split || new_order != 0)
> - return -EINVAL;
> -
> - swap_cache = swap_address_space(folio->swap);
> - xa_lock(&swap_cache->i_pages);
> - }
> + struct folio *next;
> + int split_order;
> + int ret = 0;
>
> if (folio_test_anon(folio))
> mod_mthp_stat(order, MTHP_STAT_NR_ANON, -1);
>
> - /* lock lru list/PageCompound, ref frozen by page_ref_freeze */
> - lruvec = folio_lruvec_lock(folio);
> -
> folio_clear_has_hwpoisoned(folio);
>
> /*
> @@ -3469,9 +3439,9 @@ static int __split_unmapped_folio(struct folio *folio, int new_order,
> for (split_order = start_order;
> split_order >= new_order && !stop_split;
> split_order--) {
> - int old_order = folio_order(folio);
> - struct folio *release;
> struct folio *end_folio = folio_next(folio);
> + int old_order = folio_order(folio);
> + struct folio *new_folio;
>
> /* order-1 anonymous folio is not supported */
> if (folio_test_anon(folio) && split_order == 1)
> @@ -3506,113 +3476,32 @@ static int __split_unmapped_folio(struct folio *folio, int new_order,
>
> after_split:
> /*
> - * Iterate through after-split folios and perform related
> - * operations. But in buddy allocator like split, the folio
> + * Iterate through after-split folios and update folio stats.
> + * But in buddy allocator like split, the folio
> * containing the specified page is skipped until its order
> * is new_order, since the folio will be worked on in next
> * iteration.
> */
> - for (release = folio; release != end_folio; release = next) {
> - next = folio_next(release);
> + for (new_folio = folio; new_folio != end_folio; new_folio = next) {
> + next = folio_next(new_folio);
> /*
> - * for buddy allocator like split, the folio containing
> - * page will be split next and should not be released,
> - * until the folio's order is new_order or stop_split
> - * is set to true by the above xas_split() failure.
> + * for buddy allocator like split, new_folio containing
> + * @split_at page could be split again, thus do not
> + * change stats yet. Wait until new_folio's order is
> + * @new_order or stop_split is set to true by the above
> + * xas_split() failure.
> */
> - if (release == page_folio(split_at)) {
> - folio = release;
> + if (new_folio == page_folio(split_at)) {
> + folio = new_folio;
> if (split_order != new_order && !stop_split)
> continue;
> }
> - if (folio_test_anon(release)) {
> - mod_mthp_stat(folio_order(release),
> - MTHP_STAT_NR_ANON, 1);
> - }
> -
> - /*
> - * origin_folio should be kept frozon until page cache
> - * entries are updated with all the other after-split
> - * folios to prevent others seeing stale page cache
> - * entries.
> - */
> - if (release == origin_folio)
> - continue;
> -
> - folio_ref_unfreeze(release, 1 +
> - ((mapping || swap_cache) ?
> - folio_nr_pages(release) : 0));
> -
> - lru_add_split_folio(origin_folio, release, lruvec,
> - list);
> -
> - /* Some pages can be beyond EOF: drop them from cache */
> - if (release->index >= end) {
> - if (shmem_mapping(mapping))
> - nr_dropped += folio_nr_pages(release);
> - else if (folio_test_clear_dirty(release))
> - folio_account_cleaned(release,
> - inode_to_wb(mapping->host));
> - __filemap_remove_folio(release, NULL);
> - folio_put_refs(release, folio_nr_pages(release));
> - } else if (mapping) {
> - __xa_store(&mapping->i_pages,
> - release->index, release, 0);
> - } else if (swap_cache) {
> - __xa_store(&swap_cache->i_pages,
> - swap_cache_index(release->swap),
> - release, 0);
> - }
> + if (folio_test_anon(new_folio))
> + mod_mthp_stat(folio_order(new_folio),
> + MTHP_STAT_NR_ANON, 1);
> }
> }
>
> - /*
> - * Unfreeze origin_folio only after all page cache entries, which used
> - * to point to it, have been updated with new folios. Otherwise,
> - * a parallel folio_try_get() can grab origin_folio and its caller can
> - * see stale page cache entries.
> - */
> - folio_ref_unfreeze(origin_folio, 1 +
> - ((mapping || swap_cache) ? folio_nr_pages(origin_folio) : 0));
> -
> - unlock_page_lruvec(lruvec);
> -
> - if (swap_cache)
> - xa_unlock(&swap_cache->i_pages);
> - if (mapping)
> - xa_unlock(&mapping->i_pages);
> -
> - /* Caller disabled irqs, so they are still disabled here */
> - local_irq_enable();
> -
> - if (nr_dropped)
> - shmem_uncharge(mapping->host, nr_dropped);
> -
> - remap_page(origin_folio, 1 << order,
> - folio_test_anon(origin_folio) ?
> - RMP_USE_SHARED_ZEROPAGE : 0);
> -
> - /*
> - * At this point, folio should contain the specified page.
> - * For uniform split, it is left for caller to unlock.
> - * For buddy allocator like split, the first after-split folio is left
> - * for caller to unlock.
> - */
> - for (new_folio = origin_folio; new_folio != next_folio; new_folio = next) {
> - next = folio_next(new_folio);
> - if (new_folio == page_folio(lock_at))
> - continue;
> -
> - folio_unlock(new_folio);
> - /*
> - * Subpages may be freed if there wasn't any mapping
> - * like if add_to_swap() is running on a lru page that
> - * had its mapping zapped. And freeing these pages
> - * requires taking the lru_lock so we do the put_page
> - * of the tail pages after the split is complete.
> - */
> - free_folio_and_swap_cache(new_folio);
> - }
> return ret;
> }
>
> @@ -3686,6 +3575,11 @@ bool uniform_split_supported(struct folio *folio, unsigned int new_order,
> * It is in charge of checking whether the split is supported or not and
> * preparing @folio for __split_unmapped_folio().
> *
> + * After splitting, the after-split folio containing @lock_at remains locked
> + * and others are unlocked:
> + * 1. for uniform split, @lock_at points to one of @folio's subpages;
> + * 2. for buddy allocator like (non-uniform) split, @lock_at points to @folio.
> + *
> * return: 0: successful, <0 failed (if -ENOMEM is returned, @folio might be
> * split but not to @new_order, the caller needs to check)
> */
> @@ -3695,10 +3589,12 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
> {
> struct deferred_split *ds_queue = get_deferred_split_queue(folio);
> XA_STATE(xas, &folio->mapping->i_pages, folio->index);
> + struct folio *end_folio = folio_next(folio);
> bool is_anon = folio_test_anon(folio);
> struct address_space *mapping = NULL;
> struct anon_vma *anon_vma = NULL;
> int order = folio_order(folio);
> + struct folio *new_folio, *next;
> int extra_pins, ret;
> pgoff_t end;
> bool is_hzp;
> @@ -3829,6 +3725,10 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
> /* Prevent deferred_split_scan() touching ->_refcount */
> spin_lock(&ds_queue->split_queue_lock);
> if (folio_ref_freeze(folio, 1 + extra_pins)) {
> + struct address_space *swap_cache = NULL;
> + int nr_dropped = 0;
> + struct lruvec *lruvec;
> +
> if (folio_order(folio) > 1 &&
> !list_empty(&folio->_deferred_list)) {
> ds_queue->split_queue_len--;
> @@ -3862,9 +3762,105 @@ static int __folio_split(struct folio *folio, unsigned int new_order,
> }
> }
>
> - ret = __split_unmapped_folio(folio, new_order,
> - split_at, lock_at, list, end, &xas, mapping,
> - uniform_split);
> + if (folio_test_swapcache(folio)) {
> + VM_BUG_ON(mapping);
> +
> + swap_cache = swap_address_space(folio->swap);
> + xa_lock(&swap_cache->i_pages);
> + }
> +
> + /* lock lru list/PageCompound, ref frozen by page_ref_freeze */
> + lruvec = folio_lruvec_lock(folio);
> +
> + ret = __split_unmapped_folio(folio, new_order, split_at, &xas,
> + mapping, uniform_split);
> +
> + /*
> + * Unfreeze after-split folios and put them back to the right
> + * list. @folio should be kept frozon until page cache
> + * entries are updated with all the other after-split folios
> + * to prevent others seeing stale page cache entries.
> + * As a result, new_folio starts from the next folio of
> + * @folio.
> + */
> + for (new_folio = folio_next(folio); new_folio != end_folio;
> + new_folio = next) {
> + next = folio_next(new_folio);
> +
> + folio_ref_unfreeze(
> + new_folio,
> + 1 + ((mapping || swap_cache) ?
> + folio_nr_pages(new_folio) :
> + 0));
> +
> + lru_add_split_folio(folio, new_folio, lruvec, list);
> +
> + /* Some pages can be beyond EOF: drop them from cache */
> + if (new_folio->index >= end) {
> + if (shmem_mapping(mapping))
> + nr_dropped += folio_nr_pages(new_folio);
> + else if (folio_test_clear_dirty(new_folio))
> + folio_account_cleaned(
> + new_folio,
> + inode_to_wb(mapping->host));
> + __filemap_remove_folio(new_folio, NULL);
> + folio_put_refs(new_folio,
> + folio_nr_pages(new_folio));
> + } else if (mapping) {
> + __xa_store(&mapping->i_pages, new_folio->index,
> + new_folio, 0);
> + } else if (swap_cache) {
> + __xa_store(&swap_cache->i_pages,
> + swap_cache_index(new_folio->swap),
> + new_folio, 0);
> + }
> + }
> + /*
> + * Unfreeze @folio only after all page cache entries, which
> + * used to point to it, have been updated with new folios.
> + * Otherwise, a parallel folio_try_get() can grab @folio
> + * and its caller can see stale page cache entries.
> + */
> + folio_ref_unfreeze(folio, 1 +
> + ((mapping || swap_cache) ? folio_nr_pages(folio) : 0));
> +
> + unlock_page_lruvec(lruvec);
> +
> + if (swap_cache)
> + xa_unlock(&swap_cache->i_pages);
> + if (mapping)
> + xas_unlock(&xas);
> +
> + local_irq_enable();
> +
> + if (nr_dropped)
> + shmem_uncharge(mapping->host, nr_dropped);
> +
> + remap_page(folio, 1 << order,
> + !ret && folio_test_anon(folio) ?
> + RMP_USE_SHARED_ZEROPAGE :
> + 0);
> +
> + /*
> + * Unlock all after-split folios except the one containing
> + * @lock_at page. If @folio is not split, it will be kept locked.
> + */
> + for (new_folio = folio; new_folio != end_folio;
> + new_folio = next) {
> + next = folio_next(new_folio);
> + if (new_folio == page_folio(lock_at))
> + continue;
> +
> + folio_unlock(new_folio);
> + /*
> + * Subpages may be freed if there wasn't any mapping
> + * like if add_to_swap() is running on a lru page that
> + * had its mapping zapped. And freeing these pages
> + * requires taking the lru_lock so we do the put_page
> + * of the tail pages after the split is complete.
> + */
> + free_folio_and_swap_cache(new_folio);
> + }
> } else {
> spin_unlock(&ds_queue->split_queue_lock);
> fail:
> --
> 2.47.2
>
On 18.07.25 04:29, Zi Yan wrote: > remap(), folio_ref_unfreeze(), lru_add_split_folio() are not relevant to > splitting unmapped folio operations. Move them out to __folio_split() so > that __split_unmapped_folio() only handles unmapped folio splits. This > makes __split_unmapped_folio() reusable. > > Remove the swapcache folio split check code before __split_unmapped_folio() > call, since it is already checked at the beginning of __folio_split() in > uniform_split_supported() and non_uniform_split_supported(). > > Along with the code move, there are some variable renames: > > 1. release is renamed to new_folio, > 2. origin_folio is now folio, since __folio_split() has folio pointing to > the original folio already. > > Signed-off-by: Zi Yan <ziy@nvidia.com> > --- Acked-by: David Hildenbrand <david@redhat.com> -- Cheers, David / dhildenb
© 2016 - 2025 Red Hat, Inc.