When a zone device page is split (via huge pmd folio split). The
driver callback for folio_split is invoked to let the device driver
know that the folio size has been split into a smaller order.
The HMM test driver has been updated to handle the split, since the
test driver uses backing pages, it requires a mechanism of reorganizing
the backing pages (backing pages are used to create a mirror device)
again into the right sized order pages. This is supported by exporting
prep_compound_page().
Cc: Karol Herbst <kherbst@redhat.com>
Cc: Lyude Paul <lyude@redhat.com>
Cc: Danilo Krummrich <dakr@kernel.org>
Cc: David Airlie <airlied@gmail.com>
Cc: Simona Vetter <simona@ffwll.ch>
Cc: "Jérôme Glisse" <jglisse@redhat.com>
Cc: Shuah Khan <shuah@kernel.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: Barry Song <baohua@kernel.org>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Ryan Roberts <ryan.roberts@arm.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Peter Xu <peterx@redhat.com>
Cc: Zi Yan <ziy@nvidia.com>
Cc: Kefeng Wang <wangkefeng.wang@huawei.com>
Cc: Jane Chu <jane.chu@oracle.com>
Cc: Alistair Popple <apopple@nvidia.com>
Cc: Donet Tom <donettom@linux.ibm.com>
Signed-off-by: Balbir Singh <balbirs@nvidia.com>
---
include/linux/memremap.h | 7 +++++++
include/linux/mm.h | 1 +
lib/test_hmm.c | 42 ++++++++++++++++++++++++++++++++++++++++
mm/huge_memory.c | 14 ++++++++++++++
mm/page_alloc.c | 1 +
5 files changed, 65 insertions(+)
diff --git a/include/linux/memremap.h b/include/linux/memremap.h
index 11d586dd8ef1..2091b754f1da 100644
--- a/include/linux/memremap.h
+++ b/include/linux/memremap.h
@@ -100,6 +100,13 @@ struct dev_pagemap_ops {
*/
int (*memory_failure)(struct dev_pagemap *pgmap, unsigned long pfn,
unsigned long nr_pages, int mf_flags);
+
+ /*
+ * Used for private (un-addressable) device memory only.
+ * This callback is used when a folio is split into
+ * a smaller folio
+ */
+ void (*folio_split)(struct folio *head, struct folio *tail);
};
#define PGMAP_ALTMAP_VALID (1 << 0)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index ef40f68c1183..f7bda8b1e46c 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -1183,6 +1183,7 @@ static inline struct folio *virt_to_folio(const void *x)
void __folio_put(struct folio *folio);
void split_page(struct page *page, unsigned int order);
+void prep_compound_page(struct page *page, unsigned int order);
void folio_copy(struct folio *dst, struct folio *src);
int folio_mc_copy(struct folio *dst, struct folio *src);
diff --git a/lib/test_hmm.c b/lib/test_hmm.c
index 95b4276a17fd..e20021fb7c69 100644
--- a/lib/test_hmm.c
+++ b/lib/test_hmm.c
@@ -1646,9 +1646,51 @@ static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
return ret;
}
+static void dmirror_devmem_folio_split(struct folio *head, struct folio *tail)
+{
+ struct page *rpage = BACKING_PAGE(folio_page(head, 0));
+ struct page *rpage_tail;
+ struct folio *rfolio;
+ unsigned long offset = 0;
+ unsigned int tail_order;
+ unsigned int head_order = folio_order(head);
+
+ if (!rpage) {
+ tail->page.zone_device_data = NULL;
+ return;
+ }
+
+ rfolio = page_folio(rpage);
+
+ if (tail == NULL) {
+ folio_reset_order(rfolio);
+ rfolio->mapping = NULL;
+ if (head_order)
+ prep_compound_page(rpage, head_order);
+ folio_set_count(rfolio, 1 << head_order);
+ return;
+ }
+
+ offset = folio_pfn(tail) - folio_pfn(head);
+
+ rpage_tail = folio_page(rfolio, offset);
+ tail->page.zone_device_data = rpage_tail;
+ clear_compound_head(rpage_tail);
+ rpage_tail->mapping = NULL;
+
+ tail_order = folio_order(tail);
+ if (tail_order)
+ prep_compound_page(rpage_tail, tail_order);
+
+ folio_page(tail, 0)->mapping = folio_page(head, 0)->mapping;
+ tail->pgmap = head->pgmap;
+ folio_set_count(page_folio(rpage_tail), 1 << tail_order);
+}
+
static const struct dev_pagemap_ops dmirror_devmem_ops = {
.page_free = dmirror_devmem_free,
.migrate_to_ram = dmirror_devmem_fault,
+ .folio_split = dmirror_devmem_folio_split,
};
static int dmirror_device_init(struct dmirror_device *mdevice, int id)
diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index f29add796931..d55e36ae0c39 100644
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -3630,6 +3630,11 @@ static int __split_unmapped_folio(struct folio *folio, int new_order,
if (release == origin_folio)
continue;
+ if (folio_is_device_private(origin_folio) &&
+ origin_folio->pgmap->ops->folio_split)
+ origin_folio->pgmap->ops->folio_split(
+ origin_folio, release);
+
folio_ref_unfreeze(release, 1 +
((mapping || swap_cache) ?
folio_nr_pages(release) : 0));
@@ -3661,6 +3666,15 @@ static int __split_unmapped_folio(struct folio *folio, int new_order,
}
}
+ /*
+ * Mark the end of the split, so that the driver can setup origin_folio's
+ * order and other metadata
+ */
+ if (folio_is_device_private(origin_folio) &&
+ origin_folio->pgmap->ops->folio_split)
+ origin_folio->pgmap->ops->folio_split(
+ origin_folio, NULL);
+
/*
* Unfreeze origin_folio only after all page cache entries, which used
* to point to it, have been updated with new folios. Otherwise,
diff --git a/mm/page_alloc.c b/mm/page_alloc.c
index 4f55f8ed65c7..0a538e9c24bd 100644
--- a/mm/page_alloc.c
+++ b/mm/page_alloc.c
@@ -722,6 +722,7 @@ void prep_compound_page(struct page *page, unsigned int order)
prep_compound_head(page, order);
}
+EXPORT_SYMBOL_GPL(prep_compound_page);
static inline void set_buddy_order(struct page *page, unsigned int order)
{
--
2.49.0
On 7/4/25 02:35, Balbir Singh wrote:
> When a zone device page is split (via huge pmd folio split). The
> driver callback for folio_split is invoked to let the device driver
> know that the folio size has been split into a smaller order.
>
> The HMM test driver has been updated to handle the split, since the
> test driver uses backing pages, it requires a mechanism of reorganizing
> the backing pages (backing pages are used to create a mirror device)
> again into the right sized order pages. This is supported by exporting
> prep_compound_page().
>
> Cc: Karol Herbst <kherbst@redhat.com>
> Cc: Lyude Paul <lyude@redhat.com>
> Cc: Danilo Krummrich <dakr@kernel.org>
> Cc: David Airlie <airlied@gmail.com>
> Cc: Simona Vetter <simona@ffwll.ch>
> Cc: "Jérôme Glisse" <jglisse@redhat.com>
> Cc: Shuah Khan <shuah@kernel.org>
> Cc: David Hildenbrand <david@redhat.com>
> Cc: Barry Song <baohua@kernel.org>
> Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
> Cc: Ryan Roberts <ryan.roberts@arm.com>
> Cc: Matthew Wilcox <willy@infradead.org>
> Cc: Peter Xu <peterx@redhat.com>
> Cc: Zi Yan <ziy@nvidia.com>
> Cc: Kefeng Wang <wangkefeng.wang@huawei.com>
> Cc: Jane Chu <jane.chu@oracle.com>
> Cc: Alistair Popple <apopple@nvidia.com>
> Cc: Donet Tom <donettom@linux.ibm.com>
>
> Signed-off-by: Balbir Singh <balbirs@nvidia.com>
> ---
> include/linux/memremap.h | 7 +++++++
> include/linux/mm.h | 1 +
> lib/test_hmm.c | 42 ++++++++++++++++++++++++++++++++++++++++
> mm/huge_memory.c | 14 ++++++++++++++
> mm/page_alloc.c | 1 +
> 5 files changed, 65 insertions(+)
>
> diff --git a/include/linux/memremap.h b/include/linux/memremap.h
> index 11d586dd8ef1..2091b754f1da 100644
> --- a/include/linux/memremap.h
> +++ b/include/linux/memremap.h
> @@ -100,6 +100,13 @@ struct dev_pagemap_ops {
> */
> int (*memory_failure)(struct dev_pagemap *pgmap, unsigned long pfn,
> unsigned long nr_pages, int mf_flags);
> +
> + /*
> + * Used for private (un-addressable) device memory only.
> + * This callback is used when a folio is split into
> + * a smaller folio
> + */
> + void (*folio_split)(struct folio *head, struct folio *tail);
> };
>
> #define PGMAP_ALTMAP_VALID (1 << 0)
> diff --git a/include/linux/mm.h b/include/linux/mm.h
> index ef40f68c1183..f7bda8b1e46c 100644
> --- a/include/linux/mm.h
> +++ b/include/linux/mm.h
> @@ -1183,6 +1183,7 @@ static inline struct folio *virt_to_folio(const void *x)
> void __folio_put(struct folio *folio);
>
> void split_page(struct page *page, unsigned int order);
> +void prep_compound_page(struct page *page, unsigned int order);
> void folio_copy(struct folio *dst, struct folio *src);
> int folio_mc_copy(struct folio *dst, struct folio *src);
>
> diff --git a/lib/test_hmm.c b/lib/test_hmm.c
> index 95b4276a17fd..e20021fb7c69 100644
> --- a/lib/test_hmm.c
> +++ b/lib/test_hmm.c
> @@ -1646,9 +1646,51 @@ static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
> return ret;
> }
>
> +static void dmirror_devmem_folio_split(struct folio *head, struct folio *tail)
> +{
> + struct page *rpage = BACKING_PAGE(folio_page(head, 0));
> + struct page *rpage_tail;
> + struct folio *rfolio;
> + unsigned long offset = 0;
> + unsigned int tail_order;
> + unsigned int head_order = folio_order(head);
> +
> + if (!rpage) {
> + tail->page.zone_device_data = NULL;
> + return;
> + }
> +
> + rfolio = page_folio(rpage);
> +
> + if (tail == NULL) {
> + folio_reset_order(rfolio);
> + rfolio->mapping = NULL;
> + if (head_order)
> + prep_compound_page(rpage, head_order);
> + folio_set_count(rfolio, 1 << head_order);
> + return;
> + }
> +
> + offset = folio_pfn(tail) - folio_pfn(head);
> +
> + rpage_tail = folio_page(rfolio, offset);
> + tail->page.zone_device_data = rpage_tail;
> + clear_compound_head(rpage_tail);
> + rpage_tail->mapping = NULL;
> +
> + tail_order = folio_order(tail);
> + if (tail_order)
> + prep_compound_page(rpage_tail, tail_order);
> +
> + folio_page(tail, 0)->mapping = folio_page(head, 0)->mapping;
> + tail->pgmap = head->pgmap;
> + folio_set_count(page_folio(rpage_tail), 1 << tail_order);
> +}
> +
> static const struct dev_pagemap_ops dmirror_devmem_ops = {
> .page_free = dmirror_devmem_free,
> .migrate_to_ram = dmirror_devmem_fault,
> + .folio_split = dmirror_devmem_folio_split,
> };
>
> static int dmirror_device_init(struct dmirror_device *mdevice, int id)
> diff --git a/mm/huge_memory.c b/mm/huge_memory.c
> index f29add796931..d55e36ae0c39 100644
> --- a/mm/huge_memory.c
> +++ b/mm/huge_memory.c
> @@ -3630,6 +3630,11 @@ static int __split_unmapped_folio(struct folio *folio, int new_order,
> if (release == origin_folio)
> continue;
>
> + if (folio_is_device_private(origin_folio) &&
> + origin_folio->pgmap->ops->folio_split)
> + origin_folio->pgmap->ops->folio_split(
> + origin_folio, release);
Should folio split fail if pgmap->ops->folio_split() is not defined? If not then at least the >pgmap pointer copy should be in the common code.
> +
> folio_ref_unfreeze(release, 1 +
> ((mapping || swap_cache) ?
> folio_nr_pages(release) : 0));
> @@ -3661,6 +3666,15 @@ static int __split_unmapped_folio(struct folio *folio, int new_order,
> }
> }
>
> + /*
> + * Mark the end of the split, so that the driver can setup origin_folio's
> + * order and other metadata
> + */
> + if (folio_is_device_private(origin_folio) &&
> + origin_folio->pgmap->ops->folio_split)
> + origin_folio->pgmap->ops->folio_split(
> + origin_folio, NULL);
> +
> /*
> * Unfreeze origin_folio only after all page cache entries, which used
> * to point to it, have been updated with new folios. Otherwise,
> diff --git a/mm/page_alloc.c b/mm/page_alloc.c
> index 4f55f8ed65c7..0a538e9c24bd 100644
> --- a/mm/page_alloc.c
> +++ b/mm/page_alloc.c
> @@ -722,6 +722,7 @@ void prep_compound_page(struct page *page, unsigned int order)
>
> prep_compound_head(page, order);
> }
> +EXPORT_SYMBOL_GPL(prep_compound_page);
>
> static inline void set_buddy_order(struct page *page, unsigned int order)
> {
On 7/4/25 21:14, Mika Penttilä wrote: > > On 7/4/25 02:35, Balbir Singh wrote: >> >> + if (folio_is_device_private(origin_folio) && >> + origin_folio->pgmap->ops->folio_split) >> + origin_folio->pgmap->ops->folio_split( >> + origin_folio, release); > > Should folio split fail if pgmap->ops->folio_split() is not defined? If not then at least the >pgmap pointer copy should be in the common code. > We could change the code can check ->folio_split callback is not NULL when MIGRATE_VMA_SELECT_COMPOUND is used and cause migrate_vma to fail Providing a default implementation might surprise drivers about the split Balbir Singh
© 2016 - 2025 Red Hat, Inc.