From: "Mike Rapoport (Microsoft)" <rppt@kernel.org>
When userspace resolves a page fault in a shmem VMA with UFFDIO_CONTINUE
it needs to get a folio that already exists in the pagecache backing
that VMA.
Instead of using shmem_get_folio() for that, add a get_folio() method to
'struct vm_operations_struct' that will return a folio if it exists in
the VMA's pagecache at given pgoff.
Implement get_folio() method for shmem and slightly refactor
userfaultfd's mfill_atomic() and mfill_atomic_pte_continue() to support
this new API.
Signed-off-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
---
include/linux/mm.h | 9 ++++++++
mm/shmem.c | 18 ++++++++++++++++
mm/userfaultfd.c | 52 +++++++++++++++++++++++++++++-----------------
3 files changed, 60 insertions(+), 19 deletions(-)
diff --git a/include/linux/mm.h b/include/linux/mm.h
index 7c79b3369b82..c8647707d75b 100644
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -690,6 +690,15 @@ struct vm_operations_struct {
struct page *(*find_normal_page)(struct vm_area_struct *vma,
unsigned long addr);
#endif /* CONFIG_FIND_NORMAL_PAGE */
+#ifdef CONFIG_USERFAULTFD
+ /*
+ * Called by userfault to resolve UFFDIO_CONTINUE request.
+ * Should return the folio found at pgoff in the VMA's pagecache if it
+ * exists or ERR_PTR otherwise.
+ * The returned folio is locked and with reference held.
+ */
+ struct folio *(*get_folio)(struct inode *inode, pgoff_t pgoff);
+#endif
};
#ifdef CONFIG_NUMA_BALANCING
diff --git a/mm/shmem.c b/mm/shmem.c
index 58701d14dd96..e16c7c8c3e1e 100644
--- a/mm/shmem.c
+++ b/mm/shmem.c
@@ -3263,6 +3263,18 @@ int shmem_mfill_atomic_pte(pmd_t *dst_pmd,
shmem_inode_unacct_blocks(inode, 1);
return ret;
}
+
+static struct folio *shmem_get_folio_noalloc(struct inode *inode, pgoff_t pgoff)
+{
+ struct folio *folio;
+ int err;
+
+ err = shmem_get_folio(inode, pgoff, 0, &folio, SGP_NOALLOC);
+ if (err)
+ return ERR_PTR(err);
+
+ return folio;
+}
#endif /* CONFIG_USERFAULTFD */
#ifdef CONFIG_TMPFS
@@ -5295,6 +5307,9 @@ static const struct vm_operations_struct shmem_vm_ops = {
.set_policy = shmem_set_policy,
.get_policy = shmem_get_policy,
#endif
+#ifdef CONFIG_USERFAULTFD
+ .get_folio = shmem_get_folio_noalloc,
+#endif
};
static const struct vm_operations_struct shmem_anon_vm_ops = {
@@ -5304,6 +5319,9 @@ static const struct vm_operations_struct shmem_anon_vm_ops = {
.set_policy = shmem_set_policy,
.get_policy = shmem_get_policy,
#endif
+#ifdef CONFIG_USERFAULTFD
+ .get_folio = shmem_get_folio_noalloc,
+#endif
};
int shmem_init_fs_context(struct fs_context *fc)
diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
index 8dc964389b0d..9f0f879b603a 100644
--- a/mm/userfaultfd.c
+++ b/mm/userfaultfd.c
@@ -388,15 +388,12 @@ static int mfill_atomic_pte_continue(pmd_t *dst_pmd,
struct page *page;
int ret;
- ret = shmem_get_folio(inode, pgoff, 0, &folio, SGP_NOALLOC);
+ folio = dst_vma->vm_ops->get_folio(inode, pgoff);
/* Our caller expects us to return -EFAULT if we failed to find folio */
- if (ret == -ENOENT)
- ret = -EFAULT;
- if (ret)
- goto out;
- if (!folio) {
- ret = -EFAULT;
- goto out;
+ if (IS_ERR_OR_NULL(folio)) {
+ if (PTR_ERR(folio) == -ENOENT || !folio)
+ return -EFAULT;
+ return PTR_ERR(folio);
}
page = folio_file_page(folio, pgoff);
@@ -411,13 +408,12 @@ static int mfill_atomic_pte_continue(pmd_t *dst_pmd,
goto out_release;
folio_unlock(folio);
- ret = 0;
-out:
- return ret;
+ return 0;
+
out_release:
folio_unlock(folio);
folio_put(folio);
- goto out;
+ return ret;
}
/* Handles UFFDIO_POISON for all non-hugetlb VMAs. */
@@ -694,6 +690,15 @@ static __always_inline ssize_t mfill_atomic_pte(pmd_t *dst_pmd,
return err;
}
+static __always_inline bool vma_can_mfill_atomic(struct vm_area_struct *vma,
+ uffd_flags_t flags)
+{
+ if (uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE))
+ return vma->vm_ops && vma->vm_ops->get_folio;
+
+ return vma_is_anonymous(vma) || vma_is_shmem(vma);
+}
+
static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx,
unsigned long dst_start,
unsigned long src_start,
@@ -766,10 +771,7 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx,
return mfill_atomic_hugetlb(ctx, dst_vma, dst_start,
src_start, len, flags);
- if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma))
- goto out_unlock;
- if (!vma_is_shmem(dst_vma) &&
- uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE))
+ if (!vma_can_mfill_atomic(dst_vma, flags))
goto out_unlock;
while (src_addr < src_start + len) {
@@ -1985,9 +1987,21 @@ bool vma_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags,
if (vma->vm_flags & VM_DROPPABLE)
return false;
- if ((vm_flags & VM_UFFD_MINOR) &&
- (!is_vm_hugetlb_page(vma) && !vma_is_shmem(vma)))
- return false;
+ if (vm_flags & VM_UFFD_MINOR) {
+ /*
+ * If only MINOR mode is requested and we can request an
+ * existing folio from VMA's page cache, allow it
+ */
+ if (vm_flags == VM_UFFD_MINOR && vma->vm_ops &&
+ vma->vm_ops->get_folio)
+ return true;
+ /*
+ * Only hugetlb and shmem can support MINOR mode in combination
+ * with other modes
+ */
+ if (!is_vm_hugetlb_page(vma) && !vma_is_shmem(vma))
+ return false;
+ }
/*
* If wp async enabled, and WP is the only mode enabled, allow any
--
2.50.1* Mike Rapoport <rppt@kernel.org> [251125 13:39]:
> From: "Mike Rapoport (Microsoft)" <rppt@kernel.org>
>
> When userspace resolves a page fault in a shmem VMA with UFFDIO_CONTINUE
> it needs to get a folio that already exists in the pagecache backing
> that VMA.
>
> Instead of using shmem_get_folio() for that, add a get_folio() method to
> 'struct vm_operations_struct' that will return a folio if it exists in
> the VMA's pagecache at given pgoff.
>
> Implement get_folio() method for shmem and slightly refactor
> userfaultfd's mfill_atomic() and mfill_atomic_pte_continue() to support
> this new API.
>
> Signed-off-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
Reviewed-by: Liam R. Howlett <Liam.Howlett@oracle.com>
> ---
> include/linux/mm.h | 9 ++++++++
> mm/shmem.c | 18 ++++++++++++++++
> mm/userfaultfd.c | 52 +++++++++++++++++++++++++++++-----------------
> 3 files changed, 60 insertions(+), 19 deletions(-)
>
> diff --git a/include/linux/mm.h b/include/linux/mm.h
> index 7c79b3369b82..c8647707d75b 100644
> --- a/include/linux/mm.h
> +++ b/include/linux/mm.h
> @@ -690,6 +690,15 @@ struct vm_operations_struct {
> struct page *(*find_normal_page)(struct vm_area_struct *vma,
> unsigned long addr);
> #endif /* CONFIG_FIND_NORMAL_PAGE */
> +#ifdef CONFIG_USERFAULTFD
> + /*
> + * Called by userfault to resolve UFFDIO_CONTINUE request.
> + * Should return the folio found at pgoff in the VMA's pagecache if it
> + * exists or ERR_PTR otherwise.
> + * The returned folio is locked and with reference held.
> + */
> + struct folio *(*get_folio)(struct inode *inode, pgoff_t pgoff);
> +#endif
> };
>
> #ifdef CONFIG_NUMA_BALANCING
> diff --git a/mm/shmem.c b/mm/shmem.c
> index 58701d14dd96..e16c7c8c3e1e 100644
> --- a/mm/shmem.c
> +++ b/mm/shmem.c
> @@ -3263,6 +3263,18 @@ int shmem_mfill_atomic_pte(pmd_t *dst_pmd,
> shmem_inode_unacct_blocks(inode, 1);
> return ret;
> }
> +
> +static struct folio *shmem_get_folio_noalloc(struct inode *inode, pgoff_t pgoff)
> +{
> + struct folio *folio;
> + int err;
> +
> + err = shmem_get_folio(inode, pgoff, 0, &folio, SGP_NOALLOC);
> + if (err)
> + return ERR_PTR(err);
> +
> + return folio;
> +}
> #endif /* CONFIG_USERFAULTFD */
>
> #ifdef CONFIG_TMPFS
> @@ -5295,6 +5307,9 @@ static const struct vm_operations_struct shmem_vm_ops = {
> .set_policy = shmem_set_policy,
> .get_policy = shmem_get_policy,
> #endif
> +#ifdef CONFIG_USERFAULTFD
> + .get_folio = shmem_get_folio_noalloc,
> +#endif
> };
>
> static const struct vm_operations_struct shmem_anon_vm_ops = {
> @@ -5304,6 +5319,9 @@ static const struct vm_operations_struct shmem_anon_vm_ops = {
> .set_policy = shmem_set_policy,
> .get_policy = shmem_get_policy,
> #endif
> +#ifdef CONFIG_USERFAULTFD
> + .get_folio = shmem_get_folio_noalloc,
> +#endif
> };
>
> int shmem_init_fs_context(struct fs_context *fc)
> diff --git a/mm/userfaultfd.c b/mm/userfaultfd.c
> index 8dc964389b0d..9f0f879b603a 100644
> --- a/mm/userfaultfd.c
> +++ b/mm/userfaultfd.c
> @@ -388,15 +388,12 @@ static int mfill_atomic_pte_continue(pmd_t *dst_pmd,
> struct page *page;
> int ret;
>
> - ret = shmem_get_folio(inode, pgoff, 0, &folio, SGP_NOALLOC);
> + folio = dst_vma->vm_ops->get_folio(inode, pgoff);
> /* Our caller expects us to return -EFAULT if we failed to find folio */
> - if (ret == -ENOENT)
> - ret = -EFAULT;
> - if (ret)
> - goto out;
> - if (!folio) {
> - ret = -EFAULT;
> - goto out;
> + if (IS_ERR_OR_NULL(folio)) {
> + if (PTR_ERR(folio) == -ENOENT || !folio)
> + return -EFAULT;
> + return PTR_ERR(folio);
> }
>
> page = folio_file_page(folio, pgoff);
> @@ -411,13 +408,12 @@ static int mfill_atomic_pte_continue(pmd_t *dst_pmd,
> goto out_release;
>
> folio_unlock(folio);
> - ret = 0;
> -out:
> - return ret;
> + return 0;
> +
> out_release:
> folio_unlock(folio);
> folio_put(folio);
> - goto out;
> + return ret;
I really like this part.
> }
>
> /* Handles UFFDIO_POISON for all non-hugetlb VMAs. */
> @@ -694,6 +690,15 @@ static __always_inline ssize_t mfill_atomic_pte(pmd_t *dst_pmd,
> return err;
> }
>
> +static __always_inline bool vma_can_mfill_atomic(struct vm_area_struct *vma,
> + uffd_flags_t flags)
> +{
> + if (uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE))
> + return vma->vm_ops && vma->vm_ops->get_folio;
> +
> + return vma_is_anonymous(vma) || vma_is_shmem(vma);
> +}
> +
> static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx,
> unsigned long dst_start,
> unsigned long src_start,
> @@ -766,10 +771,7 @@ static __always_inline ssize_t mfill_atomic(struct userfaultfd_ctx *ctx,
> return mfill_atomic_hugetlb(ctx, dst_vma, dst_start,
> src_start, len, flags);
>
> - if (!vma_is_anonymous(dst_vma) && !vma_is_shmem(dst_vma))
> - goto out_unlock;
> - if (!vma_is_shmem(dst_vma) &&
> - uffd_flags_mode_is(flags, MFILL_ATOMIC_CONTINUE))
> + if (!vma_can_mfill_atomic(dst_vma, flags))
> goto out_unlock;
>
> while (src_addr < src_start + len) {
> @@ -1985,9 +1987,21 @@ bool vma_can_userfault(struct vm_area_struct *vma, vm_flags_t vm_flags,
> if (vma->vm_flags & VM_DROPPABLE)
> return false;
>
> - if ((vm_flags & VM_UFFD_MINOR) &&
> - (!is_vm_hugetlb_page(vma) && !vma_is_shmem(vma)))
> - return false;
> + if (vm_flags & VM_UFFD_MINOR) {
> + /*
> + * If only MINOR mode is requested and we can request an
> + * existing folio from VMA's page cache, allow it
> + */
> + if (vm_flags == VM_UFFD_MINOR && vma->vm_ops &&
> + vma->vm_ops->get_folio)
> + return true;
> + /*
> + * Only hugetlb and shmem can support MINOR mode in combination
> + * with other modes
> + */
> + if (!is_vm_hugetlb_page(vma) && !vma_is_shmem(vma))
> + return false;
> + }
>
> /*
> * If wp async enabled, and WP is the only mode enabled, allow any
> --
> 2.50.1
>
On 11/25/25 19:38, Mike Rapoport wrote: > From: "Mike Rapoport (Microsoft)" <rppt@kernel.org> > > When userspace resolves a page fault in a shmem VMA with UFFDIO_CONTINUE > it needs to get a folio that already exists in the pagecache backing > that VMA. > > Instead of using shmem_get_folio() for that, add a get_folio() method to > 'struct vm_operations_struct' that will return a folio if it exists in > the VMA's pagecache at given pgoff. > > Implement get_folio() method for shmem and slightly refactor > userfaultfd's mfill_atomic() and mfill_atomic_pte_continue() to support > this new API. > > Signed-off-by: Mike Rapoport (Microsoft) <rppt@kernel.org> > --- Acked-by: David Hildenbrand (Red Hat) <david@kernel.org> -- Cheers David
© 2016 - 2025 Red Hat, Inc.