Add a foliated version of copy_page_from_iter_atomic()
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
Cc: Alexander Viro <viro@zeniv.linux.org.uk>
Cc: Matthew Wilcox <willy@infradead.org>
---
include/linux/uio.h | 2 ++
lib/iov_iter.c | 53 ++++++++++++++++++++++++++++++++++++---------
2 files changed, 45 insertions(+), 10 deletions(-)
diff --git a/include/linux/uio.h b/include/linux/uio.h
index 27e3fd9429..b2c281cb10 100644
--- a/include/linux/uio.h
+++ b/include/linux/uio.h
@@ -154,6 +154,8 @@ static inline struct iovec iov_iter_iovec(const struct iov_iter *iter)
size_t copy_page_from_iter_atomic(struct page *page, unsigned offset,
size_t bytes, struct iov_iter *i);
+size_t copy_folio_from_iter_atomic(struct folio *folio, size_t offset,
+ size_t bytes, struct iov_iter *i);
void iov_iter_advance(struct iov_iter *i, size_t bytes);
void iov_iter_revert(struct iov_iter *i, size_t bytes);
size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t bytes);
diff --git a/lib/iov_iter.c b/lib/iov_iter.c
index 274014e4ea..27ba7e9f9e 100644
--- a/lib/iov_iter.c
+++ b/lib/iov_iter.c
@@ -800,18 +800,10 @@ size_t iov_iter_zero(size_t bytes, struct iov_iter *i)
}
EXPORT_SYMBOL(iov_iter_zero);
-size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t bytes,
- struct iov_iter *i)
+static inline size_t __copy_page_from_iter_atomic(struct page *page, unsigned offset,
+ size_t bytes, struct iov_iter *i)
{
char *kaddr = kmap_atomic(page), *p = kaddr + offset;
- if (!page_copy_sane(page, offset, bytes)) {
- kunmap_atomic(kaddr);
- return 0;
- }
- if (WARN_ON_ONCE(!i->data_source)) {
- kunmap_atomic(kaddr);
- return 0;
- }
iterate_and_advance(i, bytes, base, len, off,
copyin(p + off, base, len),
memcpy(p + off, base, len)
@@ -819,8 +811,49 @@ size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t byt
kunmap_atomic(kaddr);
return bytes;
}
+
+size_t copy_page_from_iter_atomic(struct page *page, unsigned offset, size_t bytes,
+ struct iov_iter *i)
+{
+ if (!page_copy_sane(page, offset, bytes))
+ return 0;
+ if (WARN_ON_ONCE(!i->data_source))
+ return 0;
+ return __copy_page_from_iter_atomic(page, offset, bytes, i);
+}
EXPORT_SYMBOL(copy_page_from_iter_atomic);
+size_t copy_folio_from_iter_atomic(struct folio *folio, size_t offset,
+ size_t bytes, struct iov_iter *i)
+{
+ size_t ret = 0;
+
+ if (WARN_ON(offset + bytes > folio_size(folio)))
+ return 0;
+ if (WARN_ON_ONCE(!i->data_source))
+ return 0;
+
+#ifdef CONFIG_HIGHMEM
+ while (bytes) {
+ struct page *page = folio_page(folio, offset >> PAGE_SHIFT);
+ unsigned b = min(bytes, PAGE_SIZE - (offset & PAGE_MASK));
+ unsigned r = __copy_page_from_iter_atomic(page, offset, b, i);
+
+ offset += r;
+ bytes -= r;
+ ret += r;
+
+ if (r != b)
+ break;
+ }
+#else
+ ret = __copy_page_from_iter_atomic(&folio->page, offset, bytes, i);
+#endif
+
+ return ret;
+}
+EXPORT_SYMBOL(copy_folio_from_iter_atomic);
+
static void pipe_advance(struct iov_iter *i, size_t size)
{
struct pipe_inode_info *pipe = i->pipe;
--
2.40.1