For direct block device, use copy_file_range to issue device copy offload,
and fallback to generic_copy_file_range incase device copy offload
capability is absent. Modify checks to allow bdevs to use copy_file_range.
Suggested-by: Ming Lei <ming.lei@redhat.com>
Signed-off-by: Anuj Gupta <anuj20.g@samsung.com>
Signed-off-by: Nitesh Shetty <nj.shetty@samsung.com>
---
block/blk-lib.c | 22 ++++++++++++++++++++++
block/fops.c | 18 ++++++++++++++++++
fs/read_write.c | 11 +++++++++--
include/linux/blkdev.h | 3 +++
4 files changed, 52 insertions(+), 2 deletions(-)
diff --git a/block/blk-lib.c b/block/blk-lib.c
index 74f58faf82d8..6593de525a26 100644
--- a/block/blk-lib.c
+++ b/block/blk-lib.c
@@ -475,6 +475,28 @@ static inline bool blk_check_copy_offload(struct request_queue *q_in,
return blk_queue_copy(q_in) && blk_queue_copy(q_out);
}
+int blkdev_copy_offload(struct block_device *bdev_in, loff_t pos_in,
+ struct block_device *bdev_out, loff_t pos_out, size_t len,
+ cio_iodone_t end_io, void *private, gfp_t gfp_mask)
+{
+ struct request_queue *in_q = bdev_get_queue(bdev_in);
+ struct request_queue *out_q = bdev_get_queue(bdev_out);
+ int ret = -EINVAL;
+ bool offload = false;
+
+ ret = blk_copy_sanity_check(bdev_in, pos_in, bdev_out, pos_out, len);
+ if (ret)
+ return ret;
+
+ offload = blk_check_copy_offload(in_q, out_q);
+ if (offload)
+ ret = __blk_copy_offload(bdev_in, pos_in, bdev_out, pos_out,
+ len, end_io, private, gfp_mask);
+
+ return ret;
+}
+EXPORT_SYMBOL_GPL(blkdev_copy_offload);
+
/*
* @bdev_in: source block device
* @pos_in: source offset
diff --git a/block/fops.c b/block/fops.c
index 50d245e8c913..bcb9ee6565ea 100644
--- a/block/fops.c
+++ b/block/fops.c
@@ -596,6 +596,23 @@ static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)
return ret;
}
+ssize_t blkdev_copy_file_range(struct file *file_in, loff_t pos_in,
+ struct file *file_out, loff_t pos_out,
+ size_t len, unsigned int flags)
+{
+ struct block_device *in_bdev = I_BDEV(bdev_file_inode(file_in));
+ struct block_device *out_bdev = I_BDEV(bdev_file_inode(file_out));
+ int comp_len;
+
+ comp_len = blkdev_copy_offload(in_bdev, pos_in, out_bdev, pos_out, len,
+ NULL, NULL, GFP_KERNEL);
+ if (comp_len != len)
+ comp_len = generic_copy_file_range(file_in, pos_in + comp_len,
+ file_out, pos_out + comp_len, len - comp_len, flags);
+
+ return comp_len;
+}
+
#define BLKDEV_FALLOC_FL_SUPPORTED \
(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | \
FALLOC_FL_ZERO_RANGE | FALLOC_FL_NO_HIDE_STALE)
@@ -679,6 +696,7 @@ const struct file_operations def_blk_fops = {
.splice_read = generic_file_splice_read,
.splice_write = iter_file_splice_write,
.fallocate = blkdev_fallocate,
+ .copy_file_range = blkdev_copy_file_range,
};
static __init int blkdev_init(void)
diff --git a/fs/read_write.c b/fs/read_write.c
index 7a2ff6157eda..62e925e9b2f0 100644
--- a/fs/read_write.c
+++ b/fs/read_write.c
@@ -20,6 +20,7 @@
#include <linux/compat.h>
#include <linux/mount.h>
#include <linux/fs.h>
+#include <linux/blkdev.h>
#include "internal.h"
#include <linux/uaccess.h>
@@ -1448,7 +1449,11 @@ static int generic_copy_file_checks(struct file *file_in, loff_t pos_in,
return -EOVERFLOW;
/* Shorten the copy to EOF */
- size_in = i_size_read(inode_in);
+ if (S_ISBLK(inode_in->i_mode))
+ size_in = bdev_nr_bytes(I_BDEV(file_in->f_mapping->host));
+ else
+ size_in = i_size_read(inode_in);
+
if (pos_in >= size_in)
count = 0;
else
@@ -1709,7 +1714,9 @@ int generic_file_rw_checks(struct file *file_in, struct file *file_out)
/* Don't copy dirs, pipes, sockets... */
if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode))
return -EISDIR;
- if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode))
+
+ if ((!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode)) &&
+ (!S_ISBLK(inode_in->i_mode) || !S_ISBLK(inode_out->i_mode)))
return -EINVAL;
if (!(file_in->f_mode & FMODE_READ) ||
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 766761911190..ca7828b25d90 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -1066,6 +1066,9 @@ int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector,
int blkdev_issue_copy(struct block_device *bdev_in, loff_t pos_in,
struct block_device *bdev_out, loff_t pos_out, size_t len,
cio_iodone_t end_io, void *private, gfp_t gfp_mask);
+int blkdev_copy_offload(struct block_device *bdev_in, loff_t pos_in,
+ struct block_device *bdev_out, loff_t pos_out, size_t len,
+ cio_iodone_t end_io, void *private, gfp_t gfp_mask);
struct bio *bio_map_kern(struct request_queue *q, void *data, unsigned int len,
gfp_t gfp_mask);
void bio_map_kern_endio(struct bio *bio);
--
2.35.1.500.gb896f729e2Hi Nitesh,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on device-mapper-dm/for-next]
[also build test ERROR on linus/master v6.2 next-20230220]
[cannot apply to axboe-block/for-next]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Nitesh-Shetty/block-Add-copy-offload-support-infrastructure/20230220-205057
base: https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git for-next
patch link: https://lore.kernel.org/r/20230220105336.3810-5-nj.shetty%40samsung.com
patch subject: [PATCH v7 4/8] fs, block: copy_file_range for def_blk_ops for direct block device.
config: arm-randconfig-r046-20230220 (https://download.01.org/0day-ci/archive/20230221/202302210631.4JSFH2VI-lkp@intel.com/config)
compiler: arm-linux-gnueabi-gcc (GCC) 12.1.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/intel-lab-lkp/linux/commit/0f95ad2cb727ac6ac8406a01ff216d9237b403b7
git remote add linux-review https://github.com/intel-lab-lkp/linux
git fetch --no-tags linux-review Nitesh-Shetty/block-Add-copy-offload-support-infrastructure/20230220-205057
git checkout 0f95ad2cb727ac6ac8406a01ff216d9237b403b7
# save the config file
mkdir build_dir && cp config build_dir/.config
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross W=1 O=build_dir ARCH=arm olddefconfig
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross W=1 O=build_dir ARCH=arm SHELL=/bin/bash
If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot <lkp@intel.com>
| Link: https://lore.kernel.org/oe-kbuild-all/202302210631.4JSFH2VI-lkp@intel.com/
All errors (new ones prefixed by >>):
arm-linux-gnueabi-ld: arm-linux-gnueabi-ld: DWARF error: could not find abbrev number 19
fs/read_write.o: in function `generic_copy_file_checks':
>> read_write.c:(.text+0x8980): undefined reference to `I_BDEV'
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests
Hi Nitesh,
Thank you for the patch! Yet something to improve:
[auto build test ERROR on device-mapper-dm/for-next]
[also build test ERROR on linus/master v6.2 next-20230220]
[cannot apply to axboe-block/for-next]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Nitesh-Shetty/block-Add-copy-offload-support-infrastructure/20230220-205057
base: https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git for-next
patch link: https://lore.kernel.org/r/20230220105336.3810-5-nj.shetty%40samsung.com
patch subject: [PATCH v7 4/8] fs, block: copy_file_range for def_blk_ops for direct block device.
config: riscv-randconfig-r042-20230219 (https://download.01.org/0day-ci/archive/20230221/202302210520.EIfbuJLy-lkp@intel.com/config)
compiler: riscv64-linux-gcc (GCC) 12.1.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/intel-lab-lkp/linux/commit/0f95ad2cb727ac6ac8406a01ff216d9237b403b7
git remote add linux-review https://github.com/intel-lab-lkp/linux
git fetch --no-tags linux-review Nitesh-Shetty/block-Add-copy-offload-support-infrastructure/20230220-205057
git checkout 0f95ad2cb727ac6ac8406a01ff216d9237b403b7
# save the config file
mkdir build_dir && cp config build_dir/.config
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross W=1 O=build_dir ARCH=riscv olddefconfig
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross W=1 O=build_dir ARCH=riscv SHELL=/bin/bash
If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot <lkp@intel.com>
| Link: https://lore.kernel.org/oe-kbuild-all/202302210520.EIfbuJLy-lkp@intel.com/
All errors (new ones prefixed by >>):
riscv64-linux-ld: fs/read_write.o: in function `__do_compat_sys_preadv2':
>> fs/read_write.c:1134: undefined reference to `I_BDEV'
vim +1134 fs/read_write.c
3ebfd81f7fb3e8 H.J. Lu 2016-07-14 1128
f17d8b35452cab Milosz Tanski 2016-03-03 1129 COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd,
3523a9d4547898 Christoph Hellwig 2020-09-25 1130 const struct iovec __user *, vec,
f17d8b35452cab Milosz Tanski 2016-03-03 1131 compat_ulong_t, vlen, u32, pos_low, u32, pos_high,
ddef7ed2b5cbaf Christoph Hellwig 2017-07-06 1132 rwf_t, flags)
f17d8b35452cab Milosz Tanski 2016-03-03 1133 {
f17d8b35452cab Milosz Tanski 2016-03-03 @1134 loff_t pos = ((loff_t)pos_high << 32) | pos_low;
f17d8b35452cab Milosz Tanski 2016-03-03 1135
f17d8b35452cab Milosz Tanski 2016-03-03 1136 if (pos == -1)
3523a9d4547898 Christoph Hellwig 2020-09-25 1137 return do_readv(fd, vec, vlen, flags);
3523a9d4547898 Christoph Hellwig 2020-09-25 1138 return do_preadv(fd, vec, vlen, pos, flags);
72ec35163f9f72 Al Viro 2013-03-20 1139 }
72ec35163f9f72 Al Viro 2013-03-20 1140
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests
Hi Nitesh,
Thank you for the patch! Perhaps something to improve:
[auto build test WARNING on device-mapper-dm/for-next]
[also build test WARNING on linus/master v6.2 next-20230220]
[cannot apply to axboe-block/for-next]
[If your patch is applied to the wrong git tree, kindly drop us a note.
And when submitting patch, we suggest to use '--base' as documented in
https://git-scm.com/docs/git-format-patch#_base_tree_information]
url: https://github.com/intel-lab-lkp/linux/commits/Nitesh-Shetty/block-Add-copy-offload-support-infrastructure/20230220-205057
base: https://git.kernel.org/pub/scm/linux/kernel/git/device-mapper/linux-dm.git for-next
patch link: https://lore.kernel.org/r/20230220105336.3810-5-nj.shetty%40samsung.com
patch subject: [PATCH v7 4/8] fs, block: copy_file_range for def_blk_ops for direct block device.
config: powerpc-allnoconfig (https://download.01.org/0day-ci/archive/20230220/202302202321.zfUe705N-lkp@intel.com/config)
compiler: powerpc-linux-gcc (GCC) 12.1.0
reproduce (this is a W=1 build):
wget https://raw.githubusercontent.com/intel/lkp-tests/master/sbin/make.cross -O ~/bin/make.cross
chmod +x ~/bin/make.cross
# https://github.com/intel-lab-lkp/linux/commit/0f95ad2cb727ac6ac8406a01ff216d9237b403b7
git remote add linux-review https://github.com/intel-lab-lkp/linux
git fetch --no-tags linux-review Nitesh-Shetty/block-Add-copy-offload-support-infrastructure/20230220-205057
git checkout 0f95ad2cb727ac6ac8406a01ff216d9237b403b7
# save the config file
mkdir build_dir && cp config build_dir/.config
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross W=1 O=build_dir ARCH=powerpc olddefconfig
COMPILER_INSTALL_PATH=$HOME/0day COMPILER=gcc-12.1.0 make.cross W=1 O=build_dir ARCH=powerpc SHELL=/bin/bash
If you fix the issue, kindly add following tag where applicable
| Reported-by: kernel test robot <lkp@intel.com>
| Link: https://lore.kernel.org/oe-kbuild-all/202302202321.zfUe705N-lkp@intel.com/
All warnings (new ones prefixed by >>):
>> block/fops.c:614:9: warning: no previous prototype for 'blkdev_copy_file_range' [-Wmissing-prototypes]
614 | ssize_t blkdev_copy_file_range(struct file *file_in, loff_t pos_in,
| ^~~~~~~~~~~~~~~~~~~~~~
vim +/blkdev_copy_file_range +614 block/fops.c
613
> 614 ssize_t blkdev_copy_file_range(struct file *file_in, loff_t pos_in,
615 struct file *file_out, loff_t pos_out,
616 size_t len, unsigned int flags)
617 {
618 struct block_device *in_bdev = I_BDEV(bdev_file_inode(file_in));
619 struct block_device *out_bdev = I_BDEV(bdev_file_inode(file_out));
620 int comp_len;
621
622 comp_len = blkdev_copy_offload(in_bdev, pos_in, out_bdev, pos_out, len,
623 NULL, NULL, GFP_KERNEL);
624 if (comp_len != len)
625 comp_len = generic_copy_file_range(file_in, pos_in + comp_len,
626 file_out, pos_out + comp_len, len - comp_len, flags);
627
628 return comp_len;
629 }
630
--
0-DAY CI Kernel Test Service
https://github.com/intel/lkp-tests
© 2016 - 2025 Red Hat, Inc.