fs/ocfs2/Kconfig | 1 + fs/ocfs2/aops.c | 580 +++++++++++++++++++++++++++++++++++++- fs/ocfs2/aops.h | 26 ++ fs/ocfs2/buffer_head_io.c | 19 +- fs/ocfs2/file.c | 108 ++++++- fs/ocfs2/ocfs2.h | 49 +++- fs/ocfs2/ocfs2_fs.h | 3 + 7 files changed, 753 insertions(+), 33 deletions(-)
This patch set migrates ocfs2 Direct I/O (DIO) from buffer_head mode
to iomap mode.
For patch [2/3], this patch leverages the top 2 bits of `iocb->ki_flags`
(`IOCB_OCFS2_RW_LOCK` and `IOCB_OCFS2_RW_LOCK_LEVEL`) to safely carry
the lock state into the `end_io` callback. If this patch set is accepted,
I plan to submit a follow-up patch to the iomap module to introduce two
dedicated flags for `iocb->ki_flags`, avoiding the need to use the top
two bits internally in iomap.
i.e.:
```
diff --git a/include/linux/fs.h b/include/linux/fs.h
index 50ce731a2b78..ad9fd768746e 100644
--- a/include/linux/fs.h
+++ b/include/linux/fs.h
@@ -357,6 +357,8 @@ struct readahead_control;
/* kiocb is a read or write operation submitted by fs/aio.c. */
#define IOCB_AIO_RW (1 << 22)
#define IOCB_HAS_METADATA (1 << 23)
+#define IOCB_FS_PRIVATE1 (1 << 30)
+#define IOCB_FS_PRIVATE2 (1 << 31)
/* for use in trace events */
#define TRACE_IOCB_STRINGS \
```
----------------
This patch set has been tested on 7.2.0 kernel with the following test
cases.
## ltp
runltp -f dio -d /mnt/ocfs2/
The test results: passed
## xfstests
./check -g quick -T -b -s ocfs2 -e generic/081 \
-e generic/266 -e generic/272 -e generic/281 \
-e generic/338 -e generic/347 -e generic/361 \
-e generic/628 -e generic/629 -e generic/648 -e generic/650
The test results are the same before and after the patch set.
Failures: generic/003 generic/322 generic/329 generic/331 generic/370
generic/376 generic/420 generic/424 generic/448 generic/449 generic/467
generic/477 generic/510 generic/513 generic/537 generic/552 generic/563
generic/578 generic/607 generic/620 generic/630 generic/755 generic/777
generic/784
Failed 24 of 632 tests
## fio on single node
fio --direct=1 --iodepth=32 --rw=write --ioengine=libaio --bs=128k --size=1g \
--numjobs=4 --runtime=100 --group_reporting --filename=/mnt/ocfs2/write.1g --name=tst
- before patch : write: IOPS=18.8k, BW=4708MiB/s (4937MB/s)(4096MiB/870msec)
- applied patch: write: IOPS=39.6k, BW=4953MiB/s (5193MB/s)(4096MiB/827msec)
- overwrite (don't remove write.1g, run above fio command) speed seems same.
fio --direct=1 --iodepth=32 --rw=read --ioengine=libaio --bs=128k --size=1g \
--numjobs=4 --runtime=100 --group_reporting --filename=/mnt/ocfs2/write.1g --name=tst
- before patch : read: IOPS=52.4k, BW=6554MiB/s (6872MB/s)(4096MiB/625msec)
- applied patch: read: IOPS=52.2k, BW=6522MiB/s (6839MB/s)(4096MiB/628msec)
fio --direct=1 --iodepth=32 --rw=randwrite --ioengine=libaio --bs=4k --size=1g \
--numjobs=4 --runtime=100 --group_reporting --filename=/mnt/ocfs2/write.1g --name=tst
- before patch
- creating write: IOPS=20.9k, BW=81.7MiB/s (85.6MB/s)(4096MiB/50159msec)
- overwriting write.1g: write: IOPS=281k, BW=1097MiB/s (1151MB/s)(4096MiB/3733msec)
- applied patch
- creating write: IOPS=21.0k, BW=82.0MiB/s (86.0MB/s)(4096MiB/49938msec)
- overwriting write.1g: write: IOPS=329k, BW=1286MiB/s (1348MB/s)(4096MiB/3185msec)
fio --direct=1 --iodepth=32 --rw=randread --ioengine=libaio --bs=4k --size=1g \
--numjobs=4 --runtime=100 --group_reporting --filename=/mnt/ocfs2/write.1g --name=tst
- before patch : read: IOPS=372k, BW=1452MiB/s (1522MB/s)(4096MiB/2821msec)
- applied patch: read: IOPS=369k, BW=1441MiB/s (1511MB/s)(4096MiB/2842msec)
## fio on two nodes
### write in different folder
node1 /root/fio/diff/ocfs2-w.sh:
> fio --direct=1 --iodepth=32 --rw=write --ioengine=libaio --bs=128k --size=2g \
--numjobs=2 --runtime=100 --group_reporting --filename=/mnt/ocfs2/node1/write.2g --name=tst
node2 /root/fio/diff/ocfs2-w.sh:
> fio --direct=1 --iodepth=32 --rw=write --ioengine=libaio --bs=128k --size=2g \
--numjobs=2 --runtime=100 --group_reporting --filename=/mnt/ocfs2/node2/write.2g --name=tst
node1 /root/fio/diff/ocfs2-randw.sh:
> fio --direct=1 --iodepth=32 --rw=randwrite --ioengine=libaio --bs=4k --size=2g \
--numjobs=2 --runtime=100 --group_reporting --filename=/mnt/ocfs2/node1/write.2g --name=tst
node2 /root/fio/diff/ocfs2-randw.sh:
> fio --direct=1 --iodepth=32 --rw=randwrite --ioengine=libaio --bs=4k --size=2g \
--numjobs=2 --runtime=100 --group_reporting --filename=/mnt/ocfs2/node2/write.2g --name=tst
run on both node,
pdsh -w ssh:tw2510-[1-2] /root/fio/diff/ocfs2-w.sh
- before patch : write: IOPS=19.3k, BW=2417MiB/s (2534MB/s)(4096MiB/1695msec)
- applied patch: write: IOPS=20.8k, BW=2599MiB/s (2725MB/s)(4096MiB/1576msec)
pdsh -w ssh:tw2510-[1-2] /root/fio/diff/ocfs2-randw.sh
- before patch : write: IOPS=3742, BW=14.6MiB/s (15.3MB/s)(1462MiB/100007msec)
- applied patch: write: IOPS=3765, BW=14.7MiB/s (15.4MB/s)(1471MiB/100010msec)
### write in same file
node1 & node2: (/root/fio/share/ocfs2-w.sh)
fio --direct=1 --iodepth=32 --rw=write --ioengine=libaio --bs=128k --size=2g \
--numjobs=2 --runtime=100 --group_reporting --filename=/mnt/ocfs2/share/write.2g --name=tst
node1 & node2: (/root/fio/share/ocfs2-randw.sh)
fio --direct=1 --iodepth=32 --rw=randwrite --ioengine=libaio --bs=4k --size=2g \
--numjobs=2 --runtime=100 --group_reporting --filename=/mnt/ocfs2/share/write.2g --name=tst
run on both node
pdsh -w ssh:tw2510-[1-2] /root/fio/share/ocfs2-w.sh
- before patch : write: IOPS=2052, BW=257MiB/s (269MB/s)(4096MiB/15967msec)
- applied patch: write: IOPS=2558, BW=320MiB/s (335MB/s)(4096MiB/12808msec)
pdsh -w ssh:tw2510-[1-2] /root/fio/share/ocfs2-randw.sh
- before patch : write: IOPS=3588, BW=14.0MiB/s (14.7MB/s)(1402MiB/100001msec)
- applied patch: write: IOPS=3590, BW=14.0MiB/s (14.7MB/s)(1403MiB/100001msec)
Heming Zhao (3):
ocfs2: Add new ocfs2_map_blocks() to introduce iomap feature
ocfs2: switch dio read path from buffer_head to iomap
ocfs2: switch dio write path from buffer_head to iomap
fs/ocfs2/Kconfig | 1 +
fs/ocfs2/aops.c | 580 +++++++++++++++++++++++++++++++++++++-
fs/ocfs2/aops.h | 26 ++
fs/ocfs2/buffer_head_io.c | 19 +-
fs/ocfs2/file.c | 108 ++++++-
fs/ocfs2/ocfs2.h | 49 +++-
fs/ocfs2/ocfs2_fs.h | 3 +
7 files changed, 753 insertions(+), 33 deletions(-)
--
2.54.0As part of migrating OCFS2 DIO read/write code paths towards the modern
and high-performant iomap framework, this patch introduces an iomap API
to replace old high-overhead VFS buffer_head structure paths.
This patch establishes the foundational block mapping routines required by
subsequent iomap integration patches. The implementation draws
inspiration from ext4_map_blocks().
Signed-off-by: Heming Zhao <heming.zhao@suse.com>
---
fs/ocfs2/aops.c | 94 +++++++++++++++++++++++++++++++++++++++
fs/ocfs2/aops.h | 2 +
fs/ocfs2/buffer_head_io.c | 12 -----
fs/ocfs2/ocfs2.h | 45 ++++++++++++++++++-
4 files changed, 140 insertions(+), 13 deletions(-)
diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c
index 4acdbb70882c..08df5e3b5196 100644
--- a/fs/ocfs2/aops.c
+++ b/fs/ocfs2/aops.c
@@ -126,6 +126,100 @@ static int ocfs2_lock_get_block(struct inode *inode, sector_t iblock,
return ret;
}
+int ocfs2_map_blocks(struct inode *inode, struct ocfs2_map_block *map,
+ int flags)
+{
+ int err = 0;
+ unsigned int ext_flags;
+ u64 max_blocks = map->len;
+ u64 p_blkno, count, past_eof;
+ struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
+ int create = flags & OCFS2_GET_BLOCKS_CREATE;
+
+ if (OCFS2_I(inode)->ip_flags & OCFS2_INODE_SYSTEM_FILE)
+ mlog(ML_NOTICE, "map_block on system inode 0x%p (%llu)\n",
+ inode, inode->i_ino);
+
+ if (S_ISLNK(inode->i_mode)) {
+ /*
+ * TODO: refer ocfs2_get_block() to handle
+ * ocfs2_read_folio in the future
+ */
+ mlog(ML_NOTICE, "map_block on S_ISLNK file, node 0x%p (%llu)\n",
+ inode, inode->i_ino);
+ dump_stack();
+ goto bail;
+ }
+
+ err = ocfs2_extent_map_get_blocks(inode, map->lblk, &p_blkno, &count,
+ &ext_flags);
+ if (err) {
+ mlog(ML_ERROR, "get_blocks() failed, inode: 0x%p, "
+ "block: %llu\n", inode, map->lblk);
+ goto bail;
+ }
+
+ if (max_blocks < count)
+ count = max_blocks;
+
+ map->pblk = p_blkno;
+ map->len = count;
+
+ /*
+ * ocfs2 never allocates in this function - the only time we
+ * need to use MAP_NEW is when we're extending i_size on a file
+ * system which doesn't support holes, in which case MAP_NEW
+ * allows __block_write_begin() to zero.
+ *
+ * If we see this on a sparse file system, then a truncate has
+ * raced us and removed the cluster. In this case, we clear
+ * the buffers dirty and uptodate bits and let the buffer code
+ * ignore it as a hole.
+ */
+ if (create && map->pblk == 0 && ocfs2_sparse_alloc(osb)) {
+ map->flags &= ~(OCFS2_MAP_DIRTY | OCFS2_MAP_UPTODATE);
+ goto bail;
+ }
+
+ if (p_blkno) {
+ if (ext_flags & OCFS2_EXT_UNWRITTEN) {
+ map->flags |= OCFS2_MAP_UNWRITTEN;
+ } else if (!(ext_flags & OCFS2_EXT_UNWRITTEN)) {
+ /* Treat the unwritten extent as a hole for zeroing purposes. */
+ map->flags |= OCFS2_MAP_MAPPED;
+ } else {
+ /* nothing to do */
+ }
+ }
+
+ if (!ocfs2_sparse_alloc(osb)) {
+ if (map->pblk == 0) {
+ err = -EIO;
+ mlog(ML_ERROR,
+ "iblock = %llu p_blkno = %llu blkno=(%llu)\n",
+ (unsigned long long)map->lblk,
+ (unsigned long long)map->pblk,
+ (unsigned long long)OCFS2_I(inode)->ip_blkno);
+ mlog(ML_ERROR, "Size %llu, clusters %u\n",
+ (unsigned long long)i_size_read(inode),
+ OCFS2_I(inode)->ip_clusters);
+ dump_stack();
+ goto bail;
+ }
+ }
+
+ past_eof = ocfs2_blocks_for_bytes(inode->i_sb, i_size_read(inode));
+
+ if (create && (map->lblk >= past_eof))
+ map->flags |= OCFS2_MAP_NEW;
+
+bail:
+ if (err < 0)
+ return -EIO;
+ else
+ return map->len;
+}
+
int ocfs2_get_block(struct inode *inode, sector_t iblock,
struct buffer_head *bh_result, int create)
{
diff --git a/fs/ocfs2/aops.h b/fs/ocfs2/aops.h
index 114efc9111e4..8dd6edd7c1a1 100644
--- a/fs/ocfs2/aops.h
+++ b/fs/ocfs2/aops.h
@@ -42,6 +42,8 @@ int ocfs2_size_fits_inline_data(struct buffer_head *di_bh, u64 new_size);
int ocfs2_get_block(struct inode *inode, sector_t iblock,
struct buffer_head *bh_result, int create);
+int ocfs2_map_blocks(struct inode *inode, struct ocfs2_map_block *map,
+ int flags);
/* all ocfs2_dio_end_io()'s fault */
#define ocfs2_iocb_is_rw_locked(iocb) \
test_bit(0, (unsigned long *)&iocb->private)
diff --git a/fs/ocfs2/buffer_head_io.c b/fs/ocfs2/buffer_head_io.c
index 7bfe377af2df..493f2209cca5 100644
--- a/fs/ocfs2/buffer_head_io.c
+++ b/fs/ocfs2/buffer_head_io.c
@@ -23,18 +23,6 @@
#include "buffer_head_io.h"
#include "ocfs2_trace.h"
-/*
- * Bits on bh->b_state used by ocfs2.
- *
- * These MUST be after the JBD2 bits. Hence, we use BH_JBDPrivateStart.
- */
-enum ocfs2_state_bits {
- BH_NeedsValidate = BH_JBDPrivateStart,
-};
-
-/* Expand the magic b_state functions */
-BUFFER_FNS(NeedsValidate, needs_validate);
-
int ocfs2_write_block(struct ocfs2_super *osb, struct buffer_head *bh,
struct ocfs2_caching_info *ci)
{
diff --git a/fs/ocfs2/ocfs2.h b/fs/ocfs2/ocfs2.h
index 62cad6522c7a..095f7ae5dded 100644
--- a/fs/ocfs2/ocfs2.h
+++ b/fs/ocfs2/ocfs2.h
@@ -509,7 +509,50 @@ struct ocfs2_super
struct ocfs2_filecheck_sysfs_entry osb_fc_ent;
};
-#define OCFS2_SB(sb) ((struct ocfs2_super *)(sb)->s_fs_info)
+/*
+ * Bits on bh->b_state used by ocfs2.
+ *
+ * These MUST be after the JBD2 bits. Hence, we use BH_JBDPrivateStart.
+ */
+enum ocfs2_state_bits {
+ BH_NeedsValidate = BH_JBDPrivateStart,
+};
+
+/* Expand the magic b_state functions */
+BUFFER_FNS(NeedsValidate, needs_validate);
+
+/*
+ * Logical to physical block mapping, used by ocfs2_map_blocks()
+ *
+ * This structure is used to pass requests into ocfs2_map_blocks() as
+ * well as to store the information returned by ocfs2_map_blocks(). It
+ * takes less room on the stack than a struct buffer_head.
+ */
+#define OCFS2_MAP_NEW BIT(BH_New)
+#define OCFS2_MAP_MAPPED BIT(BH_Mapped)
+#define OCFS2_MAP_UNWRITTEN BIT(BH_Unwritten)
+/* useless? #define OCFS2_MAP_BOUNDARY BIT(BH_Boundary) */
+/* useless? #define OCFS2_MAP_DELAYED BIT(BH_Delay) */
+#define OCFS2_MAP_DIRTY BIT(BH_Dirty)
+#define OCFS2_MAP_UPTODATE BIT(BH_Uptodate)
+#define OCFS2_MAP_NEEDS_VALIDATE BIT(BH_NeedsValidate)
+#define OCFS2_MAP_DEFER_COMPLETION BIT(BH_Defer_Completion)
+#define OCFS2_MAP_FLAGS (OCFS2_MAP_NEW | OCFS2_MAP_MAPPED |\
+ OCFS2_MAP_DIRTY | OCFS2_MAP_UPTODATE |\
+ OCFS2_MAP_NEEDS_VALIDATE |\
+ OCFS2_MAP_DEFER_COMPLETION)
+
+struct ocfs2_map_block {
+ u64 pblk; /* physical block# */
+ u64 lblk; /* logical block# */
+ u64 len; /* number of block */
+ unsigned int flags;
+};
+
+/* Flags used by ocfs2_map_blocks() */
+#define OCFS2_GET_BLOCKS_CREATE (0x0001)
+
+#define OCFS2_SB(sb) ((struct ocfs2_super *)(sb)->s_fs_info)
/* Useful typedef for passing around journal access functions */
typedef int (*ocfs2_journal_access_func)(handle_t *handle,
--
2.54.0This patch migrates the DIO read path from the legacy buffer_head
infrastructure to the modern and more efficient iomap framework.
One thing need to care: The generic iomap layer clears
`iocb->private` during its internal execution. To preserve the OCFS2
rw lock state across the DIO lifecycle, we leverage the top 2 bits
of `iocb->ki_flags` (`IOCB_OCFS2_RW_LOCK` and `IOCB_OCFS2_RW_LOCK_LEVEL`)
to safely carry the lock state into the `end_io` callback.
Signed-off-by: Heming Zhao <heming.zhao@suse.com>
---
fs/ocfs2/Kconfig | 1 +
fs/ocfs2/aops.c | 101 ++++++++++++++++++++++++++++++++++++++++++++
fs/ocfs2/aops.h | 24 +++++++++++
fs/ocfs2/file.c | 53 ++++++++++++++++++++---
fs/ocfs2/ocfs2.h | 3 ++
fs/ocfs2/ocfs2_fs.h | 3 ++
6 files changed, 179 insertions(+), 6 deletions(-)
diff --git a/fs/ocfs2/Kconfig b/fs/ocfs2/Kconfig
index 2514d36cbe01..bf1678a5eb01 100644
--- a/fs/ocfs2/Kconfig
+++ b/fs/ocfs2/Kconfig
@@ -7,6 +7,7 @@ config OCFS2_FS
select CRC32
select QUOTA
select QUOTA_TREE
+ select FS_IOMAP
select FS_POSIX_ACL
select LEGACY_DIRECT_IO
help
diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c
index 08df5e3b5196..59d445dd61f6 100644
--- a/fs/ocfs2/aops.c
+++ b/fs/ocfs2/aops.c
@@ -4,6 +4,7 @@
*/
#include <linux/fs.h>
+#include <linux/iomap.h>
#include <linux/slab.h>
#include <linux/highmem.h>
#include <linux/pagemap.h>
@@ -2556,6 +2557,106 @@ static ssize_t ocfs2_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
ocfs2_dio_end_io, 0);
}
+static int ocfs2_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
+ unsigned int flags, struct iomap *iomap, struct iomap *srcmap)
+{
+ int ret;
+ struct ocfs2_map_block map;
+ struct ocfs2_inode_info *oi = OCFS2_I(inode);
+ u8 blkbits = inode->i_blkbits;
+
+ if ((offset >> blkbits) > OCFS2_MAX_LOGICAL_BLOCK)
+ return -EINVAL;
+
+ /*
+ * Calculate the first and last logical blocks respectively.
+ */
+ map.lblk = offset >> blkbits;
+ map.len = min_t(loff_t, (offset + length - 1) >> blkbits,
+ OCFS2_MAX_LOGICAL_BLOCK) - map.lblk + 1;
+ map.flags = 0;
+
+ if (flags & IOMAP_WRITE) {
+ /* todo */
+ } else {
+ down_read(&oi->ip_alloc_sem);
+ ret = ocfs2_map_blocks(inode, &map, 0);
+ up_read(&oi->ip_alloc_sem);
+ }
+
+ if (ret < 0)
+ return ret;
+
+ /*
+ * Before returning to iomap, let's ensure the allocated mapping
+ * covers the entire requested length for atomic writes.
+ */
+ if (flags & IOMAP_ATOMIC) {
+ if (map.len < (length >> blkbits)) {
+ WARN_ON_ONCE(1);
+ return -EINVAL;
+ }
+ }
+
+ /* may use ocfs2_set_iomap() to replace all blow code */
+ iomap->bdev = inode->i_sb->s_bdev;
+ iomap->offset = (u64)map.lblk << blkbits;
+ iomap->length = (u64)map.len << blkbits;
+
+ if (map.pblk == 0) {
+ iomap->type = IOMAP_HOLE;
+ iomap->addr = IOMAP_NULL_ADDR;
+ } else {
+ if (map.flags & OCFS2_MAP_UNWRITTEN) {
+ iomap->type = IOMAP_UNWRITTEN;
+ } else if (map.flags & OCFS2_MAP_MAPPED) {
+ iomap->type = IOMAP_MAPPED;
+ } else {
+ WARN_ON_ONCE(1);
+ return -EIO;
+ }
+ iomap->addr = (sector_t)map.pblk << blkbits;
+ }
+
+ if (map.flags & OCFS2_MAP_NEW)
+ iomap->flags |= IOMAP_F_NEW;
+
+ return 0;
+}
+
+const struct iomap_ops ocfs2_iomap_ops = {
+ .iomap_begin = ocfs2_iomap_begin,
+};
+
+static int ocfs2_dio_read_end_io(struct kiocb *iocb, ssize_t size, int error,
+ unsigned int flags)
+{
+ struct inode *inode = file_inode(iocb->ki_filp);
+ int level;
+
+ if (error) {
+ mlog_ratelimited(ML_ERROR, "Direct IO failed, bytes = %lld errno:%d",
+ (long long)size, error);
+ goto bail;
+ }
+
+ BUG_ON(!ocfs2_iomap_iocb_is_rw_locked(iocb));
+
+ level = ocfs2_iomap_iocb_rw_locked_level(iocb);
+
+ /* sync dio unlock job done in .read_iter or .write_iter */
+ if (ocfs2_iomap_iocb_is_rw_locked(iocb))
+ ocfs2_rw_unlock(inode, level);
+
+ ocfs2_iocb_clear_rw_locked(iocb);
+ ocfs2_iomap_iocb_clear_rw_locked(iocb);
+bail:
+ return error;
+}
+
+const struct iomap_dio_ops ocfs2_dio_r_ops = {
+ .end_io = ocfs2_dio_read_end_io,
+};
const struct address_space_operations ocfs2_aops = {
.dirty_folio = block_dirty_folio,
.read_folio = ocfs2_read_folio,
diff --git a/fs/ocfs2/aops.h b/fs/ocfs2/aops.h
index 8dd6edd7c1a1..5966fa725135 100644
--- a/fs/ocfs2/aops.h
+++ b/fs/ocfs2/aops.h
@@ -74,4 +74,28 @@ enum ocfs2_iocb_lock_bits {
#define ocfs2_iocb_rw_locked_level(iocb) \
test_bit(OCFS2_IOCB_RW_LOCK_LEVEL, (unsigned long *)&iocb->private)
+/*
+ * Since iomap clear iocb->private in some cases, we use iocb->ki_flags
+ * top 2-bit to store the rw lock state. Hope ki_flags never touch these 2-bit.
+ */
+#define IOCB_OCFS2_RW_LOCK (1 << 30)
+#define IOCB_OCFS2_RW_LOCK_LEVEL (1 << 31)
+static inline void ocfs2_iomap_iocb_set_rw_locked(struct kiocb *iocb, int level)
+{
+ iocb->ki_flags |= IOCB_OCFS2_RW_LOCK;
+ if (level)
+ iocb->ki_flags |= IOCB_OCFS2_RW_LOCK_LEVEL;
+ else
+ iocb->ki_flags &= ~IOCB_OCFS2_RW_LOCK_LEVEL;
+}
+
+#define ocfs2_iomap_iocb_is_rw_locked(iocb) \
+ (iocb->ki_flags & IOCB_OCFS2_RW_LOCK)
+#define ocfs2_iomap_iocb_init_rw_locked(iocb) \
+ (iocb->ki_flags &= ~(IOCB_OCFS2_RW_LOCK | IOCB_OCFS2_RW_LOCK_LEVEL))
+#define ocfs2_iomap_iocb_clear_rw_locked(iocb) \
+ (iocb->ki_flags &= ~IOCB_OCFS2_RW_LOCK)
+#define ocfs2_iomap_iocb_rw_locked_level(iocb) \
+ ((iocb->ki_flags & IOCB_OCFS2_RW_LOCK_LEVEL) ? 1 : 0)
+
#endif /* OCFS2_FILE_H */
diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c
index d6e977ba6565..e879312ff86b 100644
--- a/fs/ocfs2/file.c
+++ b/fs/ocfs2/file.c
@@ -9,6 +9,7 @@
#include <linux/capability.h>
#include <linux/fs.h>
+#include <linux/iomap.h>
#include <linux/types.h>
#include <linux/slab.h>
#include <linux/highmem.h>
@@ -2374,6 +2375,29 @@ static int ocfs2_prepare_inode_for_write(struct file *file,
return ret;
}
+static bool ocfs2_should_use_dio(struct kiocb *iocb, struct iov_iter *iter,
+ struct inode *inode)
+{
+ unsigned int dio_align = bdev_logical_block_size(inode->i_sb->s_bdev);
+
+ /*
+ * Fallback to buffered I/O if we see an inode without
+ * extents.
+ */
+ if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL)
+ return false;
+
+ /*
+ * Allow Direct I/O for any write requests aligned to the underlying
+ * logical sector size (e.g., 512 bytes).
+ */
+ if (IS_ALIGNED(iocb->ki_pos | iov_iter_alignment(iter), dio_align)) {
+ return true;
+ } else {
+ return false;
+ }
+}
+
static ssize_t ocfs2_file_write_iter(struct kiocb *iocb,
struct iov_iter *from)
{
@@ -2548,7 +2572,6 @@ static ssize_t ocfs2_file_read_iter(struct kiocb *iocb,
filp->f_path.dentry->d_name.name,
to->nr_segs); /* GRRRRR */
-
if (!inode) {
ret = -EINVAL;
mlog_errno(ret);
@@ -2558,26 +2581,37 @@ static ssize_t ocfs2_file_read_iter(struct kiocb *iocb,
if (!direct_io && nowait)
return -EOPNOTSUPP;
+ if (!iov_iter_count(to))
+ return 0; /* skip atime */
+
+ /*
+ * the following two lines do the same thing. Once iomap completely
+ * replaces buffer_head, we can remove ocfs2_iocb_init_rw_locked(),
+ * ocfs2_iocb_clear_rw_locked() and ocfs2_iocb_rw_locked_level().
+ */
ocfs2_iocb_init_rw_locked(iocb);
+ ocfs2_iomap_iocb_init_rw_locked(iocb);
/*
* buffered reads protect themselves in ->read_folio(). O_DIRECT reads
* need locks to protect pending reads from racing with truncate.
*/
if (direct_io) {
+ rw_level = 0;
if (nowait)
- ret = ocfs2_try_rw_lock(inode, 0);
+ ret = ocfs2_try_rw_lock(inode, rw_level);
else
- ret = ocfs2_rw_lock(inode, 0);
+ ret = ocfs2_rw_lock(inode, rw_level);
if (ret < 0) {
if (ret != -EAGAIN)
mlog_errno(ret);
goto bail;
}
- rw_level = 0;
/* communicate with ocfs2_dio_end_io */
ocfs2_iocb_set_rw_locked(iocb, rw_level);
+ /* communicate with ocfs2_dio_read_end_io */
+ ocfs2_iomap_iocb_set_rw_locked(iocb, rw_level);
}
/*
@@ -2598,7 +2632,12 @@ static ssize_t ocfs2_file_read_iter(struct kiocb *iocb,
}
ocfs2_inode_unlock(inode, lock_level);
- ret = generic_file_read_iter(iocb, to);
+ if (direct_io && ocfs2_should_use_dio(iocb, to, inode)) {
+ ret = iomap_dio_rw(iocb, to, &ocfs2_iomap_ops, &ocfs2_dio_r_ops, 0, NULL, 0);
+ } else {
+ iocb->ki_flags &= ~IOCB_DIRECT;
+ ret = generic_file_read_iter(iocb, to);
+ }
trace_generic_file_read_iter_ret(ret);
/* buffered aio wouldn't have proper lock coverage today */
@@ -2610,8 +2649,10 @@ static ssize_t ocfs2_file_read_iter(struct kiocb *iocb,
}
bail:
- if (rw_level != -1)
+ if (rw_level != -1) {
ocfs2_rw_unlock(inode, rw_level);
+ ocfs2_iomap_iocb_clear_rw_locked(iocb);
+ }
return ret;
}
diff --git a/fs/ocfs2/ocfs2.h b/fs/ocfs2/ocfs2.h
index 095f7ae5dded..9397947583fb 100644
--- a/fs/ocfs2/ocfs2.h
+++ b/fs/ocfs2/ocfs2.h
@@ -549,6 +549,9 @@ struct ocfs2_map_block {
unsigned int flags;
};
+extern const struct iomap_ops ocfs2_iomap_ops;
+extern const struct iomap_dio_ops ocfs2_dio_r_ops;
+
/* Flags used by ocfs2_map_blocks() */
#define OCFS2_GET_BLOCKS_CREATE (0x0001)
diff --git a/fs/ocfs2/ocfs2_fs.h b/fs/ocfs2/ocfs2_fs.h
index c501eb3cdcda..000bd014acbd 100644
--- a/fs/ocfs2/ocfs2_fs.h
+++ b/fs/ocfs2/ocfs2_fs.h
@@ -314,6 +314,9 @@
*/
#define OCFS2_CLUSTER_O2CB_GLOBAL_HEARTBEAT (0x01)
+/* Max logical block we can support */
+#define OCFS2_MAX_LOGICAL_BLOCK (0xFFFFFFFE)
+
struct ocfs2_system_inode_info {
char *si_name;
int si_iflags;
--
2.54.0This patch converts OCFS2's DIO write path from the legacy
buffer_head infrastructure to the modern iomap framework.
Key modifications and designs are as follows:
1. Dynamic Context Allocation:
Refactor 'struct ocfs2_write_ctxt' to use a flexible array 'w_desc[]'
instead of a fixed-size array. Dynamically allocate the context based on
the write length ('w_clen') in 'ocfs2_alloc_write_ctxt()'. This prevents
static limits overflow and optimizes kernel heap memory utilization.
2. Robust Mapping and Limits:
Introduce 'ocfs2_dio_wr_map_blocks()' to allocate and map direct write
blocks. Implement a 1 MiB cap ('OCFS2_DIO_WR_MAX_MAX_BYTES') per mapping
call to restrict allocation granularity, preventing JBD2 transaction
credit exhaustion during huge asynchronous sequential writes.
3. Reliable Completion Work and Fallback:
- Implement 'ocfs2_iomap_dio_end_io_write()' to handle metadata completion.
It converts UNWRITTEN extents, updates inode size (EOF), and deletes the
inode from the orphan directory if it was appended.
- Implement 'ocfs2_dio_write_end_io()' to finalize the dio lifecycle and
release cluster locks safely.
- Intercept '-ENOTBLK' errors from 'iomap_dio_rw()' caused by page cache
invalidation failures (due to mmap/buffered collisions). Gracefully clear
the error, strip the IOCB_DIRECT flag, and fall back to buffered write
4. Moved ocfs2_add_inode_to_orphan():
- moved ocfs2_add_inode_to_orphan() from ocfs2_dio_wr_map_blocks() to
ocfs2_file_write_iter().
5. Uncertain logic in code
For the following code block in ocfs2_dio_wr_map_blocks():
```
if (extend) {
if (ocfs2_sparse_alloc(osb))
ret = ocfs2_zero_tail(inode, di_bh, pos);
else
ret = ocfs2_expand_nonsparse_inode(inode, di_bh, pos,
map_len, NULL);
if (ret < 0) {
mlog_errno(ret);
goto unlock;
}
}
```
I am not completely certain whether it is called only once per
ocfs2_file_write_iter(), but I believe calling it multiple times will
not introduce any side effects. Furthermore, testing across various
scenarios showed no instances of multiple calls.
Assisted-by: Gemini:gemini-3.5-flash
Assisted-by: Claude:claude-sonnet-4-5
Signed-off-by: Heming Zhao <heming.zhao@suse.com>
---
fs/ocfs2/aops.c | 387 +++++++++++++++++++++++++++++++++++++-
fs/ocfs2/buffer_head_io.c | 7 +-
fs/ocfs2/file.c | 55 +++++-
fs/ocfs2/ocfs2.h | 1 +
4 files changed, 435 insertions(+), 15 deletions(-)
diff --git a/fs/ocfs2/aops.c b/fs/ocfs2/aops.c
index 59d445dd61f6..60bf0973f38d 100644
--- a/fs/ocfs2/aops.c
+++ b/fs/ocfs2/aops.c
@@ -764,6 +764,8 @@ int ocfs2_map_folio_blocks(struct folio *folio, u64 *p_blkno,
#define OCFS2_MAX_CTXT_PAGES (OCFS2_MAX_CLUSTERSIZE / PAGE_SIZE)
#endif
+#define OCFS2_DIO_WR_MAX_MAP_BYTES (1U << 20) /* 1 MiB */
+
#define OCFS2_MAX_CLUSTERS_PER_PAGE (PAGE_SIZE / OCFS2_MIN_CLUSTERSIZE)
struct ocfs2_unwritten_extent {
@@ -799,8 +801,6 @@ struct ocfs2_write_ctxt {
/* Type of caller. Must be one of buffer, mmap, direct. */
ocfs2_write_type_t w_type;
- struct ocfs2_write_cluster_desc w_desc[OCFS2_MAX_CLUSTERS_PER_PAGE];
-
/*
* This is true if page_size > cluster_size.
*
@@ -850,6 +850,8 @@ struct ocfs2_write_ctxt {
struct list_head w_unwritten_list;
unsigned int w_unwritten_count;
+
+ struct ocfs2_write_cluster_desc w_desc[];
};
void ocfs2_unlock_and_free_folios(struct folio **folios, int num_folios)
@@ -917,17 +919,24 @@ static int ocfs2_alloc_write_ctxt(struct ocfs2_write_ctxt **wcp,
unsigned len, ocfs2_write_type_t type,
struct buffer_head *di_bh)
{
- u32 cend;
+ u32 w_cpos, cend, w_clen;
struct ocfs2_write_ctxt *wc;
+ size_t alloc_size;
+
+ w_cpos = pos >> osb->s_clustersize_bits;
+ cend = (pos + len - 1) >> osb->s_clustersize_bits;
+ w_clen = cend - w_cpos + 1;
- wc = kzalloc_obj(struct ocfs2_write_ctxt, GFP_NOFS);
+ alloc_size = sizeof(struct ocfs2_write_ctxt) +
+ w_clen * sizeof(struct ocfs2_write_cluster_desc);
+
+ wc = kzalloc(alloc_size, GFP_NOFS);
if (!wc)
return -ENOMEM;
- wc->w_cpos = pos >> osb->s_clustersize_bits;
+ wc->w_cpos = w_cpos;
wc->w_first_new_cpos = UINT_MAX;
- cend = (pos + len - 1) >> osb->s_clustersize_bits;
- wc->w_clen = cend - wc->w_cpos + 1;
+ wc->w_clen = w_clen;
get_bh(di_bh);
wc->w_di_bh = di_bh;
wc->w_type = type;
@@ -2218,7 +2227,7 @@ static int ocfs2_dio_wr_get_block(struct inode *inode, sector_t iblock,
struct ocfs2_write_cluster_desc *desc = NULL;
struct ocfs2_dio_write_ctxt *dwc = NULL;
struct buffer_head *di_bh = NULL;
- u64 p_blkno;
+ u64 p_blkno = 0;
unsigned int i_blkbits = inode->i_sb->s_blocksize_bits;
loff_t pos = iblock << i_blkbits;
sector_t endblk = (i_size_read(inode) - 1) >> i_blkbits;
@@ -2358,6 +2367,148 @@ static int ocfs2_dio_wr_get_block(struct inode *inode, sector_t iblock,
return ret;
}
+/* copy from ocfs2_dio_wr_get_block */
+static int ocfs2_dio_wr_map_blocks(struct inode *inode,
+ struct ocfs2_map_block *map, int create)
+{
+ struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
+ struct ocfs2_inode_info *oi = OCFS2_I(inode);
+ struct ocfs2_write_ctxt *wc;
+ struct ocfs2_write_cluster_desc *desc = NULL;
+ struct buffer_head *di_bh = NULL;
+ u64 p_blkno;
+ sector_t iblock = map->lblk;
+ unsigned int i_blkbits = inode->i_sb->s_blocksize_bits;
+ loff_t pos = iblock << i_blkbits;
+ sector_t endblk = (i_size_read(inode) - 1) >> i_blkbits;
+ unsigned int total_len = map->len << i_blkbits;
+ unsigned int map_len;
+ unsigned int cont_clusters = 1;
+ int ret = 0, extend = 0, idx;
+
+ /*
+ * map->len is count in ocfs2_iomap_begin according to write
+ * "pos" and "end", we need map twice to return different buffer state:
+ * 1. area in file size, not set NEW;
+ * 2. area out file size, set NEW.
+ *
+ * iblock endblk
+ * |--------|---------|---------|---------
+ * |<-------area in file------->|
+ */
+ if ((iblock <= endblk) &&
+ ((iblock + ((total_len - 1) >> i_blkbits)) > endblk))
+ total_len = (endblk - iblock + 1) << i_blkbits;
+
+ /*
+ * Bound how much we allocate/prepare (and journal) per call. map->len
+ * below still reports only the physically-contiguous prefix to iomap;
+ * iomap calls us again for the remainder.
+ */
+ map_len = min(total_len, OCFS2_DIO_WR_MAX_MAP_BYTES);
+
+ mlog(0, "get block of %llu at %llu:%u req %u\n",
+ inode->i_ino, pos, map_len, total_len);
+
+ /*
+ * Because we need to change file size in ocfs2_iomap_dio_end_io_write(),
+ * or we may need to add it to orphan dir. So can not fall to fast path
+ * while file size will be changed.
+ */
+ if (pos + total_len <= i_size_read(inode)) {
+ /* This is the fast path for re-write. */
+ ret = ocfs2_map_blocks(inode, map, OCFS2_GET_BLOCKS_CREATE);
+ if ((map->flags & OCFS2_MAP_MAPPED) &&
+ !(map->flags & OCFS2_MAP_NEW) && ret == 0)
+ goto out;
+
+ /* Clear state set by ocfs2_map_blocks. */
+ map->flags = 0;
+ } else {
+ extend = 1;
+ }
+
+ ret = ocfs2_inode_lock(inode, &di_bh, 1);
+ if (ret) {
+ mlog_errno(ret);
+ goto out;
+ }
+
+ down_write(&oi->ip_alloc_sem);
+
+ if (extend) {
+ if (ocfs2_sparse_alloc(osb))
+ ret = ocfs2_zero_tail(inode, di_bh, pos);
+ else
+ ret = ocfs2_expand_nonsparse_inode(inode, di_bh, pos,
+ map_len, NULL);
+ if (ret < 0) {
+ mlog_errno(ret);
+ goto unlock;
+ }
+ }
+
+ ret = ocfs2_write_begin_nolock(inode->i_mapping, pos, map_len,
+ OCFS2_WRITE_DIRECT, NULL,
+ (void **)&wc, di_bh, NULL);
+ if (ret) {
+ mlog_errno(ret);
+ goto unlock;
+ }
+
+ desc = &wc->w_desc[0];
+
+ p_blkno = ocfs2_clusters_to_blocks(inode->i_sb, desc->c_phys);
+ BUG_ON(p_blkno == 0);
+ p_blkno += iblock & (u64)(ocfs2_clusters_to_blocks(inode->i_sb, 1) - 1);
+
+ /*
+ * Calculate physically contiguous clusters by checking descriptor arrays.
+ * This strictly prevents map->len from expanding past non-contiguous physical bounds.
+ */
+ for (idx = 1; idx < wc->w_clen; idx++) {
+ if (wc->w_desc[idx].c_phys == wc->w_desc[idx - 1].c_phys + 1 &&
+ wc->w_desc[idx].c_needs_zero == wc->w_desc[idx - 1].c_needs_zero) {
+ cont_clusters++;
+ } else {
+ break;
+ }
+ }
+ map->pblk = p_blkno;
+ /*
+ * Only report the mapping as UNWRITTEN when the cluster is newly
+ * allocated or was already unwritten (desc->c_needs_zero). Reporting
+ * an already-written cluster as UNWRITTEN makes the iomap direct-io
+ * core zero the sub-block head/tail padding, which destroys existing
+ * data when a sub-block write lands inside a written cluster.
+ */
+ map->flags |= OCFS2_MAP_MAPPED;
+ if (desc->c_needs_zero)
+ map->flags |= OCFS2_MAP_UNWRITTEN;
+ map->len = cont_clusters * ocfs2_clusters_to_blocks(inode->i_sb, 1);
+
+ if (desc->c_needs_zero)
+ map->flags |= OCFS2_MAP_NEW;
+
+ if (iblock > endblk)
+ map->flags |= OCFS2_MAP_NEW;
+
+ map->flags |= OCFS2_MAP_DEFER_COMPLETION;
+
+ ocfs2_free_unwritten_list(inode, &wc->w_unwritten_list);
+ ret = ocfs2_write_end_nolock(inode->i_mapping, pos, map_len, map_len, wc);
+ BUG_ON(ret != map_len);
+ ret = 0;
+
+unlock:
+ up_write(&oi->ip_alloc_sem);
+ ocfs2_inode_unlock(inode, 1);
+ brelse(di_bh);
+
+out:
+ return ret;
+}
+
static int ocfs2_dio_end_io_write(struct inode *inode,
struct ocfs2_dio_write_ctxt *dwc,
loff_t offset,
@@ -2525,6 +2676,8 @@ static int ocfs2_dio_end_io(struct kiocb *iocb,
level = ocfs2_iocb_rw_locked_level(iocb);
ocfs2_rw_unlock(inode, level);
+ /* match iomap rw lock action */
+ ocfs2_iomap_iocb_clear_rw_locked(iocb);
return ret;
}
@@ -2557,6 +2710,12 @@ static ssize_t ocfs2_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
ocfs2_dio_end_io, 0);
}
+static int ocfs2_iomap_alloc(struct inode *inode, struct ocfs2_map_block *map,
+ unsigned int flags)
+{
+ return ocfs2_dio_wr_map_blocks(inode, map, 1);
+}
+
static int ocfs2_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
unsigned int flags, struct iomap *iomap, struct iomap *srcmap)
{
@@ -2564,6 +2723,7 @@ static int ocfs2_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
struct ocfs2_map_block map;
struct ocfs2_inode_info *oi = OCFS2_I(inode);
u8 blkbits = inode->i_blkbits;
+ unsigned int orig_mlen;
if ((offset >> blkbits) > OCFS2_MAX_LOGICAL_BLOCK)
return -EINVAL;
@@ -2575,9 +2735,32 @@ static int ocfs2_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
map.len = min_t(loff_t, (offset + length - 1) >> blkbits,
OCFS2_MAX_LOGICAL_BLOCK) - map.lblk + 1;
map.flags = 0;
+ orig_mlen = map.len;
if (flags & IOMAP_WRITE) {
- /* todo */
+ /*
+ * We check here if the blocks are already allocated, then we
+ * don't need to start a journal txn and we can directly return
+ * the mapping information. This could boost performance
+ * especially in multi-threaded overwrite requests.
+ */
+ if (offset + length <= i_size_read(inode)) {
+ down_read(&oi->ip_alloc_sem);
+ ret = ocfs2_map_blocks(inode, &map, 0);
+ up_read(&oi->ip_alloc_sem);
+ /*
+ * For atomic writes the entire requested length should
+ * be mapped.
+ */
+ if (map.flags & OCFS2_MAP_MAPPED) {
+ if ((!(flags & IOMAP_ATOMIC) && ret > 0) ||
+ (flags & IOMAP_ATOMIC && ret >= orig_mlen)) {
+ goto out;
+ }
+ }
+ map.len = orig_mlen;
+ }
+ ret = ocfs2_iomap_alloc(inode, &map, flags);
} else {
down_read(&oi->ip_alloc_sem);
ret = ocfs2_map_blocks(inode, &map, 0);
@@ -2587,6 +2770,7 @@ static int ocfs2_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
if (ret < 0)
return ret;
+out:
/*
* Before returning to iomap, let's ensure the allocated mapping
* covers the entire requested length for atomic writes.
@@ -2657,6 +2841,191 @@ static int ocfs2_dio_read_end_io(struct kiocb *iocb, ssize_t size, int error,
const struct iomap_dio_ops ocfs2_dio_r_ops = {
.end_io = ocfs2_dio_read_end_io,
};
+
+/* copy from ocfs2_dio_end_io_write */
+static int ocfs2_iomap_dio_end_io_write(struct inode *inode,
+ loff_t offset,
+ ssize_t bytes)
+{
+ struct ocfs2_cached_dealloc_ctxt dealloc;
+ struct ocfs2_extent_tree et;
+ struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
+ struct ocfs2_inode_info *oi = OCFS2_I(inode);
+ struct buffer_head *di_bh = NULL;
+ struct ocfs2_dinode *di;
+ struct ocfs2_alloc_context *data_ac = NULL;
+ struct ocfs2_alloc_context *meta_ac = NULL;
+ handle_t *handle = NULL;
+ loff_t end = offset + bytes;
+ int ret = 0, credits = 0;
+ struct ocfs2_map_block map;
+ unsigned int blkbits = inode->i_blkbits;
+ unsigned int max_blocks;
+ unsigned int ue_cpos = 0, ue_phys = 0, ue_len = 0;
+ unsigned int curr_lblk, end_lblk;
+
+ map.lblk = offset >> blkbits;
+ max_blocks = (bytes + offset) >> osb->s_clustersize_bits;
+
+ ocfs2_init_dealloc_ctxt(&dealloc);
+
+ ret = ocfs2_inode_lock(inode, &di_bh, 1);
+ if (ret < 0) {
+ mlog_errno(ret);
+ goto out;
+ }
+
+ down_write(&oi->ip_alloc_sem);
+
+ di = (struct ocfs2_dinode *)di_bh->b_data;
+
+ ocfs2_init_dinode_extent_tree(&et, INODE_CACHE(inode), di_bh);
+
+ /* Attach dealloc with extent tree in case that we may reuse extents
+ * which are already unlinked from current extent tree due to extent
+ * rotation and merging.
+ */
+ et.et_dealloc = &dealloc;
+
+ ret = ocfs2_lock_allocators(inode, &et, 0, (bytes >> osb->s_clustersize_bits)*2,
+ &data_ac, &meta_ac);
+ if (ret) {
+ mlog_errno(ret);
+ goto unlock;
+ }
+
+ credits = ocfs2_calc_extend_credits(inode->i_sb, &di->id2.i_list);
+
+ handle = ocfs2_start_trans(osb, credits);
+ if (IS_ERR(handle)) {
+ ret = PTR_ERR(handle);
+ mlog_errno(ret);
+ goto unlock;
+ }
+ ret = ocfs2_journal_access_di(handle, INODE_CACHE(inode), di_bh,
+ OCFS2_JOURNAL_ACCESS_WRITE);
+ if (ret) {
+ mlog_errno(ret);
+ goto commit;
+ }
+
+ curr_lblk = offset >> blkbits;
+ /*
+ * Round the end up so the final partial block (sub-block direct I/O)
+ * is included; otherwise the last, partially-written cluster is left
+ * unwritten and reads back as zero.
+ */
+ end_lblk = (offset + bytes + (1 << blkbits) - 1) >> blkbits;
+ while (ret >= 0 && curr_lblk < end_lblk) {
+ memset(&map, 0, sizeof(map));
+ map.lblk += curr_lblk;
+ map.len = end_lblk - curr_lblk;
+
+ ret = ocfs2_assure_trans_credits(handle, credits);
+ if (ret < 0) {
+ mlog_errno(ret);
+ break;
+ }
+
+ ret = ocfs2_map_blocks(inode, &map, 0);
+ if (ret < 0) {
+ mlog_errno(ret);
+ break;
+ }
+ if (ret == 0)
+ break;
+
+ if (map.flags & OCFS2_MAP_UNWRITTEN) {
+ ue_cpos = ocfs2_blocks_to_clusters(inode->i_sb, map.lblk);
+ ue_phys = ocfs2_blocks_to_clusters(inode->i_sb, map.pblk);
+ ue_len = ocfs2_clusters_for_blocks(inode->i_sb,
+ map.lblk + map.len) - ue_cpos;
+
+ ret = ocfs2_mark_extent_written(inode, &et, handle,
+ ue_cpos, ue_len,
+ ue_phys,
+ meta_ac, &dealloc);
+ if (ret < 0) {
+ mlog_errno(ret);
+ break;
+ }
+ }
+ curr_lblk += map.len;
+ }
+
+ if (end > i_size_read(inode)) {
+ ret = ocfs2_set_inode_size(handle, inode, di_bh, end);
+ if (ret < 0)
+ mlog_errno(ret);
+ }
+
+commit:
+ ocfs2_commit_trans(osb, handle);
+
+unlock:
+ up_write(&oi->ip_alloc_sem);
+
+ if (data_ac)
+ ocfs2_free_alloc_context(data_ac);
+ if (meta_ac)
+ ocfs2_free_alloc_context(meta_ac);
+
+ down_write(&oi->ip_alloc_sem);
+ if (!ret && (di->i_flags & cpu_to_le32(OCFS2_DIO_ORPHANED_FL))) {
+ ret = ocfs2_del_inode_from_orphan(osb, inode, di_bh, 0, 0);
+ if (ret < 0)
+ mlog_errno(ret);
+ }
+ up_write(&oi->ip_alloc_sem);
+ ocfs2_inode_unlock(inode, 1);
+ brelse(di_bh);
+
+out:
+ ocfs2_run_deallocs(osb, &dealloc);
+ return ret;
+}
+
+static int ocfs2_dio_write_end_io(struct kiocb *iocb, ssize_t size, int error,
+ unsigned int flags)
+{
+ struct inode *inode = file_inode(iocb->ki_filp);
+ loff_t offset = iocb->ki_pos;
+ int level;
+ int ret = 0;
+
+ if (error) {
+ mlog_ratelimited(ML_ERROR, "Direct IO failed, bytes = %lld errno:%d",
+ (long long)size, error);
+ goto bail;
+ }
+
+ BUG_ON(!ocfs2_iomap_iocb_is_rw_locked(iocb));
+
+
+ if (size && ((flags & IOMAP_DIO_UNWRITTEN) ||
+ (offset + size > i_size_read(inode)))) {
+ ret = ocfs2_iomap_dio_end_io_write(inode, offset, size);
+ if (ret < 0)
+ mlog_errno(ret);
+ }
+
+ level = ocfs2_iomap_iocb_rw_locked_level(iocb);
+
+ /* sync dio unlock job done in .read_iter or .write_iter */
+ if (ocfs2_iomap_iocb_is_rw_locked(iocb))
+ ocfs2_rw_unlock(inode, level);
+
+ ocfs2_iocb_clear_rw_locked(iocb);
+ ocfs2_iomap_iocb_clear_rw_locked(iocb);
+
+bail:
+ return error;
+}
+
+const struct iomap_dio_ops ocfs2_dio_w_ops = {
+ .end_io = ocfs2_dio_write_end_io,
+};
+
const struct address_space_operations ocfs2_aops = {
.dirty_folio = block_dirty_folio,
.read_folio = ocfs2_read_folio,
diff --git a/fs/ocfs2/buffer_head_io.c b/fs/ocfs2/buffer_head_io.c
index 493f2209cca5..03906ab737ec 100644
--- a/fs/ocfs2/buffer_head_io.c
+++ b/fs/ocfs2/buffer_head_io.c
@@ -332,6 +332,8 @@ int ocfs2_read_blocks(struct ocfs2_caching_info *ci, u64 block, int nr,
wait_on_buffer(bh);
put_bh(bh);
bhs[i] = NULL;
+ } else if (bh && buffer_uptodate(bh)) {
+ clear_buffer_uptodate(bh);
}
continue;
}
@@ -360,11 +362,8 @@ int ocfs2_read_blocks(struct ocfs2_caching_info *ci, u64 block, int nr,
BUG_ON(buffer_jbd(bh));
clear_buffer_needs_validate(bh);
status = validate(sb, bh);
- if (status) {
- if (buffer_uptodate(bh))
- clear_buffer_uptodate(bh);
+ if (status)
goto read_failure;
- }
}
}
diff --git a/fs/ocfs2/file.c b/fs/ocfs2/file.c
index e879312ff86b..f84e5e3ed391 100644
--- a/fs/ocfs2/file.c
+++ b/fs/ocfs2/file.c
@@ -48,6 +48,7 @@
#include "quota.h"
#include "refcounttree.h"
#include "ocfs2_trace.h"
+#include "namei.h"
#include "buffer_head_io.h"
@@ -2378,6 +2379,7 @@ static int ocfs2_prepare_inode_for_write(struct file *file,
static bool ocfs2_should_use_dio(struct kiocb *iocb, struct iov_iter *iter,
struct inode *inode)
{
+ struct ocfs2_super *osb = OCFS2_SB(inode->i_sb);
unsigned int dio_align = bdev_logical_block_size(inode->i_sb->s_bdev);
/*
@@ -2387,6 +2389,11 @@ static bool ocfs2_should_use_dio(struct kiocb *iocb, struct iov_iter *iter,
if (OCFS2_I(inode)->ip_dyn_features & OCFS2_INLINE_DATA_FL)
return false;
+ /* Fallback to buffered I/O if we do not support append dio. */
+ if (iocb->ki_pos + iter->count > i_size_read(inode) &&
+ !ocfs2_supports_append_dio(osb))
+ return false;
+
/*
* Allow Direct I/O for any write requests aligned to the underlying
* logical sector size (e.g., 512 bytes).
@@ -2415,6 +2422,8 @@ static ssize_t ocfs2_file_write_iter(struct kiocb *iocb,
i_size_read(inode) ? 1 : 0);
int direct_io = iocb->ki_flags & IOCB_DIRECT ? 1 : 0;
int nowait = iocb->ki_flags & IOCB_NOWAIT ? 1 : 0;
+ int dio_flags = 0;
+ ssize_t buffered;
trace_ocfs2_file_write_iter(inode, file, file->f_path.dentry,
(unsigned long long)OCFS2_I(inode)->ip_blkno,
@@ -2434,7 +2443,9 @@ static ssize_t ocfs2_file_write_iter(struct kiocb *iocb,
} else
inode_lock(inode);
+ /* check the comments in ocfs2_file_read_iter() */
ocfs2_iocb_init_rw_locked(iocb);
+ ocfs2_iomap_iocb_init_rw_locked(iocb);
/*
* Concurrent O_DIRECT writes are allowed with
@@ -2501,8 +2512,46 @@ static ssize_t ocfs2_file_write_iter(struct kiocb *iocb,
/* communicate with ocfs2_dio_end_io */
ocfs2_iocb_set_rw_locked(iocb, rw_level);
+ /* communicate with ocfs2_dio_write_end_io */
+ ocfs2_iomap_iocb_set_rw_locked(iocb, rw_level);
+
+ if (direct_io && ocfs2_should_use_dio(iocb, from, inode)) {
+ if (ocfs2_clusters_for_bytes(inode->i_sb, iocb->ki_pos + count) >
+ ocfs2_clusters_for_bytes(inode->i_sb, i_size_read(inode))) {
+ /*
+ * when we are going to alloc extents beyond file size, add the
+ * inode to orphan dir, so we can recall those spaces when
+ * system crashed during write.
+ */
+ ret = ocfs2_add_inode_to_orphan(osb, inode);
+ if (ret < 0) {
+ mlog_errno(ret);
+ goto out;
+ }
+ dio_flags = IOMAP_DIO_FORCE_WAIT;
+ }
+
+ written = iomap_dio_rw(iocb, from, &ocfs2_iomap_ops, &ocfs2_dio_w_ops,
+ dio_flags, NULL, 0);
+ /*
+ * iomap_dio_rw() returns -ENOTBLK when it could not invalidate
+ * the page cache for the DIO range (e.g. racing buffered/mmap
+ * I/O on the same file, as exercised by generic/095). Fall back
+ * to a buffered write for the remaining data, the same way
+ * ext4/xfs do, instead of leaking -ENOTBLK to userspace.
+ */
+ if (written == -ENOTBLK)
+ written = 0;
+ if (written >= 0 && iov_iter_count(from)) {
+ iocb->ki_flags &= ~IOCB_DIRECT;
+ buffered = __generic_file_write_iter(iocb, from);
+ written = direct_write_fallback(iocb, from, written, buffered);
+ }
+ } else {
+ iocb->ki_flags &= ~IOCB_DIRECT;
+ written = __generic_file_write_iter(iocb, from);
+ }
- written = __generic_file_write_iter(iocb, from);
/* buffered aio wouldn't have proper lock coverage today */
BUG_ON(written == -EIOCBQUEUED && !direct_io);
@@ -2546,8 +2595,10 @@ static ssize_t ocfs2_file_write_iter(struct kiocb *iocb,
if (saved_ki_complete)
xchg(&iocb->ki_complete, saved_ki_complete);
- if (rw_level != -1)
+ if (rw_level != -1) {
ocfs2_rw_unlock(inode, rw_level);
+ ocfs2_iomap_iocb_clear_rw_locked(iocb);
+ }
out_mutex:
inode_unlock(inode);
diff --git a/fs/ocfs2/ocfs2.h b/fs/ocfs2/ocfs2.h
index 9397947583fb..7a7a2ecdaeaf 100644
--- a/fs/ocfs2/ocfs2.h
+++ b/fs/ocfs2/ocfs2.h
@@ -551,6 +551,7 @@ struct ocfs2_map_block {
extern const struct iomap_ops ocfs2_iomap_ops;
extern const struct iomap_dio_ops ocfs2_dio_r_ops;
+extern const struct iomap_dio_ops ocfs2_dio_w_ops;
/* Flags used by ocfs2_map_blocks() */
#define OCFS2_GET_BLOCKS_CREATE (0x0001)
--
2.54.0© 2016 - 2026 Red Hat, Inc.