Add xfs_file_dio_write_atomic() for dedicated handling of atomic writes.

In case of -EAGAIN being returned from iomap_dio_rw(), reissue the write
in CoW-based atomic write mode.

In the CoW-based atomic write mode, first unshare blocks so that we don't
have a cow fork for the data in the range which we are writing.

Signed-off-by: John Garry <john.g.garry@oracle.com>
---
 fs/xfs/xfs_file.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 51 insertions(+)

diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c
index fd05b66aea3f..12af5cdc3094 100644
--- a/fs/xfs/xfs_file.c
+++ b/fs/xfs/xfs_file.c
@@ -619,6 +619,55 @@ xfs_file_dio_write_aligned(
 	return ret;
 }
 
+static noinline ssize_t
+xfs_file_dio_write_atomic(
+	struct xfs_inode	*ip,
+	struct kiocb		*iocb,
+	struct iov_iter		*from)
+{
+	unsigned int		iolock = XFS_IOLOCK_SHARED;
+	bool			use_cow = false;
+	unsigned int		dio_flags;
+	ssize_t			ret;
+
+retry:
+	ret = xfs_ilock_iocb_for_write(iocb, &iolock);
+	if (ret)
+		return ret;
+
+	ret = xfs_file_write_checks(iocb, from, &iolock);
+	if (ret)
+		goto out_unlock;
+
+	if (use_cow) {
+		ret = xfs_reflink_unshare(ip, iocb->ki_pos,
+			iov_iter_count(from));
+		if (ret)
+			goto out_unlock;
+	}
+
+	trace_xfs_file_direct_write(iocb, from);
+	if (use_cow)
+		dio_flags = IOMAP_DIO_ATOMIC_COW;
+	else
+		dio_flags = 0;
+
+	ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
+			&xfs_dio_write_ops, dio_flags, NULL, 0);
+
+	if (ret == -EAGAIN && !(iocb->ki_flags & IOCB_NOWAIT) && !use_cow) {
+		xfs_iunlock(ip, iolock);
+		iolock = XFS_IOLOCK_EXCL;
+		use_cow = true;
+		goto retry;
+	}
+
+out_unlock:
+	if (iolock)
+		xfs_iunlock(ip, iolock);
+	return ret;
+}
+
 /*
  * Handle block unaligned direct I/O writes
  *
@@ -723,6 +772,8 @@ xfs_file_dio_write(
 		return -EINVAL;
 	if ((iocb->ki_pos | count) & ip->i_mount->m_blockmask)
 		return xfs_file_dio_write_unaligned(ip, iocb, from);
+	if (iocb->ki_flags & IOCB_ATOMIC)
+		return xfs_file_dio_write_atomic(ip, iocb, from);
 	return xfs_file_dio_write_aligned(ip, iocb, from);
 }
 
-- 
2.31.1

On 04/02/2025 12:01, John Garry wrote:
> +static noinline ssize_t
> +xfs_file_dio_write_atomic(
> +	struct xfs_inode	*ip,
> +	struct kiocb		*iocb,
> +	struct iov_iter		*from)
> +{
> +	unsigned int		iolock = XFS_IOLOCK_SHARED;
> +	bool			use_cow = false;
> +	unsigned int		dio_flags;
> +	ssize_t			ret;
> +
> +retry:
> +	ret = xfs_ilock_iocb_for_write(iocb, &iolock);
> +	if (ret)
> +		return ret;
> +
> +	ret = xfs_file_write_checks(iocb, from, &iolock);
> +	if (ret)
> +		goto out_unlock;
> +
> +	if (use_cow) {
> +		ret = xfs_reflink_unshare(ip, iocb->ki_pos,
> +			iov_iter_count(from));
> +		if (ret)
> +			goto out_unlock;
> +	}
> +
> +	trace_xfs_file_direct_write(iocb, from);
> +	if (use_cow)
> +		dio_flags = IOMAP_DIO_ATOMIC_COW;

note to self:

We should also set IOMAP_DIO_FORCE_WAIT and call inode_dio_wait() here, 
similar to xfs_file_dio_write_unaligned()

> +	else
> +		dio_flags = 0;
> +
> +	ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
> +			&xfs_dio_write_ops, dio_flags, NULL, 0);
> +
> +	if (ret == -EAGAIN && !(iocb->ki_flags & IOCB_NOWAIT) && !use_cow) {
> +		xfs_iunlock(ip, iolock);
> +		iolock = XFS_IOLOCK_EXCL;
> +		use_cow = true;
> +		goto retry;
> +	}
> +
> +out_unlock:
> +	if (iolock)
> +		xfs_iunlock(ip, iolock);
> +	return ret;
> +}
> +

On Tue, Feb 04, 2025 at 12:01:24PM +0000, John Garry wrote:
> Add xfs_file_dio_write_atomic() for dedicated handling of atomic writes.
> 
> In case of -EAGAIN being returned from iomap_dio_rw(), reissue the write
> in CoW-based atomic write mode.
> 
> In the CoW-based atomic write mode, first unshare blocks so that we don't
> have a cow fork for the data in the range which we are writing.
> 
> Signed-off-by: John Garry <john.g.garry@oracle.com>
> ---
>  fs/xfs/xfs_file.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++
>  1 file changed, 51 insertions(+)
> 
> diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c
> index fd05b66aea3f..12af5cdc3094 100644
> --- a/fs/xfs/xfs_file.c
> +++ b/fs/xfs/xfs_file.c
> @@ -619,6 +619,55 @@ xfs_file_dio_write_aligned(
>  	return ret;
>  }
>  
> +static noinline ssize_t
> +xfs_file_dio_write_atomic(
> +	struct xfs_inode	*ip,
> +	struct kiocb		*iocb,
> +	struct iov_iter		*from)
> +{
> +	unsigned int		iolock = XFS_IOLOCK_SHARED;
> +	bool			use_cow = false;
> +	unsigned int		dio_flags;
> +	ssize_t			ret;
> +
> +retry:
> +	ret = xfs_ilock_iocb_for_write(iocb, &iolock);
> +	if (ret)
> +		return ret;
> +
> +	ret = xfs_file_write_checks(iocb, from, &iolock);
> +	if (ret)
> +		goto out_unlock;
> +
> +	if (use_cow) {
> +		ret = xfs_reflink_unshare(ip, iocb->ki_pos,
> +			iov_iter_count(from));

Nit: continuation lines should be indented two tabs:

		ret = xfs_reflink_unshare(ip, iocb->ki_pos,
				iov_iter_count(from));

> +		if (ret)
> +			goto out_unlock;
> +	}
> +
> +	trace_xfs_file_direct_write(iocb, from);
> +	if (use_cow)
> +		dio_flags = IOMAP_DIO_ATOMIC_COW;
> +	else
> +		dio_flags = 0;

I also think you could eliminate use_cow by initializing dio_flags to
zero at the top, OR'ing in IOMAP_DIO_ATOMIC_COW in the retry clause
below, and using (dio_flags & IOMAP_DIO_ATOMIC_COW) to determine if you
should call unshare above.

Note: This serializes all the software untorn direct writes.  I think
a more performant solution would allocate the cow staging blocks ondisk,
attach them to the directio ioend context, and alter ->iomap_begin and
the ioend remap to use the attached blocks, but that's a lot more
surgery.

--D

> +
> +	ret = iomap_dio_rw(iocb, from, &xfs_direct_write_iomap_ops,
> +			&xfs_dio_write_ops, dio_flags, NULL, 0);
> +
> +	if (ret == -EAGAIN && !(iocb->ki_flags & IOCB_NOWAIT) && !use_cow) {
> +		xfs_iunlock(ip, iolock);
> +		iolock = XFS_IOLOCK_EXCL;
> +		use_cow = true;
> +		goto retry;
> +	}
> +
> +out_unlock:
> +	if (iolock)
> +		xfs_iunlock(ip, iolock);
> +	return ret;
> +}
> +
>  /*
>   * Handle block unaligned direct I/O writes
>   *
> @@ -723,6 +772,8 @@ xfs_file_dio_write(
>  		return -EINVAL;
>  	if ((iocb->ki_pos | count) & ip->i_mount->m_blockmask)
>  		return xfs_file_dio_write_unaligned(ip, iocb, from);
> +	if (iocb->ki_flags & IOCB_ATOMIC)
> +		return xfs_file_dio_write_atomic(ip, iocb, from);
>  	return xfs_file_dio_write_aligned(ip, iocb, from);
>  }
>  
> -- 
> 2.31.1
> 
>

On 05/02/2025 19:55, Darrick J. Wong wrote:
> On Tue, Feb 04, 2025 at 12:01:24PM +0000, John Garry wrote:
>> Add xfs_file_dio_write_atomic() for dedicated handling of atomic writes.
>>
>> In case of -EAGAIN being returned from iomap_dio_rw(), reissue the write
>> in CoW-based atomic write mode.
>>
>> In the CoW-based atomic write mode, first unshare blocks so that we don't
>> have a cow fork for the data in the range which we are writing.
>>
>> Signed-off-by: John Garry <john.g.garry@oracle.com>
>> ---
>>   fs/xfs/xfs_file.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++
>>   1 file changed, 51 insertions(+)
>>
>> diff --git a/fs/xfs/xfs_file.c b/fs/xfs/xfs_file.c
>> index fd05b66aea3f..12af5cdc3094 100644
>> --- a/fs/xfs/xfs_file.c
>> +++ b/fs/xfs/xfs_file.c
>> @@ -619,6 +619,55 @@ xfs_file_dio_write_aligned(
>>   	return ret;
>>   }
>>   
>> +static noinline ssize_t
>> +xfs_file_dio_write_atomic(
>> +	struct xfs_inode	*ip,
>> +	struct kiocb		*iocb,
>> +	struct iov_iter		*from)
>> +{
>> +	unsigned int		iolock = XFS_IOLOCK_SHARED;
>> +	bool			use_cow = false;
>> +	unsigned int		dio_flags;
>> +	ssize_t			ret;
>> +
>> +retry:
>> +	ret = xfs_ilock_iocb_for_write(iocb, &iolock);
>> +	if (ret)
>> +		return ret;
>> +
>> +	ret = xfs_file_write_checks(iocb, from, &iolock);
>> +	if (ret)
>> +		goto out_unlock;
>> +
>> +	if (use_cow) {
>> +		ret = xfs_reflink_unshare(ip, iocb->ki_pos,
>> +			iov_iter_count(from));
> 
> Nit: continuation lines should be indented two tabs:
> 
> 		ret = xfs_reflink_unshare(ip, iocb->ki_pos,
> 				iov_iter_count(from));

ok

> 
>> +		if (ret)
>> +			goto out_unlock;
>> +	}
>> +
>> +	trace_xfs_file_direct_write(iocb, from);
>> +	if (use_cow)
>> +		dio_flags = IOMAP_DIO_ATOMIC_COW;
>> +	else
>> +		dio_flags = 0;
> 
> I also think you could eliminate use_cow by initializing dio_flags to
> zero at the top, OR'ing in IOMAP_DIO_ATOMIC_COW in the retry clause
> below, and using (dio_flags & IOMAP_DIO_ATOMIC_COW) to determine if you
> should call unshare above.

ok, fine, if you think that it is better

> 
> Note: This serializes all the software untorn direct writes.  I think
> a more performant solution would allocate the cow staging blocks ondisk,
> attach them to the directio ioend context, and alter ->iomap_begin and
> the ioend remap to use the attached blocks, but that's a lot more
> surgery.

sure, that does sound like it's quite intrusive. But whatever we do I 
would like to keep the behaviour that racing reads and atomic writes 
mean that a read sees all old or all new data. That is how SCSI and NVMe 
behaves, even though it is not an advertised atomic write feature.

Thanks,
John