Memory errors are at least somewhat more likely on disaggregated memory
than on-board memory. This commit registers to be notified by fsdev_dax
in the event that a memory failure is detected.
When a file access resolves to a daxdev with memory errors, it will fail
with an appropriate error.
If a daxdev failed fs_dax_get(), we set dd->dax_err. If a daxdev called
our notify_failure(), set dd->error. When any of the above happens, set
(file)->error and stop allowing access.
In general, the recovery from memory errors is to unmount the file
system and re-initialize the memory, but there may be usable degraded
modes of operation - particularly in the future when famfs supports
file systems backed by more than one daxdev. In those cases,
accessing data that is on a working daxdev can still work.
For now, return errors for any file that has encountered a memory or dax
error.
Signed-off-by: John Groves <john@groves.net>
---
fs/fuse/famfs.c | 115 +++++++++++++++++++++++++++++++++++++++---
fs/fuse/famfs_kfmap.h | 3 +-
2 files changed, 109 insertions(+), 9 deletions(-)
diff --git a/fs/fuse/famfs.c b/fs/fuse/famfs.c
index c02b14789c6e..4eb87c5c628e 100644
--- a/fs/fuse/famfs.c
+++ b/fs/fuse/famfs.c
@@ -20,6 +20,26 @@
#include "famfs_kfmap.h"
#include "fuse_i.h"
+static void famfs_set_daxdev_err(
+ struct fuse_conn *fc, struct dax_device *dax_devp);
+
+static int
+famfs_dax_notify_failure(struct dax_device *dax_devp, u64 offset,
+ u64 len, int mf_flags)
+{
+ struct fuse_conn *fc = dax_holder(dax_devp);
+
+ famfs_set_daxdev_err(fc, dax_devp);
+
+ return 0;
+}
+
+static const struct dax_holder_operations famfs_fuse_dax_holder_ops = {
+ .notify_failure = famfs_dax_notify_failure,
+};
+
+/*****************************************************************************/
+
/*
* famfs_teardown()
*
@@ -48,9 +68,12 @@ famfs_teardown(struct fuse_conn *fc)
if (!dd->valid)
continue;
- /* Release reference from dax_dev_get() */
- if (dd->devp)
+ /* Only call fs_put_dax if fs_dax_get succeeded */
+ if (dd->devp) {
+ if (!dd->dax_err)
+ fs_put_dax(dd->devp, fc);
put_dax(dd->devp);
+ }
kfree(dd->name);
}
@@ -174,6 +197,17 @@ famfs_fuse_get_daxdev(struct fuse_mount *fm, const u64 index)
goto out;
}
+ err = fs_dax_get(daxdev->devp, fc, &famfs_fuse_dax_holder_ops);
+ if (err) {
+ /* If fs_dax_get() fails, we don't attempt recovery;
+ * We mark the daxdev valid with dax_err
+ */
+ daxdev->dax_err = 1;
+ pr_err("%s: fs_dax_get(%lld) failed\n",
+ __func__, (u64)daxdev->devno);
+ err = -EBUSY;
+ }
+
daxdev->name = kstrdup(daxdev_out.name, GFP_KERNEL);
wmb(); /* all daxdev fields must be visible before marking it valid */
daxdev->valid = 1;
@@ -254,6 +288,38 @@ famfs_update_daxdev_table(
return 0;
}
+static void
+famfs_set_daxdev_err(
+ struct fuse_conn *fc,
+ struct dax_device *dax_devp)
+{
+ int i;
+
+ /* Gotta search the list by dax_devp;
+ * read lock because we're not adding or removing daxdev entries
+ */
+ down_read(&fc->famfs_devlist_sem);
+ for (i = 0; i < fc->dax_devlist->nslots; i++) {
+ if (fc->dax_devlist->devlist[i].valid) {
+ struct famfs_daxdev *dd = &fc->dax_devlist->devlist[i];
+
+ if (dd->devp != dax_devp)
+ continue;
+
+ dd->error = true;
+ up_read(&fc->famfs_devlist_sem);
+
+ pr_err("%s: memory error on daxdev %s (%d)\n",
+ __func__, dd->name, i);
+ goto done;
+ }
+ }
+ up_read(&fc->famfs_devlist_sem);
+ pr_err("%s: memory err on unrecognized daxdev\n", __func__);
+
+done:
+}
+
/***************************************************************************/
void
@@ -611,6 +677,26 @@ famfs_file_init_dax(
static ssize_t famfs_file_bad(struct inode *inode);
+static int famfs_dax_err(struct famfs_daxdev *dd)
+{
+ if (!dd->valid) {
+ pr_err("%s: daxdev=%s invalid\n",
+ __func__, dd->name);
+ return -EIO;
+ }
+ if (dd->dax_err) {
+ pr_err("%s: daxdev=%s dax_err\n",
+ __func__, dd->name);
+ return -EIO;
+ }
+ if (dd->error) {
+ pr_err("%s: daxdev=%s memory error\n",
+ __func__, dd->name);
+ return -EHWPOISON;
+ }
+ return 0;
+}
+
static int
famfs_interleave_fileofs_to_daxofs(struct inode *inode, struct iomap *iomap,
loff_t file_offset, off_t len, unsigned int flags)
@@ -648,6 +734,7 @@ famfs_interleave_fileofs_to_daxofs(struct inode *inode, struct iomap *iomap,
/* Is the data is in this striped extent? */
if (local_offset < ext_size) {
+ struct famfs_daxdev *dd;
u64 chunk_num = local_offset / chunk_size;
u64 chunk_offset = local_offset % chunk_size;
u64 stripe_num = chunk_num / nstrips;
@@ -656,6 +743,7 @@ famfs_interleave_fileofs_to_daxofs(struct inode *inode, struct iomap *iomap,
u64 strip_offset = chunk_offset + (stripe_num * chunk_size);
u64 strip_dax_ofs = fei->ie_strips[strip_num].ext_offset;
u64 strip_devidx = fei->ie_strips[strip_num].dev_index;
+ int rc;
if (strip_devidx >= fc->dax_devlist->nslots) {
pr_err("%s: strip_devidx %llu >= nslots %d\n",
@@ -670,6 +758,15 @@ famfs_interleave_fileofs_to_daxofs(struct inode *inode, struct iomap *iomap,
goto err_out;
}
+ dd = &fc->dax_devlist->devlist[strip_devidx];
+
+ rc = famfs_dax_err(dd);
+ if (rc) {
+ /* Shut down access to this file */
+ meta->error = true;
+ return rc;
+ }
+
iomap->addr = strip_dax_ofs + strip_offset;
iomap->offset = file_offset;
iomap->length = min_t(loff_t, len, chunk_remainder);
@@ -767,6 +864,7 @@ famfs_fileofs_to_daxofs(struct inode *inode, struct iomap *iomap,
if (local_offset < dax_ext_len) {
loff_t ext_len_remainder = dax_ext_len - local_offset;
struct famfs_daxdev *dd;
+ int rc;
if (daxdev_idx >= fc->dax_devlist->nslots) {
pr_err("%s: daxdev_idx %llu >= nslots %d\n",
@@ -777,11 +875,11 @@ famfs_fileofs_to_daxofs(struct inode *inode, struct iomap *iomap,
dd = &fc->dax_devlist->devlist[daxdev_idx];
- if (!dd->valid || dd->error) {
- pr_err("%s: daxdev=%lld %s\n", __func__,
- daxdev_idx,
- dd->valid ? "error" : "invalid");
- goto err_out;
+ rc = famfs_dax_err(dd);
+ if (rc) {
+ /* Shut down access to this file */
+ meta->error = true;
+ return rc;
}
/*
@@ -966,7 +1064,8 @@ famfs_file_bad(struct inode *inode)
return -EIO;
}
if (meta->error) {
- pr_debug("%s: previously detected metadata errors\n", __func__);
+ pr_debug("%s: previously detected metadata errors\n",
+ __func__);
return -EIO;
}
if (i_size != meta->file_size) {
diff --git a/fs/fuse/famfs_kfmap.h b/fs/fuse/famfs_kfmap.h
index e76b9057a1e0..6a6420bdff48 100644
--- a/fs/fuse/famfs_kfmap.h
+++ b/fs/fuse/famfs_kfmap.h
@@ -73,7 +73,8 @@ struct famfs_file_meta {
struct famfs_daxdev {
/* Include dev uuid? */
bool valid;
- bool error;
+ bool error; /* Dax has reported a memory error (probably poison) */
+ bool dax_err; /* fs_dax_get() failed */
dev_t devno;
struct dax_device *devp;
char *name;
--
2.49.0On Wed, 7 Jan 2026 09:33:27 -0600
John Groves <John@Groves.net> wrote:
> Memory errors are at least somewhat more likely on disaggregated memory
> than on-board memory. This commit registers to be notified by fsdev_dax
> in the event that a memory failure is detected.
>
> When a file access resolves to a daxdev with memory errors, it will fail
> with an appropriate error.
>
> If a daxdev failed fs_dax_get(), we set dd->dax_err. If a daxdev called
> our notify_failure(), set dd->error. When any of the above happens, set
> (file)->error and stop allowing access.
>
> In general, the recovery from memory errors is to unmount the file
> system and re-initialize the memory, but there may be usable degraded
> modes of operation - particularly in the future when famfs supports
> file systems backed by more than one daxdev. In those cases,
> accessing data that is on a working daxdev can still work.
>
> For now, return errors for any file that has encountered a memory or dax
> error.
>
> Signed-off-by: John Groves <john@groves.net>
> ---
> fs/fuse/famfs.c | 115 +++++++++++++++++++++++++++++++++++++++---
> fs/fuse/famfs_kfmap.h | 3 +-
> 2 files changed, 109 insertions(+), 9 deletions(-)
>
> diff --git a/fs/fuse/famfs.c b/fs/fuse/famfs.c
> index c02b14789c6e..4eb87c5c628e 100644
> --- a/fs/fuse/famfs.c
> +++ b/fs/fuse/famfs.c
> @@ -254,6 +288,38 @@ famfs_update_daxdev_table(
> return 0;
> }
>
> +static void
> +famfs_set_daxdev_err(
> + struct fuse_conn *fc,
> + struct dax_device *dax_devp)
> +{
> + int i;
> +
> + /* Gotta search the list by dax_devp;
> + * read lock because we're not adding or removing daxdev entries
> + */
> + down_read(&fc->famfs_devlist_sem);
Use a guard()
> + for (i = 0; i < fc->dax_devlist->nslots; i++) {
> + if (fc->dax_devlist->devlist[i].valid) {
> + struct famfs_daxdev *dd = &fc->dax_devlist->devlist[i];
> +
> + if (dd->devp != dax_devp)
> + continue;
> +
> + dd->error = true;
> + up_read(&fc->famfs_devlist_sem);
> +
> + pr_err("%s: memory error on daxdev %s (%d)\n",
> + __func__, dd->name, i);
> + goto done;
> + }
> + }
> + up_read(&fc->famfs_devlist_sem);
> + pr_err("%s: memory err on unrecognized daxdev\n", __func__);
> +
> +done:
If this isn't getting more interesting, just return above.
> +}
> +
> /***************************************************************************/
>
> void
> @@ -611,6 +677,26 @@ famfs_file_init_dax(
>
> static ssize_t famfs_file_bad(struct inode *inode);
>
> +static int famfs_dax_err(struct famfs_daxdev *dd)
I'd introduce this earlier in the series to reduce need
to refactor below.
> +{
> + if (!dd->valid) {
> + pr_err("%s: daxdev=%s invalid\n",
> + __func__, dd->name);
> + return -EIO;
> + }
> + if (dd->dax_err) {
> + pr_err("%s: daxdev=%s dax_err\n",
> + __func__, dd->name);
> + return -EIO;
> + }
> + if (dd->error) {
> + pr_err("%s: daxdev=%s memory error\n",
> + __func__, dd->name);
> + return -EHWPOISON;
> + }
> + return 0;
> +}
...
> @@ -966,7 +1064,8 @@ famfs_file_bad(struct inode *inode)
> return -EIO;
> }
> if (meta->error) {
> - pr_debug("%s: previously detected metadata errors\n", __func__);
> + pr_debug("%s: previously detected metadata errors\n",
> + __func__);
Spurious change.
> return -EIO;
> }
On 26/01/08 03:17PM, Jonathan Cameron wrote:
> On Wed, 7 Jan 2026 09:33:27 -0600
> John Groves <John@Groves.net> wrote:
>
> > Memory errors are at least somewhat more likely on disaggregated memory
> > than on-board memory. This commit registers to be notified by fsdev_dax
> > in the event that a memory failure is detected.
> >
> > When a file access resolves to a daxdev with memory errors, it will fail
> > with an appropriate error.
> >
> > If a daxdev failed fs_dax_get(), we set dd->dax_err. If a daxdev called
> > our notify_failure(), set dd->error. When any of the above happens, set
> > (file)->error and stop allowing access.
> >
> > In general, the recovery from memory errors is to unmount the file
> > system and re-initialize the memory, but there may be usable degraded
> > modes of operation - particularly in the future when famfs supports
> > file systems backed by more than one daxdev. In those cases,
> > accessing data that is on a working daxdev can still work.
> >
> > For now, return errors for any file that has encountered a memory or dax
> > error.
> >
> > Signed-off-by: John Groves <john@groves.net>
> > ---
> > fs/fuse/famfs.c | 115 +++++++++++++++++++++++++++++++++++++++---
> > fs/fuse/famfs_kfmap.h | 3 +-
> > 2 files changed, 109 insertions(+), 9 deletions(-)
> >
> > diff --git a/fs/fuse/famfs.c b/fs/fuse/famfs.c
> > index c02b14789c6e..4eb87c5c628e 100644
> > --- a/fs/fuse/famfs.c
> > +++ b/fs/fuse/famfs.c
>
> > @@ -254,6 +288,38 @@ famfs_update_daxdev_table(
> > return 0;
> > }
> >
> > +static void
> > +famfs_set_daxdev_err(
> > + struct fuse_conn *fc,
> > + struct dax_device *dax_devp)
> > +{
> > + int i;
> > +
> > + /* Gotta search the list by dax_devp;
> > + * read lock because we're not adding or removing daxdev entries
> > + */
> > + down_read(&fc->famfs_devlist_sem);
>
> Use a guard()
Done
>
> > + for (i = 0; i < fc->dax_devlist->nslots; i++) {
> > + if (fc->dax_devlist->devlist[i].valid) {
> > + struct famfs_daxdev *dd = &fc->dax_devlist->devlist[i];
> > +
> > + if (dd->devp != dax_devp)
> > + continue;
> > +
> > + dd->error = true;
> > + up_read(&fc->famfs_devlist_sem);
> > +
> > + pr_err("%s: memory error on daxdev %s (%d)\n",
> > + __func__, dd->name, i);
> > + goto done;
> > + }
> > + }
> > + up_read(&fc->famfs_devlist_sem);
> > + pr_err("%s: memory err on unrecognized daxdev\n", __func__);
> > +
> > +done:
>
> If this isn't getting more interesting, just return above.
Right - simplified.
>
> > +}
> > +
> > /***************************************************************************/
> >
> > void
> > @@ -611,6 +677,26 @@ famfs_file_init_dax(
> >
> > static ssize_t famfs_file_bad(struct inode *inode);
> >
> > +static int famfs_dax_err(struct famfs_daxdev *dd)
>
> I'd introduce this earlier in the series to reduce need
> to refactor below.
Will mull that over when I further mull the helpers in fuse_i.h that are
hard to rebase...
>
> > +{
> > + if (!dd->valid) {
> > + pr_err("%s: daxdev=%s invalid\n",
> > + __func__, dd->name);
> > + return -EIO;
> > + }
> > + if (dd->dax_err) {
> > + pr_err("%s: daxdev=%s dax_err\n",
> > + __func__, dd->name);
> > + return -EIO;
> > + }
> > + if (dd->error) {
> > + pr_err("%s: daxdev=%s memory error\n",
> > + __func__, dd->name);
> > + return -EHWPOISON;
> > + }
> > + return 0;
> > +}
>
> ...
>
> > @@ -966,7 +1064,8 @@ famfs_file_bad(struct inode *inode)
> > return -EIO;
> > }
> > if (meta->error) {
> > - pr_debug("%s: previously detected metadata errors\n", __func__);
> > + pr_debug("%s: previously detected metadata errors\n",
> > + __func__);
>
> Spurious change.
Derp. Reverted out
Thanks Jonathan
© 2016 - 2026 Red Hat, Inc.