[PATCH v3 21/21] nvme-fc: Extend FENCING state per TP4129 on CCR failure

Mohamed Khalfella posted 21 patches 1 month, 2 weeks ago
There is a newer version of this series
[PATCH v3 21/21] nvme-fc: Extend FENCING state per TP4129 on CCR failure
Posted by Mohamed Khalfella 1 month, 2 weeks ago
If CCR operations fail and CQT is supported, we must defer the retry of
inflight requests per TP4129. Update ctrl->fencing_work to schedule
ctrl->fenced_work, effectively extending the FENCING state. This delay
ensures that inflight requests are held until it is safe for them to be
retired.

Signed-off-by: Mohamed Khalfella <mkhalfella@purestorage.com>
---
 drivers/nvme/host/fc.c | 39 +++++++++++++++++++++++++++++++++++----
 1 file changed, 35 insertions(+), 4 deletions(-)

diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
index eac3a7ccaa5c..81088a4ce298 100644
--- a/drivers/nvme/host/fc.c
+++ b/drivers/nvme/host/fc.c
@@ -167,6 +167,7 @@ struct nvme_fc_ctrl {
 	struct blk_mq_tag_set	tag_set;
 
 	struct work_struct	fencing_work;
+	struct delayed_work	fenced_work;
 	struct work_struct	ioerr_work;
 	struct delayed_work	connect_work;
 
@@ -1878,6 +1879,18 @@ __nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl,
 	return ret;
 }
 
+static void nvme_fc_fenced_work(struct work_struct *work)
+{
+	struct nvme_fc_ctrl *fc_ctrl = container_of(to_delayed_work(work),
+					struct nvme_fc_ctrl, fenced_work);
+	struct nvme_ctrl *ctrl = &fc_ctrl->ctrl;
+
+	dev_info(ctrl->device, "Time-based recovery finished\n");
+	nvme_change_ctrl_state(ctrl, NVME_CTRL_FENCED);
+	if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
+		queue_work(nvme_reset_wq, &fc_ctrl->ioerr_work);
+}
+
 static void nvme_fc_fencing_work(struct work_struct *work)
 {
 	struct nvme_fc_ctrl *fc_ctrl =
@@ -1886,16 +1899,33 @@ static void nvme_fc_fencing_work(struct work_struct *work)
 	unsigned long rem;
 
 	rem = nvme_fence_ctrl(ctrl);
-	if (rem) {
+	if (!rem)
+		goto done;
+
+	if (!ctrl->cqt) {
 		dev_info(ctrl->device,
-			 "CCR failed, skipping time-based recovery\n");
+			 "CCR failed, CQT not supported, skip time-based recovery\n");
+		goto done;
 	}
 
+	dev_info(ctrl->device,
+		 "CCR failed, switch to time-based recovery, timeout = %ums\n",
+		 jiffies_to_msecs(rem));
+	queue_delayed_work(nvme_wq, &fc_ctrl->fenced_work, rem);
+	return;
+
+done:
 	nvme_change_ctrl_state(ctrl, NVME_CTRL_FENCED);
 	if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
 		queue_work(nvme_reset_wq, &fc_ctrl->ioerr_work);
 }
 
+static void nvme_fc_flush_fencing_works(struct nvme_fc_ctrl *ctrl)
+{
+	flush_work(&ctrl->fencing_work);
+	flush_delayed_work(&ctrl->fenced_work);
+}
+
 static void
 nvme_fc_ctrl_ioerr_work(struct work_struct *work)
 {
@@ -1917,7 +1947,7 @@ nvme_fc_ctrl_ioerr_work(struct work_struct *work)
 		return;
 	}
 
-	flush_work(&ctrl->fencing_work);
+	nvme_fc_flush_fencing_works(ctrl);
 	nvme_fc_error_recovery(ctrl);
 }
 
@@ -3396,7 +3426,7 @@ nvme_fc_reset_ctrl_work(struct work_struct *work)
 	struct nvme_fc_ctrl *ctrl =
 		container_of(work, struct nvme_fc_ctrl, ctrl.reset_work);
 
-	flush_work(&ctrl->fencing_work);
+	nvme_fc_flush_fencing_works(ctrl);
 	nvme_stop_ctrl(&ctrl->ctrl);
 
 	/* will block will waiting for io to terminate */
@@ -3573,6 +3603,7 @@ nvme_fc_alloc_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
 	INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work);
 	INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work);
 	INIT_WORK(&ctrl->fencing_work, nvme_fc_fencing_work);
+	INIT_DELAYED_WORK(&ctrl->fenced_work, nvme_fc_fenced_work);
 	INIT_WORK(&ctrl->ioerr_work, nvme_fc_ctrl_ioerr_work);
 	spin_lock_init(&ctrl->lock);
 
-- 
2.52.0
Re: [PATCH v3 21/21] nvme-fc: Extend FENCING state per TP4129 on CCR failure
Posted by James Smart 1 month ago
On 2/13/2026 8:25 PM, Mohamed Khalfella wrote:
> If CCR operations fail and CQT is supported, we must defer the retry of
> inflight requests per TP4129. Update ctrl->fencing_work to schedule
> ctrl->fenced_work, effectively extending the FENCING state. This delay
> ensures that inflight requests are held until it is safe for them to be
> retired.
> 
> Signed-off-by: Mohamed Khalfella <mkhalfella@purestorage.com>
> ---
>   drivers/nvme/host/fc.c | 39 +++++++++++++++++++++++++++++++++++----
>   1 file changed, 35 insertions(+), 4 deletions(-)
> 
> diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
> index eac3a7ccaa5c..81088a4ce298 100644
> --- a/drivers/nvme/host/fc.c
> +++ b/drivers/nvme/host/fc.c
> @@ -167,6 +167,7 @@ struct nvme_fc_ctrl {
>   	struct blk_mq_tag_set	tag_set;
>   
>   	struct work_struct	fencing_work;
> +	struct delayed_work	fenced_work;
>   	struct work_struct	ioerr_work;
>   	struct delayed_work	connect_work;
>   
> @@ -1878,6 +1879,18 @@ __nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl,
>   	return ret;
>   }
>   
> +static void nvme_fc_fenced_work(struct work_struct *work)
> +{
> +	struct nvme_fc_ctrl *fc_ctrl = container_of(to_delayed_work(work),
> +					struct nvme_fc_ctrl, fenced_work);
> +	struct nvme_ctrl *ctrl = &fc_ctrl->ctrl;
> +
> +	dev_info(ctrl->device, "Time-based recovery finished\n");
> +	nvme_change_ctrl_state(ctrl, NVME_CTRL_FENCED);
> +	if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
> +		queue_work(nvme_reset_wq, &fc_ctrl->ioerr_work);

sync with comments on patch 12

> +}
> +
>   static void nvme_fc_fencing_work(struct work_struct *work)
>   {
>   	struct nvme_fc_ctrl *fc_ctrl =
> @@ -1886,16 +1899,33 @@ static void nvme_fc_fencing_work(struct work_struct *work)
>   	unsigned long rem;
>   
>   	rem = nvme_fence_ctrl(ctrl);
> -	if (rem) {
> +	if (!rem)
> +		goto done;
> +
> +	if (!ctrl->cqt) {
>   		dev_info(ctrl->device,
> -			 "CCR failed, skipping time-based recovery\n");
> +			 "CCR failed, CQT not supported, skip time-based recovery\n");
> +		goto done;
>   	}
>   
> +	dev_info(ctrl->device,
> +		 "CCR failed, switch to time-based recovery, timeout = %ums\n",
> +		 jiffies_to_msecs(rem));
> +	queue_delayed_work(nvme_wq, &fc_ctrl->fenced_work, rem);
> +	return;
> +
> +done:
>   	nvme_change_ctrl_state(ctrl, NVME_CTRL_FENCED);
>   	if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
>   		queue_work(nvme_reset_wq, &fc_ctrl->ioerr_work);
>   }
>   
> +static void nvme_fc_flush_fencing_works(struct nvme_fc_ctrl *ctrl)
> +{
> +	flush_work(&ctrl->fencing_work);
> +	flush_delayed_work(&ctrl->fenced_work);
> +}
> +
>   static void
>   nvme_fc_ctrl_ioerr_work(struct work_struct *work)
>   {
> @@ -1917,7 +1947,7 @@ nvme_fc_ctrl_ioerr_work(struct work_struct *work)
>   		return;
>   	}
>   
> -	flush_work(&ctrl->fencing_work);
> +	nvme_fc_flush_fencing_works(ctrl);
>   	nvme_fc_error_recovery(ctrl);
>   }
>   
> @@ -3396,7 +3426,7 @@ nvme_fc_reset_ctrl_work(struct work_struct *work)
>   	struct nvme_fc_ctrl *ctrl =
>   		container_of(work, struct nvme_fc_ctrl, ctrl.reset_work);
>   
> -	flush_work(&ctrl->fencing_work);
> +	nvme_fc_flush_fencing_works(ctrl);
>   	nvme_stop_ctrl(&ctrl->ctrl);
>   
>   	/* will block will waiting for io to terminate */
> @@ -3573,6 +3603,7 @@ nvme_fc_alloc_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
>   	INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work);
>   	INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work);
>   	INIT_WORK(&ctrl->fencing_work, nvme_fc_fencing_work);
> +	INIT_DELAYED_WORK(&ctrl->fenced_work, nvme_fc_fenced_work);
>   	INIT_WORK(&ctrl->ioerr_work, nvme_fc_ctrl_ioerr_work);
>   	spin_lock_init(&ctrl->lock);
>   

looks ok.

Signed-off-by: James Smart <jsmart833426@gmail.com>

-- james
Re: [PATCH v3 21/21] nvme-fc: Extend FENCING state per TP4129 on CCR failure
Posted by Mohamed Khalfella 1 week, 1 day ago
On Fri 2026-02-27 17:20:45 -0800, James Smart wrote:
> On 2/13/2026 8:25 PM, Mohamed Khalfella wrote:
> > If CCR operations fail and CQT is supported, we must defer the retry of
> > inflight requests per TP4129. Update ctrl->fencing_work to schedule
> > ctrl->fenced_work, effectively extending the FENCING state. This delay
> > ensures that inflight requests are held until it is safe for them to be
> > retired.
> > 
> > Signed-off-by: Mohamed Khalfella <mkhalfella@purestorage.com>
> > ---
> >   drivers/nvme/host/fc.c | 39 +++++++++++++++++++++++++++++++++++----
> >   1 file changed, 35 insertions(+), 4 deletions(-)
> > 
> > diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c
> > index eac3a7ccaa5c..81088a4ce298 100644
> > --- a/drivers/nvme/host/fc.c
> > +++ b/drivers/nvme/host/fc.c
> > @@ -167,6 +167,7 @@ struct nvme_fc_ctrl {
> >   	struct blk_mq_tag_set	tag_set;
> >   
> >   	struct work_struct	fencing_work;
> > +	struct delayed_work	fenced_work;
> >   	struct work_struct	ioerr_work;
> >   	struct delayed_work	connect_work;
> >   
> > @@ -1878,6 +1879,18 @@ __nvme_fc_fcpop_chk_teardowns(struct nvme_fc_ctrl *ctrl,
> >   	return ret;
> >   }
> >   
> > +static void nvme_fc_fenced_work(struct work_struct *work)
> > +{
> > +	struct nvme_fc_ctrl *fc_ctrl = container_of(to_delayed_work(work),
> > +					struct nvme_fc_ctrl, fenced_work);
> > +	struct nvme_ctrl *ctrl = &fc_ctrl->ctrl;
> > +
> > +	dev_info(ctrl->device, "Time-based recovery finished\n");
> > +	nvme_change_ctrl_state(ctrl, NVME_CTRL_FENCED);
> > +	if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
> > +		queue_work(nvme_reset_wq, &fc_ctrl->ioerr_work);
> 
> sync with comments on patch 12

I will do that. It has been suggested to move CQT changes into a
separate patchset and focus on CCR changes for now. I will drop
patches [16 - 21] from this patchset to be re-introduced later.

> 
> > +}
> > +
> >   static void nvme_fc_fencing_work(struct work_struct *work)
> >   {
> >   	struct nvme_fc_ctrl *fc_ctrl =
> > @@ -1886,16 +1899,33 @@ static void nvme_fc_fencing_work(struct work_struct *work)
> >   	unsigned long rem;
> >   
> >   	rem = nvme_fence_ctrl(ctrl);
> > -	if (rem) {
> > +	if (!rem)
> > +		goto done;
> > +
> > +	if (!ctrl->cqt) {
> >   		dev_info(ctrl->device,
> > -			 "CCR failed, skipping time-based recovery\n");
> > +			 "CCR failed, CQT not supported, skip time-based recovery\n");
> > +		goto done;
> >   	}
> >   
> > +	dev_info(ctrl->device,
> > +		 "CCR failed, switch to time-based recovery, timeout = %ums\n",
> > +		 jiffies_to_msecs(rem));
> > +	queue_delayed_work(nvme_wq, &fc_ctrl->fenced_work, rem);
> > +	return;
> > +
> > +done:
> >   	nvme_change_ctrl_state(ctrl, NVME_CTRL_FENCED);
> >   	if (nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
> >   		queue_work(nvme_reset_wq, &fc_ctrl->ioerr_work);
> >   }
> >   
> > +static void nvme_fc_flush_fencing_works(struct nvme_fc_ctrl *ctrl)
> > +{
> > +	flush_work(&ctrl->fencing_work);
> > +	flush_delayed_work(&ctrl->fenced_work);
> > +}
> > +
> >   static void
> >   nvme_fc_ctrl_ioerr_work(struct work_struct *work)
> >   {
> > @@ -1917,7 +1947,7 @@ nvme_fc_ctrl_ioerr_work(struct work_struct *work)
> >   		return;
> >   	}
> >   
> > -	flush_work(&ctrl->fencing_work);
> > +	nvme_fc_flush_fencing_works(ctrl);
> >   	nvme_fc_error_recovery(ctrl);
> >   }
> >   
> > @@ -3396,7 +3426,7 @@ nvme_fc_reset_ctrl_work(struct work_struct *work)
> >   	struct nvme_fc_ctrl *ctrl =
> >   		container_of(work, struct nvme_fc_ctrl, ctrl.reset_work);
> >   
> > -	flush_work(&ctrl->fencing_work);
> > +	nvme_fc_flush_fencing_works(ctrl);
> >   	nvme_stop_ctrl(&ctrl->ctrl);
> >   
> >   	/* will block will waiting for io to terminate */
> > @@ -3573,6 +3603,7 @@ nvme_fc_alloc_ctrl(struct device *dev, struct nvmf_ctrl_options *opts,
> >   	INIT_WORK(&ctrl->ctrl.reset_work, nvme_fc_reset_ctrl_work);
> >   	INIT_DELAYED_WORK(&ctrl->connect_work, nvme_fc_connect_ctrl_work);
> >   	INIT_WORK(&ctrl->fencing_work, nvme_fc_fencing_work);
> > +	INIT_DELAYED_WORK(&ctrl->fenced_work, nvme_fc_fenced_work);
> >   	INIT_WORK(&ctrl->ioerr_work, nvme_fc_ctrl_ioerr_work);
> >   	spin_lock_init(&ctrl->lock);
> >   
> 
> looks ok.
> 
> Signed-off-by: James Smart <jsmart833426@gmail.com>
> 
> -- james
>