1. Patchew
  2. linux
  3. [PATCH for-rc 0/5] RDMA/hns: Bugfixes
  4. View patch

[PATCH for-rc 2/5] RDMA/hns: Fix flush cqe error when racing with destroy qp

Junxian Huang posted 5 patches 1 year, 3 months ago
There is a newer version of this series
[PATCH for-rc 2/5] RDMA/hns: Fix flush cqe error when racing with destroy qp
Posted by Junxian Huang 1 year, 3 months ago 
From: wenglianfa <wenglianfa@huawei.com>

QP needs to be modified to IB_QPS_ERROR to trigger HW flush cqe. But
when this process races with destroy qp, the destroy-qp process may
modify the QP to IB_QPS_RESET first. In this case flush cqe will fail
since it is invalid to modify qp from IB_QPS_RESET to IB_QPS_ERROR.

Add lock and bit flag to make sure pending flush cqe work is completed
first and no more new works will be added.

Fixes: ffd541d45726 ("RDMA/hns: Add the workqueue framework for flush cqe handler")
Signed-off-by: wenglianfa <wenglianfa@huawei.com>
Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
---
 drivers/infiniband/hw/hns/hns_roce_device.h |  2 ++
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  |  7 +++++++
 drivers/infiniband/hw/hns/hns_roce_qp.c     | 14 ++++++++++++--
 3 files changed, 21 insertions(+), 2 deletions(-)

diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index 73c78005901e..9b51d5a1533f 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -593,6 +593,7 @@ struct hns_roce_dev;
 
 enum {
 	HNS_ROCE_FLUSH_FLAG = 0,
+	HNS_ROCE_STOP_FLUSH_FLAG = 1,
 };
 
 struct hns_roce_work {
@@ -656,6 +657,7 @@ struct hns_roce_qp {
 	enum hns_roce_cong_type	cong_type;
 	u8			tc_mode;
 	u8			priority;
+	spinlock_t flush_lock;
 };
 
 struct hns_roce_ib_iboe {
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index e85c450e1809..aa42c5a9b254 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -5598,8 +5598,15 @@ int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
 {
 	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
 	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
+	unsigned long flags;
 	int ret;
 
+	/* Make sure flush_cqe() is completed */
+	spin_lock_irqsave(&hr_qp->flush_lock, flags);
+	set_bit(HNS_ROCE_STOP_FLUSH_FLAG, &hr_qp->flush_flag);
+	spin_unlock_irqrestore(&hr_qp->flush_lock, flags);
+	flush_work(&hr_qp->flush_work.work);
+
 	ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
 	if (ret)
 		ibdev_err(&hr_dev->ib_dev,
diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
index dcaa370d4a26..3439312b0138 100644
--- a/drivers/infiniband/hw/hns/hns_roce_qp.c
+++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
@@ -90,11 +90,18 @@ static void flush_work_handle(struct work_struct *work)
 void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 {
 	struct hns_roce_work *flush_work = &hr_qp->flush_work;
+	unsigned long flags;
+
+	spin_lock_irqsave(&hr_qp->flush_lock, flags);
+	/* Exit directly after destroy_qp() */
+	if (test_bit(HNS_ROCE_STOP_FLUSH_FLAG, &hr_qp->flush_flag)) {
+		spin_unlock_irqrestore(&hr_qp->flush_lock, flags);
+		return;
+	}
 
-	flush_work->hr_dev = hr_dev;
-	INIT_WORK(&flush_work->work, flush_work_handle);
 	refcount_inc(&hr_qp->refcount);
 	queue_work(hr_dev->irq_workq, &flush_work->work);
+	spin_unlock_irqrestore(&hr_qp->flush_lock, flags);
 }
 
 void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp)
@@ -1140,6 +1147,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
 				     struct ib_udata *udata,
 				     struct hns_roce_qp *hr_qp)
 {
+	struct hns_roce_work *flush_work = &hr_qp->flush_work;
 	struct hns_roce_ib_create_qp_resp resp = {};
 	struct ib_device *ibdev = &hr_dev->ib_dev;
 	struct hns_roce_ib_create_qp ucmd = {};
@@ -1151,6 +1159,8 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
 
 	hr_qp->state = IB_QPS_RESET;
 	hr_qp->flush_flag = 0;
+	flush_work->hr_dev = hr_dev;
+	INIT_WORK(&flush_work->work, flush_work_handle);
 
 	if (init_attr->create_flags)
 		return -EOPNOTSUPP;
-- 
2.33.0
Re: [PATCH for-rc 2/5] RDMA/hns: Fix flush cqe error when racing with destroy qp
Posted by Zhu Yanjun 1 year, 3 months ago 
在 2024/10/22 13:10, Junxian Huang 写道:
> From: wenglianfa <wenglianfa@huawei.com>
> 
> QP needs to be modified to IB_QPS_ERROR to trigger HW flush cqe. But
> when this process races with destroy qp, the destroy-qp process may
> modify the QP to IB_QPS_RESET first. In this case flush cqe will fail
> since it is invalid to modify qp from IB_QPS_RESET to IB_QPS_ERROR.
> 
> Add lock and bit flag to make sure pending flush cqe work is completed
> first and no more new works will be added.
> 
> Fixes: ffd541d45726 ("RDMA/hns: Add the workqueue framework for flush cqe handler")
> Signed-off-by: wenglianfa <wenglianfa@huawei.com>
> Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
> ---
>   drivers/infiniband/hw/hns/hns_roce_device.h |  2 ++
>   drivers/infiniband/hw/hns/hns_roce_hw_v2.c  |  7 +++++++
>   drivers/infiniband/hw/hns/hns_roce_qp.c     | 14 ++++++++++++--
>   3 files changed, 21 insertions(+), 2 deletions(-)
> 
> diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
> index 73c78005901e..9b51d5a1533f 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_device.h
> +++ b/drivers/infiniband/hw/hns/hns_roce_device.h
> @@ -593,6 +593,7 @@ struct hns_roce_dev;
>   
>   enum {
>   	HNS_ROCE_FLUSH_FLAG = 0,
> +	HNS_ROCE_STOP_FLUSH_FLAG = 1,
>   };
>   
>   struct hns_roce_work {
> @@ -656,6 +657,7 @@ struct hns_roce_qp {
>   	enum hns_roce_cong_type	cong_type;
>   	u8			tc_mode;
>   	u8			priority;
> +	spinlock_t flush_lock;
spin_lock_init is missing?

The spin lock flush_lock should be initialized before used.

Zhu Yanjun
>   };
>   
>   struct hns_roce_ib_iboe {
> diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
> index e85c450e1809..aa42c5a9b254 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
> +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
> @@ -5598,8 +5598,15 @@ int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
>   {
>   	struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
>   	struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
> +	unsigned long flags;
>   	int ret;
>   
> +	/* Make sure flush_cqe() is completed */
> +	spin_lock_irqsave(&hr_qp->flush_lock, flags);
> +	set_bit(HNS_ROCE_STOP_FLUSH_FLAG, &hr_qp->flush_flag);
> +	spin_unlock_irqrestore(&hr_qp->flush_lock, flags);
> +	flush_work(&hr_qp->flush_work.work);
> +
>   	ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
>   	if (ret)
>   		ibdev_err(&hr_dev->ib_dev,
> diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
> index dcaa370d4a26..3439312b0138 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_qp.c
> +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
> @@ -90,11 +90,18 @@ static void flush_work_handle(struct work_struct *work)
>   void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
>   {
>   	struct hns_roce_work *flush_work = &hr_qp->flush_work;
> +	unsigned long flags;
> +
> +	spin_lock_irqsave(&hr_qp->flush_lock, flags);
> +	/* Exit directly after destroy_qp() */
> +	if (test_bit(HNS_ROCE_STOP_FLUSH_FLAG, &hr_qp->flush_flag)) {
> +		spin_unlock_irqrestore(&hr_qp->flush_lock, flags);
> +		return;
> +	}
>   
> -	flush_work->hr_dev = hr_dev;
> -	INIT_WORK(&flush_work->work, flush_work_handle);
>   	refcount_inc(&hr_qp->refcount);
>   	queue_work(hr_dev->irq_workq, &flush_work->work);
> +	spin_unlock_irqrestore(&hr_qp->flush_lock, flags);
>   }
>   
>   void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp)
> @@ -1140,6 +1147,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
>   				     struct ib_udata *udata,
>   				     struct hns_roce_qp *hr_qp)
>   {
> +	struct hns_roce_work *flush_work = &hr_qp->flush_work;
>   	struct hns_roce_ib_create_qp_resp resp = {};
>   	struct ib_device *ibdev = &hr_dev->ib_dev;
>   	struct hns_roce_ib_create_qp ucmd = {};
> @@ -1151,6 +1159,8 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
>   
>   	hr_qp->state = IB_QPS_RESET;
>   	hr_qp->flush_flag = 0;
> +	flush_work->hr_dev = hr_dev;
> +	INIT_WORK(&flush_work->work, flush_work_handle);
>   
>   	if (init_attr->create_flags)
>   		return -EOPNOTSUPP;
Re: [PATCH for-rc 2/5] RDMA/hns: Fix flush cqe error when racing with destroy qp
Posted by Junxian Huang 1 year, 3 months ago 

On 2024/10/23 23:13, Zhu Yanjun wrote:
> 在 2024/10/22 13:10, Junxian Huang 写道:
>> From: wenglianfa <wenglianfa@huawei.com>
>>
>> QP needs to be modified to IB_QPS_ERROR to trigger HW flush cqe. But
>> when this process races with destroy qp, the destroy-qp process may
>> modify the QP to IB_QPS_RESET first. In this case flush cqe will fail
>> since it is invalid to modify qp from IB_QPS_RESET to IB_QPS_ERROR.
>>
>> Add lock and bit flag to make sure pending flush cqe work is completed
>> first and no more new works will be added.
>>
>> Fixes: ffd541d45726 ("RDMA/hns: Add the workqueue framework for flush cqe handler")
>> Signed-off-by: wenglianfa <wenglianfa@huawei.com>
>> Signed-off-by: Junxian Huang <huangjunxian6@hisilicon.com>
>> ---
>>   drivers/infiniband/hw/hns/hns_roce_device.h |  2 ++
>>   drivers/infiniband/hw/hns/hns_roce_hw_v2.c  |  7 +++++++
>>   drivers/infiniband/hw/hns/hns_roce_qp.c     | 14 ++++++++++++--
>>   3 files changed, 21 insertions(+), 2 deletions(-)
>>
>> diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
>> index 73c78005901e..9b51d5a1533f 100644
>> --- a/drivers/infiniband/hw/hns/hns_roce_device.h
>> +++ b/drivers/infiniband/hw/hns/hns_roce_device.h
>> @@ -593,6 +593,7 @@ struct hns_roce_dev;
>>     enum {
>>       HNS_ROCE_FLUSH_FLAG = 0,
>> +    HNS_ROCE_STOP_FLUSH_FLAG = 1,
>>   };
>>     struct hns_roce_work {
>> @@ -656,6 +657,7 @@ struct hns_roce_qp {
>>       enum hns_roce_cong_type    cong_type;
>>       u8            tc_mode;
>>       u8            priority;
>> +    spinlock_t flush_lock;
> spin_lock_init is missing?
> 
> The spin lock flush_lock should be initialized before used.
> 

Will fix it. Thanks.

Junxian

> Zhu Yanjun
>>   };
>>     struct hns_roce_ib_iboe {
>> diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
>> index e85c450e1809..aa42c5a9b254 100644
>> --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
>> +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
>> @@ -5598,8 +5598,15 @@ int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
>>   {
>>       struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
>>       struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
>> +    unsigned long flags;
>>       int ret;
>>   +    /* Make sure flush_cqe() is completed */
>> +    spin_lock_irqsave(&hr_qp->flush_lock, flags);
>> +    set_bit(HNS_ROCE_STOP_FLUSH_FLAG, &hr_qp->flush_flag);
>> +    spin_unlock_irqrestore(&hr_qp->flush_lock, flags);
>> +    flush_work(&hr_qp->flush_work.work);
>> +
>>       ret = hns_roce_v2_destroy_qp_common(hr_dev, hr_qp, udata);
>>       if (ret)
>>           ibdev_err(&hr_dev->ib_dev,
>> diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
>> index dcaa370d4a26..3439312b0138 100644
>> --- a/drivers/infiniband/hw/hns/hns_roce_qp.c
>> +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
>> @@ -90,11 +90,18 @@ static void flush_work_handle(struct work_struct *work)
>>   void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
>>   {
>>       struct hns_roce_work *flush_work = &hr_qp->flush_work;
>> +    unsigned long flags;
>> +
>> +    spin_lock_irqsave(&hr_qp->flush_lock, flags);
>> +    /* Exit directly after destroy_qp() */
>> +    if (test_bit(HNS_ROCE_STOP_FLUSH_FLAG, &hr_qp->flush_flag)) {
>> +        spin_unlock_irqrestore(&hr_qp->flush_lock, flags);
>> +        return;
>> +    }
>>   -    flush_work->hr_dev = hr_dev;
>> -    INIT_WORK(&flush_work->work, flush_work_handle);
>>       refcount_inc(&hr_qp->refcount);
>>       queue_work(hr_dev->irq_workq, &flush_work->work);
>> +    spin_unlock_irqrestore(&hr_qp->flush_lock, flags);
>>   }
>>     void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp)
>> @@ -1140,6 +1147,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
>>                        struct ib_udata *udata,
>>                        struct hns_roce_qp *hr_qp)
>>   {
>> +    struct hns_roce_work *flush_work = &hr_qp->flush_work;
>>       struct hns_roce_ib_create_qp_resp resp = {};
>>       struct ib_device *ibdev = &hr_dev->ib_dev;
>>       struct hns_roce_ib_create_qp ucmd = {};
>> @@ -1151,6 +1159,8 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
>>         hr_qp->state = IB_QPS_RESET;
>>       hr_qp->flush_flag = 0;
>> +    flush_work->hr_dev = hr_dev;
>> +    INIT_WORK(&flush_work->work, flush_work_handle);
>>         if (init_attr->create_flags)
>>           return -EOPNOTSUPP;
>

Patchew 2.1-devel-b67e4c2

© 2016 - 2026 Red Hat, Inc.