From: Yu Kuai <yukuai3@huawei.com>

Before this patch, each context will hold a global lock to dispatch one
request at a time, which introduce intense lock competition:

lock
ops.dispatch_request
unlock

Hence support dispatch a batch of requests while holding the lock to
reduce lock contention.

Signed-off-by: Yu Kuai <yukuai3@huawei.com>
---
 block/blk-mq-sched.c | 55 ++++++++++++++++++++++++++++++++++++++++----
 block/blk-mq.h       | 21 +++++++++++++++++
 2 files changed, 72 insertions(+), 4 deletions(-)

diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index 990d0f19594a..d7cb88c8e8c7 100644
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -101,6 +101,49 @@ static bool elevator_can_dispatch(struct sched_dispatch_ctx *ctx)
 	return true;
 }
 
+static void elevator_dispatch_requests(struct sched_dispatch_ctx *ctx)
+{
+	struct request *rq;
+	int budget_token[BUDGET_TOKEN_BATCH];
+	int count;
+	int i;
+
+	while (true) {
+		if (!elevator_can_dispatch(ctx))
+			return;
+
+		count = blk_mq_get_dispatch_budgets(ctx->q, budget_token);
+		if (count <= 0)
+			return;
+
+		elevator_lock(ctx->e);
+		for (i = 0; i < count; ++i) {
+			rq = ctx->e->type->ops.dispatch_request(ctx->hctx);
+			if (!rq) {
+				ctx->run_queue = true;
+				goto err_free_budgets;
+			}
+
+			blk_mq_set_rq_budget_token(rq, budget_token[i]);
+			list_add_tail(&rq->queuelist, &ctx->rq_list);
+			ctx->count++;
+			if (rq->mq_hctx != ctx->hctx)
+				ctx->multi_hctxs = true;
+
+			if (!blk_mq_get_driver_tag(rq)) {
+				i++;
+				goto err_free_budgets;
+			}
+		}
+		elevator_unlock(ctx->e);
+	}
+
+err_free_budgets:
+	elevator_unlock(ctx->e);
+	for (; i < count; ++i)
+		blk_mq_put_dispatch_budget(ctx->q, budget_token[i]);
+}
+
 static bool elevator_dispatch_one_request(struct sched_dispatch_ctx *ctx)
 {
 	struct request *rq;
@@ -202,10 +245,14 @@ static int __blk_mq_do_dispatch_sched(struct blk_mq_hw_ctx *hctx)
 	else
 		max_dispatch = hctx->queue->nr_requests;
 
-	do {
-		if (!elevator_dispatch_one_request(&ctx))
-			break;
-	} while (ctx.count < max_dispatch);
+	if (!hctx->dispatch_busy && blk_queue_sq_sched(ctx.q))
+		elevator_dispatch_requests(&ctx);
+	else {
+		do {
+			if (!elevator_dispatch_one_request(&ctx))
+				break;
+		} while (ctx.count < max_dispatch);
+	}
 
 	return elevator_finish_dispatch(&ctx);
 }
diff --git a/block/blk-mq.h b/block/blk-mq.h
index affb2e14b56e..450c16a07841 100644
--- a/block/blk-mq.h
+++ b/block/blk-mq.h
@@ -37,6 +37,7 @@ enum {
 };
 
 #define BLK_MQ_CPU_WORK_BATCH	(8)
+#define BUDGET_TOKEN_BATCH	(8)
 
 typedef unsigned int __bitwise blk_insert_t;
 #define BLK_MQ_INSERT_AT_HEAD		((__force blk_insert_t)0x01)
@@ -262,6 +263,26 @@ static inline int blk_mq_get_dispatch_budget(struct request_queue *q)
 	return 0;
 }
 
+static inline int blk_mq_get_dispatch_budgets(struct request_queue *q,
+					      int *budget_token)
+{
+	int count = 0;
+
+	while (count < BUDGET_TOKEN_BATCH) {
+		int token = 0;
+
+		if (q->mq_ops->get_budget)
+			token = q->mq_ops->get_budget(q);
+
+		if (token < 0)
+			return count;
+
+		budget_token[count++] = token;
+	}
+
+	return count;
+}
+
 static inline void blk_mq_set_rq_budget_token(struct request *rq, int token)
 {
 	if (token < 0)
-- 
2.39.2

On 6/14/25 18:25, Yu Kuai wrote:
> From: Yu Kuai <yukuai3@huawei.com>
> 
> Before this patch, each context will hold a global lock to dispatch one
> request at a time, which introduce intense lock competition:

How so ? If there is only a single context issuing IOs, there will not be any
contention on the lock.

> lock
> ops.dispatch_request
> unlock
> 
> Hence support dispatch a batch of requests while holding the lock to
> reduce lock contention.

Lock contention would happen only if you have multiple processes issuing I/Os.
For a single context case, this simply reduces the overhead of dispatching
commands by avoiding the lock+unlock per request. So please explain that clearly.

> 
> Signed-off-by: Yu Kuai <yukuai3@huawei.com>
> ---
>  block/blk-mq-sched.c | 55 ++++++++++++++++++++++++++++++++++++++++----
>  block/blk-mq.h       | 21 +++++++++++++++++
>  2 files changed, 72 insertions(+), 4 deletions(-)
> 
> diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
> index 990d0f19594a..d7cb88c8e8c7 100644
> --- a/block/blk-mq-sched.c
> +++ b/block/blk-mq-sched.c
> @@ -101,6 +101,49 @@ static bool elevator_can_dispatch(struct sched_dispatch_ctx *ctx)
>  	return true;
>  }
>  
> +static void elevator_dispatch_requests(struct sched_dispatch_ctx *ctx)
> +{
> +	struct request *rq;
> +	int budget_token[BUDGET_TOKEN_BATCH];
> +	int count;
> +	int i;

These 2 can be declared on the same line.

> +
> +	while (true) {
> +		if (!elevator_can_dispatch(ctx))
> +			return;
> +
> +		count = blk_mq_get_dispatch_budgets(ctx->q, budget_token);
> +		if (count <= 0)
> +			return;
> +
> +		elevator_lock(ctx->e);
> +		for (i = 0; i < count; ++i) {
> +			rq = ctx->e->type->ops.dispatch_request(ctx->hctx);
> +			if (!rq) {
> +				ctx->run_queue = true;
> +				goto err_free_budgets;
> +			}
> +
> +			blk_mq_set_rq_budget_token(rq, budget_token[i]);
> +			list_add_tail(&rq->queuelist, &ctx->rq_list);
> +			ctx->count++;
> +			if (rq->mq_hctx != ctx->hctx)
> +				ctx->multi_hctxs = true;
> +
> +			if (!blk_mq_get_driver_tag(rq)) {
> +				i++;
> +				goto err_free_budgets;
> +			}
> +		}
> +		elevator_unlock(ctx->e);
> +	}
> +
> +err_free_budgets:
> +	elevator_unlock(ctx->e);
> +	for (; i < count; ++i)> +		blk_mq_put_dispatch_budget(ctx->q, budget_token[i]);
> +}


-- 
Damien Le Moal
Western Digital Research