With the support from ib_core to use Dynamic Interrupt Moderation
(DIM) from legacy ULPs, which uses ib_create_cq(), we enable that
feature for the receive and send CQs in RDS.
A set of rds-stress runs have been done. bcopy read + write for
payload 8448 and 16640 bytes and ack/req of 256 bytes. Number of QPs
varies from 8 to 128, number of threads (i.e. rds-stress processes)
from one to 16 and a depth of four. A limit has been applied such that
the number of processes times the number of QPs never exceeds 128. All
in all, 61 rds-stress runs.
For brevity, only the rows showing a +/- 3% deviation or larger from
base is listed. The geometric mean of the ratios (IOPS_test /
IOPS_base) is calculated for all 61 runs, and that gives the best
possible "average" impact of the commits.
In the following, "base" is v6.11-rc7. "test" is the same
kernel with the following two commits:
* rds: ib: Add Dynamic Interrupt Moderation to CQs (this commit)
* RDMA/core: Enable legacy ULPs to use RDMA DIM
This is executed between two X8-2 with CX-5 using fw 16.35.3502. These
BM systems were instantiated with one VF, which were used for the
test:
base test
ACK REQ QPS THR DEP IOPS IOPS Percent
256 8448 8 1 4 634463 658162 3.7
256 8448 8 2 4 862648 997358 15.6
256 8448 8 4 4 950458 1113991 17.2
256 8448 8 8 4 932120 1127024 20.9
256 8448 8 16 4 944977 1133885 20.0
8448 256 8 2 4 858663 975563 13.6
8448 256 8 4 4 934884 1098854 17.5
8448 256 8 8 4 928247 1116015 20.2
8448 256 8 16 4 938864 1123455 19.7
256 8448 64 1 4 965985 918445 -4.9
8448 256 64 1 4 963280 918239 -4.7
256 16640 8 2 4 544670 582330 6.9
256 16640 8 4 4 554873 597553 7.7
256 16640 8 8 4 551799 597479 8.3
256 16640 8 16 4 553041 597898 8.1
16640 256 8 2 4 544644 578331 6.2
16640 256 8 4 4 553944 594627 7.3
16640 256 8 8 4 551388 594737 7.9
16640 256 8 16 4 552986 596581 7.9
Geometric mean of ratios: 1.03
Signed-off-by: Håkon Bugge <haakon.bugge@oracle.com>
---
net/rds/ib_cm.c | 10 ++++++++++
1 file changed, 10 insertions(+)
diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
index 26b069e1999df..79603d86b6c02 100644
--- a/net/rds/ib_cm.c
+++ b/net/rds/ib_cm.c
@@ -259,6 +259,7 @@ static void rds_ib_cq_comp_handler_recv(struct ib_cq *cq, void *context)
static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
struct ib_wc *wcs)
{
+ int ncompleted = 0;
int nr, i;
struct ib_wc *wc;
@@ -276,7 +277,10 @@ static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
rds_ib_mr_cqe_handler(ic, wc);
}
+ ncompleted += nr;
}
+ if (cq->dim)
+ rdma_dim(cq->dim, ncompleted);
}
static void rds_ib_tasklet_fn_send(unsigned long data)
@@ -304,6 +308,7 @@ static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
struct ib_wc *wcs,
struct rds_ib_ack_state *ack_state)
{
+ int ncompleted = 0;
int nr, i;
struct ib_wc *wc;
@@ -316,7 +321,10 @@ static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
rds_ib_recv_cqe_handler(ic, wc, ack_state);
}
+ ncompleted += nr;
}
+ if (cq->dim)
+ rdma_dim(cq->dim, ncompleted);
}
static void rds_ib_tasklet_fn_recv(unsigned long data)
@@ -542,6 +550,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev);
cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1;
cq_attr.comp_vector = ic->i_scq_vector;
+ cq_attr.flags |= IB_CQ_MODERATE;
ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send,
rds_ib_cq_event_handler, conn,
&cq_attr);
@@ -556,6 +565,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev);
cq_attr.cqe = ic->i_recv_ring.w_nr;
cq_attr.comp_vector = ic->i_rcq_vector;
+ cq_attr.flags |= IB_CQ_MODERATE;
ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv,
rds_ib_cq_event_handler, conn,
&cq_attr);
--
2.43.5
在 2024/9/18 16:35, Håkon Bugge 写道:
> With the support from ib_core to use Dynamic Interrupt Moderation
> (DIM) from legacy ULPs, which uses ib_create_cq(), we enable that
> feature for the receive and send CQs in RDS.
>
> A set of rds-stress runs have been done. bcopy read + write for
> payload 8448 and 16640 bytes and ack/req of 256 bytes. Number of QPs
> varies from 8 to 128, number of threads (i.e. rds-stress processes)
> from one to 16 and a depth of four. A limit has been applied such that
> the number of processes times the number of QPs never exceeds 128. All
> in all, 61 rds-stress runs.
>
> For brevity, only the rows showing a +/- 3% deviation or larger from
> base is listed. The geometric mean of the ratios (IOPS_test /
> IOPS_base) is calculated for all 61 runs, and that gives the best
> possible "average" impact of the commits.
>
> In the following, "base" is v6.11-rc7. "test" is the same
> kernel with the following two commits:
>
> * rds: ib: Add Dynamic Interrupt Moderation to CQs (this commit)
> * RDMA/core: Enable legacy ULPs to use RDMA DIM
>
> This is executed between two X8-2 with CX-5 using fw 16.35.3502. These
> BM systems were instantiated with one VF, which were used for the
> test:
>
> base test
> ACK REQ QPS THR DEP IOPS IOPS Percent
> 256 8448 8 1 4 634463 658162 3.7
> 256 8448 8 2 4 862648 997358 15.6
> 256 8448 8 4 4 950458 1113991 17.2
> 256 8448 8 8 4 932120 1127024 20.9
> 256 8448 8 16 4 944977 1133885 20.0
> 8448 256 8 2 4 858663 975563 13.6
> 8448 256 8 4 4 934884 1098854 17.5
> 8448 256 8 8 4 928247 1116015 20.2
> 8448 256 8 16 4 938864 1123455 19.7
> 256 8448 64 1 4 965985 918445 -4.9
> 8448 256 64 1 4 963280 918239 -4.7
> 256 16640 8 2 4 544670 582330 6.9
> 256 16640 8 4 4 554873 597553 7.7
> 256 16640 8 8 4 551799 597479 8.3
> 256 16640 8 16 4 553041 597898 8.1
> 16640 256 8 2 4 544644 578331 6.2
> 16640 256 8 4 4 553944 594627 7.3
> 16640 256 8 8 4 551388 594737 7.9
> 16640 256 8 16 4 552986 596581 7.9
> Geometric mean of ratios: 1.03
>
> Signed-off-by: Håkon Bugge <haakon.bugge@oracle.com>
> ---
> net/rds/ib_cm.c | 10 ++++++++++
> 1 file changed, 10 insertions(+)
>
> diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
> index 26b069e1999df..79603d86b6c02 100644
> --- a/net/rds/ib_cm.c
> +++ b/net/rds/ib_cm.c
> @@ -259,6 +259,7 @@ static void rds_ib_cq_comp_handler_recv(struct ib_cq *cq, void *context)
> static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
> struct ib_wc *wcs)
> {
> + int ncompleted = 0;
> int nr, i;
> struct ib_wc *wc;
>
> @@ -276,7 +277,10 @@ static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
> rds_ib_mr_cqe_handler(ic, wc);
>
> }
> + ncompleted += nr;
> }
> + if (cq->dim)
> + rdma_dim(cq->dim, ncompleted);
> }
>
> static void rds_ib_tasklet_fn_send(unsigned long data)
> @@ -304,6 +308,7 @@ static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
> struct ib_wc *wcs,
> struct rds_ib_ack_state *ack_state)
> {
> + int ncompleted = 0;
> int nr, i;
> struct ib_wc *wc;
>
> @@ -316,7 +321,10 @@ static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
>
> rds_ib_recv_cqe_handler(ic, wc, ack_state);
> }
> + ncompleted += nr;
> }
> + if (cq->dim)
> + rdma_dim(cq->dim, ncompleted);
> }
>
> static void rds_ib_tasklet_fn_recv(unsigned long data)
> @@ -542,6 +550,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
> ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev);
> cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1;
> cq_attr.comp_vector = ic->i_scq_vector;
> + cq_attr.flags |= IB_CQ_MODERATE;
cq_attr.flags is added IB_CQ_MODERATE here.
> ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send,
> rds_ib_cq_event_handler, conn,
> &cq_attr);
> @@ -556,6 +565,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
> ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev);
> cq_attr.cqe = ic->i_recv_ring.w_nr;
> cq_attr.comp_vector = ic->i_rcq_vector;
> + cq_attr.flags |= IB_CQ_MODERATE;
Why is cq_attr.flags add IB_CQ_MODERATE again?
Is this cq_attr.flags changed before this line?
Zhu Yanjun
> ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv,
> rds_ib_cq_event_handler, conn,
> &cq_attr);
在 2024/9/18 16:35, Håkon Bugge 写道:
> With the support from ib_core to use Dynamic Interrupt Moderation
> (DIM) from legacy ULPs, which uses ib_create_cq(), we enable that
> feature for the receive and send CQs in RDS.
Hi, Haakon
I am interested in this patch series. I just wonder if the performance
of rds is increased after DIM is used in legacy ULPs?
That is, is there any benefit to legacy ULPs after DIM is used?
Do you have any test results about this DIM?
Thanks,
Zhu Yanjun
>
> A set of rds-stress runs have been done. bcopy read + write for
> payload 8448 and 16640 bytes and ack/req of 256 bytes. Number of QPs
> varies from 8 to 128, number of threads (i.e. rds-stress processes)
> from one to 16 and a depth of four. A limit has been applied such that
> the number of processes times the number of QPs never exceeds 128. All
> in all, 61 rds-stress runs.
>
> For brevity, only the rows showing a +/- 3% deviation or larger from
> base is listed. The geometric mean of the ratios (IOPS_test /
> IOPS_base) is calculated for all 61 runs, and that gives the best
> possible "average" impact of the commits.
>
> In the following, "base" is v6.11-rc7. "test" is the same
> kernel with the following two commits:
>
> * rds: ib: Add Dynamic Interrupt Moderation to CQs (this commit)
> * RDMA/core: Enable legacy ULPs to use RDMA DIM
>
> This is executed between two X8-2 with CX-5 using fw 16.35.3502. These
> BM systems were instantiated with one VF, which were used for the
> test:
>
> base test
> ACK REQ QPS THR DEP IOPS IOPS Percent
> 256 8448 8 1 4 634463 658162 3.7
> 256 8448 8 2 4 862648 997358 15.6
> 256 8448 8 4 4 950458 1113991 17.2
> 256 8448 8 8 4 932120 1127024 20.9
> 256 8448 8 16 4 944977 1133885 20.0
> 8448 256 8 2 4 858663 975563 13.6
> 8448 256 8 4 4 934884 1098854 17.5
> 8448 256 8 8 4 928247 1116015 20.2
> 8448 256 8 16 4 938864 1123455 19.7
> 256 8448 64 1 4 965985 918445 -4.9
> 8448 256 64 1 4 963280 918239 -4.7
> 256 16640 8 2 4 544670 582330 6.9
> 256 16640 8 4 4 554873 597553 7.7
> 256 16640 8 8 4 551799 597479 8.3
> 256 16640 8 16 4 553041 597898 8.1
> 16640 256 8 2 4 544644 578331 6.2
> 16640 256 8 4 4 553944 594627 7.3
> 16640 256 8 8 4 551388 594737 7.9
> 16640 256 8 16 4 552986 596581 7.9
> Geometric mean of ratios: 1.03
>
> Signed-off-by: Håkon Bugge <haakon.bugge@oracle.com>
> ---
> net/rds/ib_cm.c | 10 ++++++++++
> 1 file changed, 10 insertions(+)
>
> diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
> index 26b069e1999df..79603d86b6c02 100644
> --- a/net/rds/ib_cm.c
> +++ b/net/rds/ib_cm.c
> @@ -259,6 +259,7 @@ static void rds_ib_cq_comp_handler_recv(struct ib_cq *cq, void *context)
> static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
> struct ib_wc *wcs)
> {
> + int ncompleted = 0;
> int nr, i;
> struct ib_wc *wc;
>
> @@ -276,7 +277,10 @@ static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
> rds_ib_mr_cqe_handler(ic, wc);
>
> }
> + ncompleted += nr;
> }
> + if (cq->dim)
> + rdma_dim(cq->dim, ncompleted);
> }
>
> static void rds_ib_tasklet_fn_send(unsigned long data)
> @@ -304,6 +308,7 @@ static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
> struct ib_wc *wcs,
> struct rds_ib_ack_state *ack_state)
> {
> + int ncompleted = 0;
> int nr, i;
> struct ib_wc *wc;
>
> @@ -316,7 +321,10 @@ static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
>
> rds_ib_recv_cqe_handler(ic, wc, ack_state);
> }
> + ncompleted += nr;
> }
> + if (cq->dim)
> + rdma_dim(cq->dim, ncompleted);
> }
>
> static void rds_ib_tasklet_fn_recv(unsigned long data)
> @@ -542,6 +550,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
> ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev);
> cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1;
> cq_attr.comp_vector = ic->i_scq_vector;
> + cq_attr.flags |= IB_CQ_MODERATE;
> ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send,
> rds_ib_cq_event_handler, conn,
> &cq_attr);
> @@ -556,6 +565,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
> ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev);
> cq_attr.cqe = ic->i_recv_ring.w_nr;
> cq_attr.comp_vector = ic->i_rcq_vector;
> + cq_attr.flags |= IB_CQ_MODERATE;
> ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv,
> rds_ib_cq_event_handler, conn,
> &cq_attr);
> On 20 Sep 2024, at 09:47, Zhu Yanjun <yanjun.zhu@linux.dev> wrote:
>
> 在 2024/9/18 16:35, Håkon Bugge 写道:
>> With the support from ib_core to use Dynamic Interrupt Moderation
>> (DIM) from legacy ULPs, which uses ib_create_cq(), we enable that
>> feature for the receive and send CQs in RDS.
>
> Hi, Haakon
>
> I am interested in this patch series. I just wonder if the performance of rds is increased after DIM is used in legacy ULPs?
> That is, is there any benefit to legacy ULPs after DIM is used?
>
> Do you have any test results about this DIM?
Yes, please see the cover letter of this commit.
Thxs, Håkon
>
> Thanks,
> Zhu Yanjun
>
>> A set of rds-stress runs have been done. bcopy read + write for
>> payload 8448 and 16640 bytes and ack/req of 256 bytes. Number of QPs
>> varies from 8 to 128, number of threads (i.e. rds-stress processes)
>> from one to 16 and a depth of four. A limit has been applied such that
>> the number of processes times the number of QPs never exceeds 128. All
>> in all, 61 rds-stress runs.
>> For brevity, only the rows showing a +/- 3% deviation or larger from
>> base is listed. The geometric mean of the ratios (IOPS_test /
>> IOPS_base) is calculated for all 61 runs, and that gives the best
>> possible "average" impact of the commits.
>> In the following, "base" is v6.11-rc7. "test" is the same
>> kernel with the following two commits:
>> * rds: ib: Add Dynamic Interrupt Moderation to CQs (this commit)
>> * RDMA/core: Enable legacy ULPs to use RDMA DIM
>> This is executed between two X8-2 with CX-5 using fw 16.35.3502. These
>> BM systems were instantiated with one VF, which were used for the
>> test:
>> base test
>> ACK REQ QPS THR DEP IOPS IOPS Percent
>> 256 8448 8 1 4 634463 658162 3.7
>> 256 8448 8 2 4 862648 997358 15.6
>> 256 8448 8 4 4 950458 1113991 17.2
>> 256 8448 8 8 4 932120 1127024 20.9
>> 256 8448 8 16 4 944977 1133885 20.0
>> 8448 256 8 2 4 858663 975563 13.6
>> 8448 256 8 4 4 934884 1098854 17.5
>> 8448 256 8 8 4 928247 1116015 20.2
>> 8448 256 8 16 4 938864 1123455 19.7
>> 256 8448 64 1 4 965985 918445 -4.9
>> 8448 256 64 1 4 963280 918239 -4.7
>> 256 16640 8 2 4 544670 582330 6.9
>> 256 16640 8 4 4 554873 597553 7.7
>> 256 16640 8 8 4 551799 597479 8.3
>> 256 16640 8 16 4 553041 597898 8.1
>> 16640 256 8 2 4 544644 578331 6.2
>> 16640 256 8 4 4 553944 594627 7.3
>> 16640 256 8 8 4 551388 594737 7.9
>> 16640 256 8 16 4 552986 596581 7.9
>> Geometric mean of ratios: 1.03
>> Signed-off-by: Håkon Bugge <haakon.bugge@oracle.com>
>> ---
>> net/rds/ib_cm.c | 10 ++++++++++
>> 1 file changed, 10 insertions(+)
>> diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
>> index 26b069e1999df..79603d86b6c02 100644
>> --- a/net/rds/ib_cm.c
>> +++ b/net/rds/ib_cm.c
>> @@ -259,6 +259,7 @@ static void rds_ib_cq_comp_handler_recv(struct ib_cq *cq, void *context)
>> static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
>> struct ib_wc *wcs)
>> {
>> + int ncompleted = 0;
>> int nr, i;
>> struct ib_wc *wc;
>> @@ -276,7 +277,10 @@ static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
>> rds_ib_mr_cqe_handler(ic, wc);
>> }
>> + ncompleted += nr;
>> }
>> + if (cq->dim)
>> + rdma_dim(cq->dim, ncompleted);
>> }
>> static void rds_ib_tasklet_fn_send(unsigned long data)
>> @@ -304,6 +308,7 @@ static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
>> struct ib_wc *wcs,
>> struct rds_ib_ack_state *ack_state)
>> {
>> + int ncompleted = 0;
>> int nr, i;
>> struct ib_wc *wc;
>> @@ -316,7 +321,10 @@ static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
>> rds_ib_recv_cqe_handler(ic, wc, ack_state);
>> }
>> + ncompleted += nr;
>> }
>> + if (cq->dim)
>> + rdma_dim(cq->dim, ncompleted);
>> }
>> static void rds_ib_tasklet_fn_recv(unsigned long data)
>> @@ -542,6 +550,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
>> ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev);
>> cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1;
>> cq_attr.comp_vector = ic->i_scq_vector;
>> + cq_attr.flags |= IB_CQ_MODERATE;
>> ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send,
>> rds_ib_cq_event_handler, conn,
>> &cq_attr);
>> @@ -556,6 +565,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
>> ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev);
>> cq_attr.cqe = ic->i_recv_ring.w_nr;
>> cq_attr.comp_vector = ic->i_rcq_vector;
>> + cq_attr.flags |= IB_CQ_MODERATE;
>> ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv,
>> rds_ib_cq_event_handler, conn,
>> &cq_attr);
>
在 2024/9/20 17:42, Haakon Bugge 写道:
>
>
>> On 20 Sep 2024, at 09:47, Zhu Yanjun <yanjun.zhu@linux.dev> wrote:
>>
>> 在 2024/9/18 16:35, Håkon Bugge 写道:
>>> With the support from ib_core to use Dynamic Interrupt Moderation
>>> (DIM) from legacy ULPs, which uses ib_create_cq(), we enable that
>>> feature for the receive and send CQs in RDS.
>>
>> Hi, Haakon
>>
>> I am interested in this patch series. I just wonder if the performance of rds is increased after DIM is used in legacy ULPs?
>> That is, is there any benefit to legacy ULPs after DIM is used?
>>
>> Do you have any test results about this DIM?
>
> Yes, please see the cover letter of this commit.
Which Oracle Linux distro includes this feature?
And what is the kernel version?
Zhu Yanjun
>
>
> Thxs, Håkon
>
>>
>> Thanks,
>> Zhu Yanjun
>>
>>> A set of rds-stress runs have been done. bcopy read + write for
>>> payload 8448 and 16640 bytes and ack/req of 256 bytes. Number of QPs
>>> varies from 8 to 128, number of threads (i.e. rds-stress processes)
>>> from one to 16 and a depth of four. A limit has been applied such that
>>> the number of processes times the number of QPs never exceeds 128. All
>>> in all, 61 rds-stress runs.
>>> For brevity, only the rows showing a +/- 3% deviation or larger from
>>> base is listed. The geometric mean of the ratios (IOPS_test /
>>> IOPS_base) is calculated for all 61 runs, and that gives the best
>>> possible "average" impact of the commits.
>>> In the following, "base" is v6.11-rc7. "test" is the same
>>> kernel with the following two commits:
>>> * rds: ib: Add Dynamic Interrupt Moderation to CQs (this commit)
>>> * RDMA/core: Enable legacy ULPs to use RDMA DIM
>>> This is executed between two X8-2 with CX-5 using fw 16.35.3502. These
>>> BM systems were instantiated with one VF, which were used for the
>>> test:
>>> base test
>>> ACK REQ QPS THR DEP IOPS IOPS Percent
>>> 256 8448 8 1 4 634463 658162 3.7
>>> 256 8448 8 2 4 862648 997358 15.6
>>> 256 8448 8 4 4 950458 1113991 17.2
>>> 256 8448 8 8 4 932120 1127024 20.9
>>> 256 8448 8 16 4 944977 1133885 20.0
>>> 8448 256 8 2 4 858663 975563 13.6
>>> 8448 256 8 4 4 934884 1098854 17.5
>>> 8448 256 8 8 4 928247 1116015 20.2
>>> 8448 256 8 16 4 938864 1123455 19.7
>>> 256 8448 64 1 4 965985 918445 -4.9
>>> 8448 256 64 1 4 963280 918239 -4.7
>>> 256 16640 8 2 4 544670 582330 6.9
>>> 256 16640 8 4 4 554873 597553 7.7
>>> 256 16640 8 8 4 551799 597479 8.3
>>> 256 16640 8 16 4 553041 597898 8.1
>>> 16640 256 8 2 4 544644 578331 6.2
>>> 16640 256 8 4 4 553944 594627 7.3
>>> 16640 256 8 8 4 551388 594737 7.9
>>> 16640 256 8 16 4 552986 596581 7.9
>>> Geometric mean of ratios: 1.03
>>> Signed-off-by: Håkon Bugge <haakon.bugge@oracle.com>
>>> ---
>>> net/rds/ib_cm.c | 10 ++++++++++
>>> 1 file changed, 10 insertions(+)
>>> diff --git a/net/rds/ib_cm.c b/net/rds/ib_cm.c
>>> index 26b069e1999df..79603d86b6c02 100644
>>> --- a/net/rds/ib_cm.c
>>> +++ b/net/rds/ib_cm.c
>>> @@ -259,6 +259,7 @@ static void rds_ib_cq_comp_handler_recv(struct ib_cq *cq, void *context)
>>> static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
>>> struct ib_wc *wcs)
>>> {
>>> + int ncompleted = 0;
>>> int nr, i;
>>> struct ib_wc *wc;
>>> @@ -276,7 +277,10 @@ static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
>>> rds_ib_mr_cqe_handler(ic, wc);
>>> }
>>> + ncompleted += nr;
>>> }
>>> + if (cq->dim)
>>> + rdma_dim(cq->dim, ncompleted);
>>> }
>>> static void rds_ib_tasklet_fn_send(unsigned long data)
>>> @@ -304,6 +308,7 @@ static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
>>> struct ib_wc *wcs,
>>> struct rds_ib_ack_state *ack_state)
>>> {
>>> + int ncompleted = 0;
>>> int nr, i;
>>> struct ib_wc *wc;
>>> @@ -316,7 +321,10 @@ static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
>>> rds_ib_recv_cqe_handler(ic, wc, ack_state);
>>> }
>>> + ncompleted += nr;
>>> }
>>> + if (cq->dim)
>>> + rdma_dim(cq->dim, ncompleted);
>>> }
>>> static void rds_ib_tasklet_fn_recv(unsigned long data)
>>> @@ -542,6 +550,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
>>> ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev);
>>> cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1;
>>> cq_attr.comp_vector = ic->i_scq_vector;
>>> + cq_attr.flags |= IB_CQ_MODERATE;
>>> ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send,
>>> rds_ib_cq_event_handler, conn,
>>> &cq_attr);
>>> @@ -556,6 +565,7 @@ static int rds_ib_setup_qp(struct rds_connection *conn)
>>> ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev);
>>> cq_attr.cqe = ic->i_recv_ring.w_nr;
>>> cq_attr.comp_vector = ic->i_rcq_vector;
>>> + cq_attr.flags |= IB_CQ_MODERATE;
>>> ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv,
>>> rds_ib_cq_event_handler, conn,
>>> &cq_attr);
>>
>
© 2016 - 2024 Red Hat, Inc.