From nobody Thu Nov 6 14:20:42 2025 Delivered-To: importer@patchew.org Received-SPF: pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; dkim=fail; spf=pass (zoho.com: domain of gnu.org designates 208.118.235.17 as permitted sender) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=oracle.com Return-Path: Received: from lists.gnu.org (lists.gnu.org [208.118.235.17]) by mx.zohomail.com with SMTPS id 1541422661167761.990853639041; Mon, 5 Nov 2018 04:57:41 -0800 (PST) Received: from localhost ([::1]:34894 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gJeRn-0002VO-W1 for importer@patchew.org; Mon, 05 Nov 2018 07:57:40 -0500 Received: from eggs.gnu.org ([2001:4830:134:3::10]:35965) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gJeHS-0007hx-2B for qemu-devel@nongnu.org; Mon, 05 Nov 2018 07:46:59 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gJeHQ-0003Xg-LX for qemu-devel@nongnu.org; Mon, 05 Nov 2018 07:46:57 -0500 Received: from userp2130.oracle.com ([156.151.31.86]:50716) by eggs.gnu.org with esmtps (TLS1.0:RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1gJeHQ-0003Wy-B0 for qemu-devel@nongnu.org; Mon, 05 Nov 2018 07:46:56 -0500 Received: from pps.filterd (userp2130.oracle.com [127.0.0.1]) by userp2130.oracle.com (8.16.0.22/8.16.0.22) with SMTP id wA5CiH7p054210; Mon, 5 Nov 2018 12:46:54 GMT Received: from aserv0022.oracle.com (aserv0022.oracle.com [141.146.126.234]) by userp2130.oracle.com with ESMTP id 2nh33tpg8h-1 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=OK); Mon, 05 Nov 2018 12:46:54 +0000 Received: from aserv0121.oracle.com (aserv0121.oracle.com [141.146.126.235]) by aserv0022.oracle.com (8.14.4/8.14.4) with ESMTP id wA5CkrLl001852 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-GCM-SHA384 bits=256 verify=OK); Mon, 5 Nov 2018 12:46:53 GMT Received: from abhmp0020.oracle.com (abhmp0020.oracle.com [141.146.116.26]) by aserv0121.oracle.com (8.14.4/8.13.8) with ESMTP id wA5Ckr0e013535; Mon, 5 Nov 2018 12:46:53 GMT Received: from localhost.localdomain (/77.138.186.148) by default (Oracle Beehive Gateway v4.0) with ESMTP ; Mon, 05 Nov 2018 04:46:53 -0800 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=oracle.com; h=from : to : subject : date : message-id : in-reply-to : references; s=corp-2018-07-02; bh=QGyijzhipO5y3CdvjurP1ja96Cmknits+X6oXQyrAT4=; b=VJ8b8zn9yOydYWGN0xME9Ov4SIpPU3i0XKpGI2bzrSUBRIKMu3agQyjp7C+1xYFTlcFB pIthSO34r+9UxL66Gt+TzSkWRgIdtcehe0JBZbxzAH4H73BOogJP4VNJorw7mCAoDgEd WbF9w3ZxSEKcyv6TBjKLOWRGDlZT0o/LKmoOX3Neso+fCOZyTvzQ5V5l42k/idNQ6xGy /X9qKztanlnLdTbQAcQADHO0FMzoJ0dOLtN3QS7WYAD1A6CxttnkNv2/oZA0EoRx2pa0 /QrAyV2yBb4ar34cJ8xRuffK4JrCiKK8gpUf/J+6S056yxKQddfgPxLI3C0W4QjlrZcE HA== From: Yuval Shaia To: yuval.shaia@oracle.com, marcel.apfelbaum@gmail.com, dmitry.fleytman@gmail.com, jasowang@redhat.com, eblake@redhat.com, armbru@redhat.com, shamir.rabinovitch@oracle.com, qemu-devel@nongnu.org Date: Mon, 5 Nov 2018 14:45:57 +0200 Message-Id: <20181105124559.21561-17-yuval.shaia@oracle.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20181105124559.21561-1-yuval.shaia@oracle.com> References: <20181105124559.21561-1-yuval.shaia@oracle.com> X-Proofpoint-Virus-Version: vendor=nai engine=5900 definitions=9067 signatures=668683 X-Proofpoint-Spam-Details: rule=notspam policy=default score=0 suspectscore=2 malwarescore=0 phishscore=0 bulkscore=0 spamscore=0 mlxscore=0 mlxlogscore=999 adultscore=0 classifier=spam adjust=0 reason=mlx scancount=1 engine=8.0.1-1807170000 definitions=main-1811050118 X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [generic] X-Received-From: 156.151.31.86 Subject: [Qemu-devel] [PATCH 16/18] hw/pvrdma: Fill all CQE fields X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail-DKIM: fail (Header signature does not verify) Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Add ability to pass specific WC attributes to CQE such as GRH_BIT flag. Signed-off-by: Yuval Shaia --- hw/rdma/rdma_backend.c | 59 +++++++++++++++++++++++-------------- hw/rdma/rdma_backend.h | 4 +-- hw/rdma/vmw/pvrdma_qp_ops.c | 31 +++++++++++-------- 3 files changed, 58 insertions(+), 36 deletions(-) diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c index 3f60c3cd6a..54171ded35 100644 --- a/hw/rdma/rdma_backend.c +++ b/hw/rdma/rdma_backend.c @@ -59,13 +59,24 @@ struct backend_umad { char mad[RDMA_MAX_PRIVATE_DATA]; }; =20 -static void (*comp_handler)(int status, unsigned int vendor_err, void *ctx= ); +static void (*comp_handler)(void *ctx, struct ibv_wc *wc); =20 -static void dummy_comp_handler(int status, unsigned int vendor_err, void *= ctx) +static void dummy_comp_handler(void *ctx, struct ibv_wc *wc) { pr_err("No completion handler is registered\n"); } =20 +static inline void complete_work(enum ibv_wc_status status, uint32_t vendo= r_err, + void *ctx) +{ + struct ibv_wc wc =3D {0}; + + wc.status =3D status; + wc.vendor_err =3D vendor_err; + + comp_handler(ctx, &wc); +} + static void poll_cq(RdmaDeviceResources *rdma_dev_res, struct ibv_cq *ibcq) { int i, ne; @@ -90,7 +101,7 @@ static void poll_cq(RdmaDeviceResources *rdma_dev_res, s= truct ibv_cq *ibcq) } pr_dbg("Processing %s CQE\n", bctx->is_tx_req ? "send" : "recv= "); =20 - comp_handler(wc[i].status, wc[i].vendor_err, bctx->up_ctx); + comp_handler(bctx->up_ctx, &wc[i]); =20 rdma_rm_dealloc_cqe_ctx(rdma_dev_res, wc[i].wr_id); g_free(bctx); @@ -184,8 +195,8 @@ static void start_comp_thread(RdmaBackendDev *backend_d= ev) comp_handler_thread, backend_dev, QEMU_THREAD_DETAC= HED); } =20 -void rdma_backend_register_comp_handler(void (*handler)(int status, - unsigned int vendor_err, void *ctx= )) +void rdma_backend_register_comp_handler(void (*handler)(void *ctx, + struct ibv_wc *wc= )) { comp_handler =3D handler; } @@ -369,14 +380,14 @@ void rdma_backend_post_send(RdmaBackendDev *backend_d= ev, if (!qp->ibqp) { /* This field does not get initialized for QP0 and QP= 1 */ if (qp_type =3D=3D IBV_QPT_SMI) { pr_dbg("QP0 unsupported\n"); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_QP0, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_QP0, ctx); } else if (qp_type =3D=3D IBV_QPT_GSI) { pr_dbg("QP1\n"); rc =3D mad_send(backend_dev, sgid_idx, sgid, sge, num_sge); if (rc) { - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_MAD_SEND, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_MAD_SEND, ctx= ); } else { - comp_handler(IBV_WC_SUCCESS, 0, ctx); + complete_work(IBV_WC_SUCCESS, 0, ctx); } } return; @@ -385,7 +396,7 @@ void rdma_backend_post_send(RdmaBackendDev *backend_dev, pr_dbg("num_sge=3D%d\n", num_sge); if (!num_sge) { pr_dbg("num_sge=3D0\n"); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_NO_SGE, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NO_SGE, ctx); return; } =20 @@ -396,21 +407,21 @@ void rdma_backend_post_send(RdmaBackendDev *backend_d= ev, rc =3D rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx= ); if (unlikely(rc)) { pr_dbg("Failed to allocate cqe_ctx\n"); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx); goto out_free_bctx; } =20 rc =3D build_host_sge_array(backend_dev->rdma_dev_res, new_sge, sge, n= um_sge); if (rc) { pr_dbg("Error: Failed to build host SGE array\n"); - comp_handler(IBV_WC_GENERAL_ERR, rc, ctx); + complete_work(IBV_WC_GENERAL_ERR, rc, ctx); goto out_dealloc_cqe_ctx; } =20 if (qp_type =3D=3D IBV_QPT_UD) { wr.wr.ud.ah =3D create_ah(backend_dev, qp->ibpd, sgid_idx, dgid); if (!wr.wr.ud.ah) { - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx= ); goto out_dealloc_cqe_ctx; } wr.wr.ud.remote_qpn =3D dqpn; @@ -428,7 +439,7 @@ void rdma_backend_post_send(RdmaBackendDev *backend_dev, if (rc) { pr_dbg("Fail (%d, %d) to post send WQE to qpn %d\n", rc, errno, qp->ibqp->qp_num); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx); goto out_dealloc_cqe_ctx; } =20 @@ -497,13 +508,13 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_d= ev, if (!qp->ibqp) { /* This field does not get initialized for QP0 and QP= 1 */ if (qp_type =3D=3D IBV_QPT_SMI) { pr_dbg("QP0 unsupported\n"); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_QP0, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_QP0, ctx); } if (qp_type =3D=3D IBV_QPT_GSI) { pr_dbg("QP1\n"); rc =3D save_mad_recv_buffer(backend_dev, sge, num_sge, ctx); if (rc) { - comp_handler(IBV_WC_GENERAL_ERR, rc, ctx); + complete_work(IBV_WC_GENERAL_ERR, rc, ctx); } } return; @@ -512,7 +523,7 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev, pr_dbg("num_sge=3D%d\n", num_sge); if (!num_sge) { pr_dbg("num_sge=3D0\n"); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_NO_SGE, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NO_SGE, ctx); return; } =20 @@ -523,14 +534,14 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_d= ev, rc =3D rdma_rm_alloc_cqe_ctx(rdma_dev_res, &bctx_id, bctx); if (unlikely(rc)) { pr_dbg("Failed to allocate cqe_ctx\n"); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx); goto out_free_bctx; } =20 rc =3D build_host_sge_array(rdma_dev_res, new_sge, sge, num_sge); if (rc) { pr_dbg("Error: Failed to build host SGE array\n"); - comp_handler(IBV_WC_GENERAL_ERR, rc, ctx); + complete_work(IBV_WC_GENERAL_ERR, rc, ctx); goto out_dealloc_cqe_ctx; } =20 @@ -542,7 +553,7 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev, if (rc) { pr_dbg("Fail (%d, %d) to post recv WQE to qpn %d\n", rc, errno, qp->ibqp->qp_num); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx); goto out_dealloc_cqe_ctx; } =20 @@ -926,9 +937,10 @@ static void mad_read(void *opaque, const uint8_t *buf,= int size) mad =3D rdma_pci_dma_map(backend_dev->dev, bctx->sge.addr, bctx->sge.length); if (!mad || bctx->sge.length < msg->umad_len + MAD_HDR_SIZE) { - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_INV_MAD_BUFF, - bctx->up_ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_INV_MAD_BUFF, + bctx->up_ctx); } else { + struct ibv_wc wc =3D {0}; pr_dbg_buf("mad", msg->umad.mad, msg->umad_len); memset(mad, 0, bctx->sge.length); build_mad_hdr((struct ibv_grh *)mad, @@ -937,7 +949,10 @@ static void mad_read(void *opaque, const uint8_t *buf,= int size) memcpy(&mad[MAD_HDR_SIZE], msg->umad.mad, msg->umad_len); rdma_pci_dma_unmap(backend_dev->dev, mad, bctx->sge.length); =20 - comp_handler(IBV_WC_SUCCESS, 0, bctx->up_ctx); + wc.byte_len =3D msg->umad_len; + wc.status =3D IBV_WC_SUCCESS; + wc.wc_flags =3D IBV_WC_GRH; + comp_handler(bctx->up_ctx, &wc); } =20 g_free(bctx); diff --git a/hw/rdma/rdma_backend.h b/hw/rdma/rdma_backend.h index 59ad2b874b..8cae40f827 100644 --- a/hw/rdma/rdma_backend.h +++ b/hw/rdma/rdma_backend.h @@ -57,8 +57,8 @@ int rdma_backend_get_gid_index(RdmaBackendDev *backend_de= v, union ibv_gid *gid); void rdma_backend_start(RdmaBackendDev *backend_dev); void rdma_backend_stop(RdmaBackendDev *backend_dev); -void rdma_backend_register_comp_handler(void (*handler)(int status, - unsigned int vendor_err, void *ctx= )); +void rdma_backend_register_comp_handler(void (*handler)(void *ctx, + struct ibv_wc *wc)= ); void rdma_backend_unregister_comp_handler(void); =20 int rdma_backend_query_port(RdmaBackendDev *backend_dev, diff --git a/hw/rdma/vmw/pvrdma_qp_ops.c b/hw/rdma/vmw/pvrdma_qp_ops.c index 2130824098..300471a4c9 100644 --- a/hw/rdma/vmw/pvrdma_qp_ops.c +++ b/hw/rdma/vmw/pvrdma_qp_ops.c @@ -47,7 +47,7 @@ typedef struct PvrdmaRqWqe { * 3. Interrupt host */ static int pvrdma_post_cqe(PVRDMADev *dev, uint32_t cq_handle, - struct pvrdma_cqe *cqe) + struct pvrdma_cqe *cqe, struct ibv_wc *wc) { struct pvrdma_cqe *cqe1; struct pvrdma_cqne *cqne; @@ -66,6 +66,7 @@ static int pvrdma_post_cqe(PVRDMADev *dev, uint32_t cq_ha= ndle, pr_dbg("Writing CQE\n"); cqe1 =3D pvrdma_ring_next_elem_write(ring); if (unlikely(!cqe1)) { + pr_dbg("No CQEs in ring\n"); return -EINVAL; } =20 @@ -73,8 +74,20 @@ static int pvrdma_post_cqe(PVRDMADev *dev, uint32_t cq_h= andle, cqe1->wr_id =3D cqe->wr_id; cqe1->qp =3D cqe->qp; cqe1->opcode =3D cqe->opcode; - cqe1->status =3D cqe->status; - cqe1->vendor_err =3D cqe->vendor_err; + cqe1->status =3D wc->status; + cqe1->byte_len =3D wc->byte_len; + cqe1->src_qp =3D wc->src_qp; + cqe1->wc_flags =3D wc->wc_flags; + cqe1->vendor_err =3D wc->vendor_err; + + pr_dbg("wr_id=3D%" PRIx64 "\n", cqe1->wr_id); + pr_dbg("qp=3D0x%lx\n", cqe1->qp); + pr_dbg("opcode=3D%d\n", cqe1->opcode); + pr_dbg("status=3D%d\n", cqe1->status); + pr_dbg("byte_len=3D%d\n", cqe1->byte_len); + pr_dbg("src_qp=3D%d\n", cqe1->src_qp); + pr_dbg("wc_flags=3D%d\n", cqe1->wc_flags); + pr_dbg("vendor_err=3D%d\n", cqe1->vendor_err); =20 pvrdma_ring_write_inc(ring); =20 @@ -99,18 +112,12 @@ static int pvrdma_post_cqe(PVRDMADev *dev, uint32_t cq= _handle, return 0; } =20 -static void pvrdma_qp_ops_comp_handler(int status, unsigned int vendor_err, - void *ctx) +static void pvrdma_qp_ops_comp_handler(void *ctx, struct ibv_wc *wc) { CompHandlerCtx *comp_ctx =3D (CompHandlerCtx *)ctx; =20 - pr_dbg("cq_handle=3D%d\n", comp_ctx->cq_handle); - pr_dbg("wr_id=3D%" PRIx64 "\n", comp_ctx->cqe.wr_id); - pr_dbg("status=3D%d\n", status); - pr_dbg("vendor_err=3D0x%x\n", vendor_err); - comp_ctx->cqe.status =3D status; - comp_ctx->cqe.vendor_err =3D vendor_err; - pvrdma_post_cqe(comp_ctx->dev, comp_ctx->cq_handle, &comp_ctx->cqe); + pvrdma_post_cqe(comp_ctx->dev, comp_ctx->cq_handle, &comp_ctx->cqe, wc= ); + g_free(ctx); } =20 --=20 2.17.2