From nobody Thu Nov 6 19:42:32 2025 Delivered-To: importer@patchew.org Received-SPF: temperror (zoho.com: Error in retrieving data from DNS) client-ip=208.118.235.17; envelope-from=qemu-devel-bounces+importer=patchew.org@nongnu.org; helo=lists.gnu.org; Authentication-Results: mx.zohomail.com; dkim=fail; spf=temperror (zoho.com: Error in retrieving data from DNS) smtp.mailfrom=qemu-devel-bounces+importer=patchew.org@nongnu.org; dmarc=fail(p=none dis=none) header.from=oracle.com Return-Path: Received: from lists.gnu.org (208.118.235.17 [208.118.235.17]) by mx.zohomail.com with SMTPS id 1542889469242778.3835939526149; Thu, 22 Nov 2018 04:24:29 -0800 (PST) Received: from localhost ([::1]:45441 helo=lists.gnu.org) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gPo1e-0000EU-0A for importer@patchew.org; Thu, 22 Nov 2018 07:24:06 -0500 Received: from eggs.gnu.org ([208.118.235.92]:55019) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1gPnta-0000Nv-4T for qemu-devel@nongnu.org; Thu, 22 Nov 2018 07:15:48 -0500 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1gPnt0-0000si-PF for qemu-devel@nongnu.org; Thu, 22 Nov 2018 07:15:14 -0500 Received: from userp2130.oracle.com ([156.151.31.86]:53218) by eggs.gnu.org with esmtps (TLS1.0:RSA_AES_256_CBC_SHA1:32) (Exim 4.71) (envelope-from ) id 1gPnt0-0000or-Go for qemu-devel@nongnu.org; Thu, 22 Nov 2018 07:15:10 -0500 Received: from pps.filterd (userp2130.oracle.com [127.0.0.1]) by userp2130.oracle.com (8.16.0.22/8.16.0.22) with SMTP id wAMCEbTV068358; Thu, 22 Nov 2018 12:15:08 GMT Received: from userv0022.oracle.com (userv0022.oracle.com [156.151.31.74]) by userp2130.oracle.com with ESMTP id 2ntadu6nt2-1 (version=TLSv1.2 cipher=ECDHE-RSA-AES256-GCM-SHA384 bits=256 verify=OK); Thu, 22 Nov 2018 12:15:08 +0000 Received: from userv0122.oracle.com (userv0122.oracle.com [156.151.31.75]) by userv0022.oracle.com (8.14.4/8.14.4) with ESMTP id wAMCF8i0001056 (version=TLSv1/SSLv3 cipher=DHE-RSA-AES256-GCM-SHA384 bits=256 verify=OK); Thu, 22 Nov 2018 12:15:08 GMT Received: from abhmp0006.oracle.com (abhmp0006.oracle.com [141.146.116.12]) by userv0122.oracle.com (8.14.4/8.14.4) with ESMTP id wAMCF7xk009751; Thu, 22 Nov 2018 12:15:07 GMT Received: from localhost.localdomain (/77.138.186.148) by default (Oracle Beehive Gateway v4.0) with ESMTP ; Thu, 22 Nov 2018 04:15:07 -0800 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=oracle.com; h=from : to : subject : date : message-id : in-reply-to : references; s=corp-2018-07-02; bh=rkFzsbUaQKGQYWt0qo0ue/85PFTk8mpTl72PaEyycw4=; b=mLGFkRJaazTxfBCFHQZOx0irH0ak4Ta9bDZ/5ImKCETucCQs1CLBcHuAVFSuZkTfSiwG J91aGl8t0ivx5LcRCg29VLfW7pDUnh5gPXx/Y0Weu0gV5vQpg3YRKnAcpeU3lefLs72M rqJ8VjWpo9NH2TiG+ESlwt0zSB1A8bDZyW1D4Ny5EDNrbmpuEwMMO10EMW/du+9r8uv7 uMNfnWb7c0GMSXrB/t3eXLPbYBZsPkX6BMDg5YSkRzm9bVWrwdxMsGF9SOyvhGV7MgMI nZIfs8No9Tt5KYRYSyWTsRGdnbQ+PpAY3pLWdtEJZNZzQvghJHNOlWEjGA0QSaS3+/VV rA== From: Yuval Shaia To: yuval.shaia@oracle.com, marcel.apfelbaum@gmail.com, dmitry.fleytman@gmail.com, jasowang@redhat.com, eblake@redhat.com, armbru@redhat.com, pbonzini@redhat.com, qemu-devel@nongnu.org, shamir.rabinovitch@oracle.com, cohuck@redhat.com Date: Thu, 22 Nov 2018 14:13:54 +0200 Message-Id: <20181122121402.13764-17-yuval.shaia@oracle.com> X-Mailer: git-send-email 2.17.2 In-Reply-To: <20181122121402.13764-1-yuval.shaia@oracle.com> References: <20181122121402.13764-1-yuval.shaia@oracle.com> X-Proofpoint-Virus-Version: vendor=nai engine=5900 definitions=9084 signatures=668683 X-Proofpoint-Spam-Details: rule=notspam policy=default score=0 suspectscore=2 malwarescore=0 phishscore=0 bulkscore=0 spamscore=0 mlxscore=0 mlxlogscore=822 adultscore=0 classifier=spam adjust=0 reason=mlx scancount=1 engine=8.0.1-1810050000 definitions=main-1811220111 X-detected-operating-system: by eggs.gnu.org: GNU/Linux 3.x [generic] [fuzzy] X-Received-From: 156.151.31.86 Subject: [Qemu-devel] [PATCH v5 16/24] hw/pvrdma: Fill all CQE fields X-BeenThere: qemu-devel@nongnu.org X-Mailman-Version: 2.1.21 Precedence: list List-Id: List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: qemu-devel-bounces+importer=patchew.org@nongnu.org Sender: "Qemu-devel" X-ZohoMail-DKIM: fail (Header signature does not verify) Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" Add ability to pass specific WC attributes to CQE such as GRH_BIT flag. Signed-off-by: Yuval Shaia Reviewed-by: Marcel Apfelbaum --- hw/rdma/rdma_backend.c | 59 +++++++++++++++++++++++-------------- hw/rdma/rdma_backend.h | 4 +-- hw/rdma/vmw/pvrdma_qp_ops.c | 31 +++++++++++-------- 3 files changed, 58 insertions(+), 36 deletions(-) diff --git a/hw/rdma/rdma_backend.c b/hw/rdma/rdma_backend.c index 8b5a111bf4..6a1e39d4c0 100644 --- a/hw/rdma/rdma_backend.c +++ b/hw/rdma/rdma_backend.c @@ -60,13 +60,24 @@ struct backend_umad { char mad[RDMA_MAX_PRIVATE_DATA]; }; =20 -static void (*comp_handler)(int status, unsigned int vendor_err, void *ctx= ); +static void (*comp_handler)(void *ctx, struct ibv_wc *wc); =20 -static void dummy_comp_handler(int status, unsigned int vendor_err, void *= ctx) +static void dummy_comp_handler(void *ctx, struct ibv_wc *wc) { pr_err("No completion handler is registered\n"); } =20 +static inline void complete_work(enum ibv_wc_status status, uint32_t vendo= r_err, + void *ctx) +{ + struct ibv_wc wc =3D {0}; + + wc.status =3D status; + wc.vendor_err =3D vendor_err; + + comp_handler(ctx, &wc); +} + static void poll_cq(RdmaDeviceResources *rdma_dev_res, struct ibv_cq *ibcq) { int i, ne; @@ -91,7 +102,7 @@ static void poll_cq(RdmaDeviceResources *rdma_dev_res, s= truct ibv_cq *ibcq) } pr_dbg("Processing %s CQE\n", bctx->is_tx_req ? "send" : "recv= "); =20 - comp_handler(wc[i].status, wc[i].vendor_err, bctx->up_ctx); + comp_handler(bctx->up_ctx, &wc[i]); =20 rdma_rm_dealloc_cqe_ctx(rdma_dev_res, wc[i].wr_id); g_free(bctx); @@ -250,8 +261,8 @@ static void start_comp_thread(RdmaBackendDev *backend_d= ev) comp_handler_thread, backend_dev, QEMU_THREAD_DETAC= HED); } =20 -void rdma_backend_register_comp_handler(void (*handler)(int status, - unsigned int vendor_err, void *ctx= )) +void rdma_backend_register_comp_handler(void (*handler)(void *ctx, + struct ibv_wc *wc= )) { comp_handler =3D handler; } @@ -445,14 +456,14 @@ void rdma_backend_post_send(RdmaBackendDev *backend_d= ev, if (!qp->ibqp) { /* This field does not get initialized for QP0 and QP= 1 */ if (qp_type =3D=3D IBV_QPT_SMI) { pr_dbg("QP0 unsupported\n"); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_QP0, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_QP0, ctx); } else if (qp_type =3D=3D IBV_QPT_GSI) { pr_dbg("QP1\n"); rc =3D mad_send(backend_dev, sgid_idx, sgid, sge, num_sge); if (rc) { - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_MAD_SEND, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_MAD_SEND, ctx= ); } else { - comp_handler(IBV_WC_SUCCESS, 0, ctx); + complete_work(IBV_WC_SUCCESS, 0, ctx); } } return; @@ -461,7 +472,7 @@ void rdma_backend_post_send(RdmaBackendDev *backend_dev, pr_dbg("num_sge=3D%d\n", num_sge); if (!num_sge) { pr_dbg("num_sge=3D0\n"); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_NO_SGE, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NO_SGE, ctx); return; } =20 @@ -472,21 +483,21 @@ void rdma_backend_post_send(RdmaBackendDev *backend_d= ev, rc =3D rdma_rm_alloc_cqe_ctx(backend_dev->rdma_dev_res, &bctx_id, bctx= ); if (unlikely(rc)) { pr_dbg("Failed to allocate cqe_ctx\n"); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx); goto out_free_bctx; } =20 rc =3D build_host_sge_array(backend_dev->rdma_dev_res, new_sge, sge, n= um_sge); if (rc) { pr_dbg("Error: Failed to build host SGE array\n"); - comp_handler(IBV_WC_GENERAL_ERR, rc, ctx); + complete_work(IBV_WC_GENERAL_ERR, rc, ctx); goto out_dealloc_cqe_ctx; } =20 if (qp_type =3D=3D IBV_QPT_UD) { wr.wr.ud.ah =3D create_ah(backend_dev, qp->ibpd, sgid_idx, dgid); if (!wr.wr.ud.ah) { - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx= ); goto out_dealloc_cqe_ctx; } wr.wr.ud.remote_qpn =3D dqpn; @@ -504,7 +515,7 @@ void rdma_backend_post_send(RdmaBackendDev *backend_dev, if (rc) { pr_dbg("Fail (%d, %d) to post send WQE to qpn %d\n", rc, errno, qp->ibqp->qp_num); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx); goto out_dealloc_cqe_ctx; } =20 @@ -573,13 +584,13 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_d= ev, if (!qp->ibqp) { /* This field does not get initialized for QP0 and QP= 1 */ if (qp_type =3D=3D IBV_QPT_SMI) { pr_dbg("QP0 unsupported\n"); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_QP0, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_QP0, ctx); } if (qp_type =3D=3D IBV_QPT_GSI) { pr_dbg("QP1\n"); rc =3D save_mad_recv_buffer(backend_dev, sge, num_sge, ctx); if (rc) { - comp_handler(IBV_WC_GENERAL_ERR, rc, ctx); + complete_work(IBV_WC_GENERAL_ERR, rc, ctx); } } return; @@ -588,7 +599,7 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev, pr_dbg("num_sge=3D%d\n", num_sge); if (!num_sge) { pr_dbg("num_sge=3D0\n"); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_NO_SGE, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NO_SGE, ctx); return; } =20 @@ -599,14 +610,14 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_d= ev, rc =3D rdma_rm_alloc_cqe_ctx(rdma_dev_res, &bctx_id, bctx); if (unlikely(rc)) { pr_dbg("Failed to allocate cqe_ctx\n"); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_NOMEM, ctx); goto out_free_bctx; } =20 rc =3D build_host_sge_array(rdma_dev_res, new_sge, sge, num_sge); if (rc) { pr_dbg("Error: Failed to build host SGE array\n"); - comp_handler(IBV_WC_GENERAL_ERR, rc, ctx); + complete_work(IBV_WC_GENERAL_ERR, rc, ctx); goto out_dealloc_cqe_ctx; } =20 @@ -618,7 +629,7 @@ void rdma_backend_post_recv(RdmaBackendDev *backend_dev, if (rc) { pr_dbg("Fail (%d, %d) to post recv WQE to qpn %d\n", rc, errno, qp->ibqp->qp_num); - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_FAIL_BACKEND, ctx); goto out_dealloc_cqe_ctx; } =20 @@ -992,9 +1003,10 @@ static void process_incoming_mad_req(RdmaBackendDev *= backend_dev, mad =3D rdma_pci_dma_map(backend_dev->dev, bctx->sge.addr, bctx->sge.length); if (!mad || bctx->sge.length < msg->umad_len + MAD_HDR_SIZE) { - comp_handler(IBV_WC_GENERAL_ERR, VENDOR_ERR_INV_MAD_BUFF, - bctx->up_ctx); + complete_work(IBV_WC_GENERAL_ERR, VENDOR_ERR_INV_MAD_BUFF, + bctx->up_ctx); } else { + struct ibv_wc wc =3D {0}; pr_dbg_buf("mad", msg->umad.mad, msg->umad_len); memset(mad, 0, bctx->sge.length); build_mad_hdr((struct ibv_grh *)mad, @@ -1003,7 +1015,10 @@ static void process_incoming_mad_req(RdmaBackendDev = *backend_dev, memcpy(&mad[MAD_HDR_SIZE], msg->umad.mad, msg->umad_len); rdma_pci_dma_unmap(backend_dev->dev, mad, bctx->sge.length); =20 - comp_handler(IBV_WC_SUCCESS, 0, bctx->up_ctx); + wc.byte_len =3D msg->umad_len; + wc.status =3D IBV_WC_SUCCESS; + wc.wc_flags =3D IBV_WC_GRH; + comp_handler(bctx->up_ctx, &wc); } =20 g_free(bctx); diff --git a/hw/rdma/rdma_backend.h b/hw/rdma/rdma_backend.h index 59ad2b874b..8cae40f827 100644 --- a/hw/rdma/rdma_backend.h +++ b/hw/rdma/rdma_backend.h @@ -57,8 +57,8 @@ int rdma_backend_get_gid_index(RdmaBackendDev *backend_de= v, union ibv_gid *gid); void rdma_backend_start(RdmaBackendDev *backend_dev); void rdma_backend_stop(RdmaBackendDev *backend_dev); -void rdma_backend_register_comp_handler(void (*handler)(int status, - unsigned int vendor_err, void *ctx= )); +void rdma_backend_register_comp_handler(void (*handler)(void *ctx, + struct ibv_wc *wc)= ); void rdma_backend_unregister_comp_handler(void); =20 int rdma_backend_query_port(RdmaBackendDev *backend_dev, diff --git a/hw/rdma/vmw/pvrdma_qp_ops.c b/hw/rdma/vmw/pvrdma_qp_ops.c index 2130824098..300471a4c9 100644 --- a/hw/rdma/vmw/pvrdma_qp_ops.c +++ b/hw/rdma/vmw/pvrdma_qp_ops.c @@ -47,7 +47,7 @@ typedef struct PvrdmaRqWqe { * 3. Interrupt host */ static int pvrdma_post_cqe(PVRDMADev *dev, uint32_t cq_handle, - struct pvrdma_cqe *cqe) + struct pvrdma_cqe *cqe, struct ibv_wc *wc) { struct pvrdma_cqe *cqe1; struct pvrdma_cqne *cqne; @@ -66,6 +66,7 @@ static int pvrdma_post_cqe(PVRDMADev *dev, uint32_t cq_ha= ndle, pr_dbg("Writing CQE\n"); cqe1 =3D pvrdma_ring_next_elem_write(ring); if (unlikely(!cqe1)) { + pr_dbg("No CQEs in ring\n"); return -EINVAL; } =20 @@ -73,8 +74,20 @@ static int pvrdma_post_cqe(PVRDMADev *dev, uint32_t cq_h= andle, cqe1->wr_id =3D cqe->wr_id; cqe1->qp =3D cqe->qp; cqe1->opcode =3D cqe->opcode; - cqe1->status =3D cqe->status; - cqe1->vendor_err =3D cqe->vendor_err; + cqe1->status =3D wc->status; + cqe1->byte_len =3D wc->byte_len; + cqe1->src_qp =3D wc->src_qp; + cqe1->wc_flags =3D wc->wc_flags; + cqe1->vendor_err =3D wc->vendor_err; + + pr_dbg("wr_id=3D%" PRIx64 "\n", cqe1->wr_id); + pr_dbg("qp=3D0x%lx\n", cqe1->qp); + pr_dbg("opcode=3D%d\n", cqe1->opcode); + pr_dbg("status=3D%d\n", cqe1->status); + pr_dbg("byte_len=3D%d\n", cqe1->byte_len); + pr_dbg("src_qp=3D%d\n", cqe1->src_qp); + pr_dbg("wc_flags=3D%d\n", cqe1->wc_flags); + pr_dbg("vendor_err=3D%d\n", cqe1->vendor_err); =20 pvrdma_ring_write_inc(ring); =20 @@ -99,18 +112,12 @@ static int pvrdma_post_cqe(PVRDMADev *dev, uint32_t cq= _handle, return 0; } =20 -static void pvrdma_qp_ops_comp_handler(int status, unsigned int vendor_err, - void *ctx) +static void pvrdma_qp_ops_comp_handler(void *ctx, struct ibv_wc *wc) { CompHandlerCtx *comp_ctx =3D (CompHandlerCtx *)ctx; =20 - pr_dbg("cq_handle=3D%d\n", comp_ctx->cq_handle); - pr_dbg("wr_id=3D%" PRIx64 "\n", comp_ctx->cqe.wr_id); - pr_dbg("status=3D%d\n", status); - pr_dbg("vendor_err=3D0x%x\n", vendor_err); - comp_ctx->cqe.status =3D status; - comp_ctx->cqe.vendor_err =3D vendor_err; - pvrdma_post_cqe(comp_ctx->dev, comp_ctx->cq_handle, &comp_ctx->cqe); + pvrdma_post_cqe(comp_ctx->dev, comp_ctx->cq_handle, &comp_ctx->cqe, wc= ); + g_free(ctx); } =20 --=20 2.17.2