From nobody Wed Dec 17 10:42:36 2025 Return-Path: X-Spam-Checker-Version: SpamAssassin 3.4.0 (2014-02-07) on aws-us-west-2-korg-lkml-1.web.codeaurora.org Received: from vger.kernel.org (vger.kernel.org [23.128.96.18]) by smtp.lore.kernel.org (Postfix) with ESMTP id 0668AEE4993 for ; Tue, 22 Aug 2023 15:47:54 +0000 (UTC) Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S237334AbjHVPrx (ORCPT ); Tue, 22 Aug 2023 11:47:53 -0400 Received: from lindbergh.monkeyblade.net ([23.128.96.19]:48754 "EHLO lindbergh.monkeyblade.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S237317AbjHVPrt (ORCPT ); Tue, 22 Aug 2023 11:47:49 -0400 Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by lindbergh.monkeyblade.net (Postfix) with ESMTP id 347AACE3; Tue, 22 Aug 2023 08:47:46 -0700 (PDT) Received: by linux.microsoft.com (Postfix, from userid 1174) id 9B98C2126CC6; Tue, 22 Aug 2023 08:47:45 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com 9B98C2126CC6 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linuxonhyperv.com; s=default; t=1692719265; bh=NkCMp454GTUcOtiLH5kTM0L3OzT2pp424P12IBewYiE=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=a9bm1O/H8WVdeGfqL47r4HkFxLxUQfjYmlusZaE4NZ6k6Gvj3qN1LXrcFM45HK81c Wf0rsJgrf+0Jrpacu3oU5NNoorQdrktscb60pYYIR9teF4yFHC0LM4t6NbP2rkq/nC MgkDfIfJuzWqWj8yzC0Ufod2QZjL1jIzAN4ReZEk= From: sharmaajay@linuxonhyperv.com To: Long Li , Jason Gunthorpe , Leon Romanovsky , Dexuan Cui , Wei Liu , "David S. Miller" , Eric Dumazet , Jakub Kicinski , Paolo Abeni Cc: linux-rdma@vger.kernel.org, linux-hyperv@vger.kernel.org, netdev@vger.kernel.org, linux-kernel@vger.kernel.org, Ajay Sharma Subject: [Patch v4 2/4] RDMA/mana_ib : Register Mana IB device with Management SW Date: Tue, 22 Aug 2023 08:47:33 -0700 Message-Id: <1692719255-20183-3-git-send-email-sharmaajay@linuxonhyperv.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1692719255-20183-1-git-send-email-sharmaajay@linuxonhyperv.com> References: <1692719255-20183-1-git-send-email-sharmaajay@linuxonhyperv.com> Precedence: bulk List-ID: X-Mailing-List: linux-kernel@vger.kernel.org Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" From: Ajay Sharma Each of the MANA infiniband devices must be registered with the management software to request services/resources. Register the Mana IB device with Management which would later help get an adapter handle. Signed-off-by: Ajay Sharma --- drivers/infiniband/hw/mana/device.c | 20 +++++-- drivers/infiniband/hw/mana/main.c | 58 ++++++------------- drivers/infiniband/hw/mana/mana_ib.h | 1 + drivers/infiniband/hw/mana/mr.c | 17 ++---- drivers/infiniband/hw/mana/qp.c | 8 +-- .../net/ethernet/microsoft/mana/gdma_main.c | 5 ++ include/net/mana/gdma.h | 3 + 7 files changed, 54 insertions(+), 58 deletions(-) diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/ma= na/device.c index 083f27246ba8..ea4c8c8fc10d 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -78,22 +78,34 @@ static int mana_ib_probe(struct auxiliary_device *adev, mib_dev->ib_dev.num_comp_vectors =3D 1; mib_dev->ib_dev.dev.parent =3D mdev->gdma_context->dev; =20 - ret =3D ib_register_device(&mib_dev->ib_dev, "mana_%d", - mdev->gdma_context->dev); + ret =3D mana_gd_register_device(&mib_dev->gc->mana_ib); if (ret) { - ib_dealloc_device(&mib_dev->ib_dev); - return ret; + ibdev_err(&mib_dev->ib_dev, "Failed to register device, ret %d", + ret); + goto free_ib_device; } =20 + ret =3D ib_register_device(&mib_dev->ib_dev, "mana_%d", + mdev->gdma_context->dev); + if (ret) + goto deregister_device; + dev_set_drvdata(&adev->dev, mib_dev); =20 return 0; + +deregister_device: + mana_gd_deregister_device(&mib_dev->gc->mana_ib); +free_ib_device: + ib_dealloc_device(&mib_dev->ib_dev); + return ret; } =20 static void mana_ib_remove(struct auxiliary_device *adev) { struct mana_ib_dev *mib_dev =3D dev_get_drvdata(&adev->dev); =20 + mana_gd_deregister_device(&mib_dev->gc->mana_ib); ib_unregister_device(&mib_dev->ib_dev); ib_dealloc_device(&mib_dev->ib_dev); } diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana= /main.c index 189e774cdab6..2c4e3c496644 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -8,7 +8,7 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *mib_dev, struct mana_ib_pd *p= d, u32 port) { - struct gdma_dev *gd =3D mib_dev->gdma_dev; + struct gdma_dev *gd =3D &mib_dev->gc->mana; struct mana_port_context *mpc; struct net_device *ndev; struct mana_context *mc; @@ -32,7 +32,7 @@ int mana_ib_cfg_vport(struct mana_ib_dev *mib_dev, u32 po= rt, struct mana_ib_pd *pd, u32 doorbell_id) { - struct gdma_dev *mdev =3D mib_dev->gdma_dev; + struct gdma_dev *mdev =3D &mib_dev->gc->mana; struct mana_port_context *mpc; struct mana_context *mc; struct net_device *ndev; @@ -81,17 +81,16 @@ int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udat= a *udata) struct gdma_create_pd_req req =3D {}; enum gdma_pd_flags flags =3D 0; struct mana_ib_dev *mib_dev; - struct gdma_dev *mdev; + int err; =20 mib_dev =3D container_of(ibdev, struct mana_ib_dev, ib_dev); - mdev =3D mib_dev->gdma_dev; =20 mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_PD, sizeof(req), sizeof(resp)); =20 req.flags =3D flags; - err =3D mana_gd_send_request(mdev->gdma_context, sizeof(req), &req, + err =3D mana_gd_send_request(mib_dev->gc, sizeof(req), &req, sizeof(resp), &resp); =20 if (err || resp.hdr.status) { @@ -121,17 +120,15 @@ int mana_ib_dealloc_pd(struct ib_pd *ibpd, struct ib_= udata *udata) struct gdma_destory_pd_resp resp =3D {}; struct gdma_destroy_pd_req req =3D {}; struct mana_ib_dev *mib_dev; - struct gdma_dev *mdev; int err; =20 mib_dev =3D container_of(ibdev, struct mana_ib_dev, ib_dev); - mdev =3D mib_dev->gdma_dev; =20 mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_PD, sizeof(req), sizeof(resp)); =20 req.pd_handle =3D pd->pd_handle; - err =3D mana_gd_send_request(mdev->gdma_context, sizeof(req), &req, + err =3D mana_gd_send_request(mib_dev->gc, sizeof(req), &req, sizeof(resp), &resp); =20 if (err || resp.hdr.status) { @@ -207,17 +204,13 @@ int mana_ib_alloc_ucontext(struct ib_ucontext *ibcont= ext, container_of(ibcontext, struct mana_ib_ucontext, ibucontext); struct ib_device *ibdev =3D ibcontext->device; struct mana_ib_dev *mib_dev; - struct gdma_context *gc; - struct gdma_dev *dev; int doorbell_page; int ret; =20 mib_dev =3D container_of(ibdev, struct mana_ib_dev, ib_dev); - dev =3D mib_dev->gdma_dev; - gc =3D dev->gdma_context; =20 /* Allocate a doorbell page index */ - ret =3D mana_gd_allocate_doorbell_page(gc, &doorbell_page); + ret =3D mana_gd_allocate_doorbell_page(mib_dev->gc, &doorbell_page); if (ret) { ibdev_dbg(ibdev, "Failed to allocate doorbell page %d\n", ret); return ret; @@ -236,20 +229,17 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext *ibc= ontext) container_of(ibcontext, struct mana_ib_ucontext, ibucontext); struct ib_device *ibdev =3D ibcontext->device; struct mana_ib_dev *mib_dev; - struct gdma_context *gc; int ret; =20 mib_dev =3D container_of(ibdev, struct mana_ib_dev, ib_dev); - gc =3D mib_dev->gdma_dev->gdma_context; =20 - ret =3D mana_gd_destroy_doorbell_page(gc, mana_ucontext->doorbell); + ret =3D mana_gd_destroy_doorbell_page(mib_dev->gc, mana_ucontext->doorbel= l); if (ret) ibdev_dbg(ibdev, "Failed to destroy doorbell page %d\n", ret); } =20 static int mana_ib_gd_first_dma_region(struct mana_ib_dev *mib_dev, - struct gdma_context *gc, struct gdma_create_dma_region_req *create_req, size_t num_pages, mana_handle_t *gdma_region, u32 expected_status) @@ -262,7 +252,7 @@ mana_ib_gd_first_dma_region(struct mana_ib_dev *mib_dev, struct_size(create_req, page_addr_list, num_pages); create_req->page_addr_list_len =3D num_pages; =20 - err =3D mana_gd_send_request(gc, create_req_msg_size, create_req, + err =3D mana_gd_send_request(mib_dev->gc, create_req_msg_size, create_req, sizeof(create_resp), &create_resp); if (err || create_resp.hdr.status !=3D expected_status) { ibdev_dbg(&mib_dev->ib_dev, @@ -282,7 +272,7 @@ mana_ib_gd_first_dma_region(struct mana_ib_dev *mib_dev, } =20 static int -mana_ib_gd_add_dma_region(struct mana_ib_dev *mib_dev, struct gdma_context= *gc, +mana_ib_gd_add_dma_region(struct mana_ib_dev *mib_dev, struct gdma_dma_region_add_pages_req *add_req, unsigned int num_pages, u32 expected_status) { @@ -295,7 +285,7 @@ mana_ib_gd_add_dma_region(struct mana_ib_dev *mib_dev, = struct gdma_context *gc, add_req_msg_size, sizeof(add_resp)); add_req->page_addr_list_len =3D num_pages; =20 - err =3D mana_gd_send_request(gc, add_req_msg_size, add_req, + err =3D mana_gd_send_request(mib_dev->gc, add_req_msg_size, add_req, sizeof(add_resp), &add_resp); if (err || add_resp.hdr.status !=3D expected_status) { ibdev_dbg(&mib_dev->ib_dev, @@ -323,18 +313,14 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *= mib_dev, struct ib_block_iter biter; size_t max_pgs_add_cmd =3D 0; size_t max_pgs_create_cmd; - struct gdma_context *gc; size_t num_pages_total; - struct gdma_dev *mdev; unsigned long page_sz; unsigned int tail =3D 0; u64 *page_addr_list; void *request_buf; int err; =20 - mdev =3D mib_dev->gdma_dev; - gc =3D mdev->gdma_context; - hwc =3D gc->hwc.driver_data; + hwc =3D mib_dev->gc->hwc.driver_data; =20 /* Hardware requires dma region to align to chosen page size */ page_sz =3D ib_umem_find_best_pgsz(umem, PAGE_SZ_BM, 0); @@ -388,7 +374,7 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *mi= b_dev, =20 if (!num_pages_processed) { /* First create message */ - err =3D mana_ib_gd_first_dma_region(mib_dev, gc, create_req, + err =3D mana_ib_gd_first_dma_region(mib_dev, create_req, tail, gdma_region, expected_status); if (err) @@ -403,7 +389,7 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *mi= b_dev, page_addr_list =3D add_req->page_addr_list; } else { /* Subsequent create messages */ - err =3D mana_ib_gd_add_dma_region(mib_dev, gc, add_req, tail, + err =3D mana_ib_gd_add_dma_region(mib_dev, add_req, tail, expected_status); if (err) break; @@ -429,13 +415,9 @@ int mana_ib_gd_create_dma_region(struct mana_ib_dev *m= ib_dev, =20 int mana_ib_gd_destroy_dma_region(struct mana_ib_dev *mib_dev, u64 gdma_re= gion) { - struct gdma_dev *mdev =3D mib_dev->gdma_dev; - struct gdma_context *gc; - - gc =3D mdev->gdma_context; ibdev_dbg(&mib_dev->ib_dev, "destroy dma region 0x%llx\n", gdma_region); =20 - return mana_gd_destroy_dma_region(gc, gdma_region); + return mana_gd_destroy_dma_region(mib_dev->gc, gdma_region); } =20 int mana_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma) @@ -444,13 +426,11 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struc= t vm_area_struct *vma) container_of(ibcontext, struct mana_ib_ucontext, ibucontext); struct ib_device *ibdev =3D ibcontext->device; struct mana_ib_dev *mib_dev; - struct gdma_context *gc; phys_addr_t pfn; pgprot_t prot; int ret; =20 mib_dev =3D container_of(ibdev, struct mana_ib_dev, ib_dev); - gc =3D mib_dev->gdma_dev->gdma_context; =20 if (vma->vm_pgoff !=3D 0) { ibdev_dbg(ibdev, "Unexpected vm_pgoff %lu\n", vma->vm_pgoff); @@ -458,18 +438,18 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struc= t vm_area_struct *vma) } =20 /* Map to the page indexed by ucontext->doorbell */ - pfn =3D (gc->phys_db_page_base + - gc->db_page_size * mana_ucontext->doorbell) >> + pfn =3D (mib_dev->gc->phys_db_page_base + + mib_dev->gc->db_page_size * mana_ucontext->doorbell) >> PAGE_SHIFT; prot =3D pgprot_writecombine(vma->vm_page_prot); =20 - ret =3D rdma_user_mmap_io(ibcontext, vma, pfn, gc->db_page_size, prot, - NULL); + ret =3D rdma_user_mmap_io(ibcontext, vma, pfn, mib_dev->gc->db_page_size, + prot, NULL); if (ret) ibdev_dbg(ibdev, "can't rdma_user_mmap_io ret %d\n", ret); else ibdev_dbg(ibdev, "mapped I/O pfn 0x%llx page_size %u, ret %d\n", - pfn, gc->db_page_size, ret); + pfn, mib_dev->gc->db_page_size, ret); =20 return ret; } diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/m= ana/mana_ib.h index ee4efd0af278..3a2ba6b96f15 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -30,6 +30,7 @@ struct mana_ib_dev { struct ib_device ib_dev; struct gdma_dev *gdma_dev; + struct gdma_context *gc; }; =20 struct mana_ib_wq { diff --git a/drivers/infiniband/hw/mana/mr.c b/drivers/infiniband/hw/mana/m= r.c index f6a53906204d..3106d1bce837 100644 --- a/drivers/infiniband/hw/mana/mr.c +++ b/drivers/infiniband/hw/mana/mr.c @@ -29,13 +29,10 @@ static int mana_ib_gd_create_mr(struct mana_ib_dev *mib= _dev, struct mana_ib_mr *mr, struct gdma_create_mr_params *mr_params) { - struct gdma_dev *mdev =3D mib_dev->gdma_dev; struct gdma_create_mr_response resp =3D {}; struct gdma_create_mr_request req =3D {}; - struct gdma_context *gc; int err; =20 - gc =3D mdev->gdma_context; =20 mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_MR, sizeof(req), sizeof(resp)); @@ -56,7 +53,8 @@ static int mana_ib_gd_create_mr(struct mana_ib_dev *mib_d= ev, return -EINVAL; } =20 - err =3D mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + err =3D mana_gd_send_request(mib_dev->gc, sizeof(req), &req, + sizeof(resp), &resp); =20 if (err || resp.hdr.status) { ibdev_dbg(&mib_dev->ib_dev, "Failed to create mr %d, %u", err, @@ -77,22 +75,19 @@ static int mana_ib_gd_create_mr(struct mana_ib_dev *mib= _dev, static int mana_ib_gd_destroy_mr(struct mana_ib_dev *mib_dev, u64 mr_handl= e) { struct gdma_destroy_mr_response resp =3D {}; - struct gdma_dev *mdev =3D mib_dev->gdma_dev; struct gdma_destroy_mr_request req =3D {}; - struct gdma_context *gc; int err; =20 - gc =3D mdev->gdma_context; - mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_MR, sizeof(req), sizeof(resp)); =20 req.mr_handle =3D mr_handle; =20 - err =3D mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + err =3D mana_gd_send_request(mib_dev->gc, sizeof(req), &req, + sizeof(resp), &resp); if (err || resp.hdr.status) { - dev_err(gc->dev, "Failed to destroy MR: %d, 0x%x\n", err, - resp.hdr.status); + dev_err(mib_dev->gc->dev, "Failed to destroy MR: %d, 0x%x\n", + err, resp.hdr.status); if (!err) err =3D -EPROTO; return err; diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/q= p.c index 3540bf5bb497..ef3275ac92a0 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -21,7 +21,7 @@ static int mana_ib_cfg_vport_steering(struct mana_ib_dev = *mib_dev, u32 req_buf_size; int i, err; =20 - mdev =3D mib_dev->gdma_dev; + mdev =3D &mib_dev->gc->mana; gc =3D mdev->gdma_context; =20 req_buf_size =3D @@ -267,7 +267,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, st= ruct ib_pd *ibpd, rdma_udata_to_drv_context(udata, struct mana_ib_ucontext, ibucontext); struct mana_ib_create_qp_resp resp =3D {}; - struct gdma_dev *gd =3D mib_dev->gdma_dev; + struct gdma_dev *gd =3D &mib_dev->gc->mana; struct mana_ib_create_qp ucmd =3D {}; struct mana_obj_spec wq_spec =3D {}; struct mana_obj_spec cq_spec =3D {}; @@ -437,7 +437,7 @@ static int mana_ib_destroy_qp_rss(struct mana_ib_qp *qp, { struct mana_ib_dev *mib_dev =3D container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); - struct gdma_dev *gd =3D mib_dev->gdma_dev; + struct gdma_dev *gd =3D &mib_dev->gc->mana; struct mana_port_context *mpc; struct mana_context *mc; struct net_device *ndev; @@ -464,7 +464,7 @@ static int mana_ib_destroy_qp_raw(struct mana_ib_qp *qp= , struct ib_udata *udata) { struct mana_ib_dev *mib_dev =3D container_of(qp->ibqp.device, struct mana_ib_dev, ib_dev); - struct gdma_dev *gd =3D mib_dev->gdma_dev; + struct gdma_dev *gd =3D &mib_dev->gc->mana; struct ib_pd *ibpd =3D qp->ibqp.pd; struct mana_port_context *mpc; struct mana_context *mc; diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/= ethernet/microsoft/mana/gdma_main.c index 8f3f78b68592..9fa7a2d6c2b2 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -139,6 +139,9 @@ static int mana_gd_detect_devices(struct pci_dev *pdev) if (dev_type =3D=3D GDMA_DEVICE_MANA) { gc->mana.gdma_context =3D gc; gc->mana.dev_id =3D dev; + } else if (dev_type =3D=3D GDMA_DEVICE_MANA_IB) { + gc->mana_ib.dev_id =3D dev; + gc->mana_ib.gdma_context =3D gc; } } =20 @@ -940,6 +943,7 @@ int mana_gd_register_device(struct gdma_dev *gd) =20 return 0; } +EXPORT_SYMBOL(mana_gd_register_device); =20 int mana_gd_deregister_device(struct gdma_dev *gd) { @@ -970,6 +974,7 @@ int mana_gd_deregister_device(struct gdma_dev *gd) =20 return err; } +EXPORT_SYMBOL(mana_gd_deregister_device); =20 u32 mana_gd_wq_avail_space(struct gdma_queue *wq) { diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index 96c120160f15..e2b212dd722b 100644 --- a/include/net/mana/gdma.h +++ b/include/net/mana/gdma.h @@ -63,6 +63,7 @@ enum { GDMA_DEVICE_NONE =3D 0, GDMA_DEVICE_HWC =3D 1, GDMA_DEVICE_MANA =3D 2, + GDMA_DEVICE_MANA_IB =3D 3, }; =20 struct gdma_resource { @@ -384,6 +385,8 @@ struct gdma_context { =20 /* Azure network adapter */ struct gdma_dev mana; + /* rdma device */ + struct gdma_dev mana_ib; }; =20 #define MAX_NUM_GDMA_DEVICES 4 --=20 2.25.1