From nobody Sat Feb 7 22:06:33 2026 Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 26A8B286422; Tue, 6 May 2025 18:52:22 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=13.77.154.182 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746557543; cv=none; b=DF/GObmCFew7v9yTdaiy64QTgQ/gtblxytaicz8Knlz7GnsLDntPosAdafbVEc7G/mAicZCRku56sJkrPAt8I9oWLCA6xjRYYlpaI2VdaRil+RW4tG0IYtqmaFuTCL9W+5Pl3hb+6o5KHXl2DLfqyA2hZswLzVOnH5oUmjKujkI= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746557543; c=relaxed/simple; bh=eWOm3tIdj0/52qr5XJF93UlruH3ToGnAu4hQP5+FsAs=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References; b=qals1uEdqOiBUQjVmMH6AhaRvSxWsiI23CdRxlOdz079hyvXYfi91+7pI+YqrJZdhfAEk3v62xufWgVWnYObAssNiygqZAd+Cz2mFmBIUnN7PTUxLEYfKr3pI8Dj5ljV45MkhIPtBN/mDE6qF7/TAPoQitohbhuyBIPJssPxSlU= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com; spf=pass smtp.mailfrom=linux.microsoft.com; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b=MPJK5BMt; arc=none smtp.client-ip=13.77.154.182 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b="MPJK5BMt" Received: by linux.microsoft.com (Postfix, from userid 1186) id B62BB2115DD4; Tue, 6 May 2025 11:52:21 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com B62BB2115DD4 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1746557541; bh=aCEe+5U2tYKwmEkS8vQ+CK+zdrUx5Hi7XJADJXyNCTU=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=MPJK5BMt02i8GuAhG+9LhtNqHA2V7oOU6ECJZaDokG7aLCzeD1WNdrNoRshVjm3kM l7+cRhqpCqAwQD8xUyaGO7c3EGW4D3KgXN0SIOw0r4dYrKFFWYPYFNg+LAoWQw8Sc5 OxKlP/yl3FwqZWDGvjXjch4LLeCOC49Tha9x3yAk= From: Konstantin Taranov To: kotaranov@microsoft.com, pabeni@redhat.com, haiyangz@microsoft.com, kys@microsoft.com, edumazet@google.com, kuba@kernel.org, davem@davemloft.net, decui@microsoft.com, wei.liu@kernel.org, longli@microsoft.com, jgg@ziepe.ca, leon@kernel.org Cc: linux-rdma@vger.kernel.org, linux-kernel@vger.kernel.org, netdev@vger.kernel.org Subject: [PATCH rdma-next v3 1/4] net: mana: Probe rdma device in mana driver Date: Tue, 6 May 2025 11:52:18 -0700 Message-Id: <1746557541-3617-2-git-send-email-kotaranov@linux.microsoft.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1746557541-3617-1-git-send-email-kotaranov@linux.microsoft.com> References: <1746557541-3617-1-git-send-email-kotaranov@linux.microsoft.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" From: Konstantin Taranov Initialize gdma device for rdma inside mana module. For each gdma device, initialize an auxiliary ib device. Signed-off-by: Konstantin Taranov Acked-by: Jakub Kicinski --- drivers/net/ethernet/microsoft/mana/gdma_main.c | 16 ++++++++-- drivers/net/ethernet/microsoft/mana/mana_en.c | 39 +++++++++++++++++++++= ++-- include/net/mana/mana.h | 3 ++ 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/= ethernet/microsoft/mana/gdma_main.c index b5156d4..1caf73c 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -978,7 +978,6 @@ int mana_gd_register_device(struct gdma_dev *gd) =20 return 0; } -EXPORT_SYMBOL_NS(mana_gd_register_device, "NET_MANA"); =20 int mana_gd_deregister_device(struct gdma_dev *gd) { @@ -1009,7 +1008,6 @@ int mana_gd_deregister_device(struct gdma_dev *gd) =20 return err; } -EXPORT_SYMBOL_NS(mana_gd_deregister_device, "NET_MANA"); =20 u32 mana_gd_wq_avail_space(struct gdma_queue *wq) { @@ -1541,8 +1539,15 @@ static int mana_gd_probe(struct pci_dev *pdev, const= struct pci_device_id *ent) if (err) goto cleanup_gd; =20 + + err =3D mana_rdma_probe(&gc->mana_ib); + if (err) + goto cleanup_mana; + return 0; =20 +cleanup_mana: + mana_remove(&gc->mana, false); cleanup_gd: mana_gd_cleanup(pdev); unmap_bar: @@ -1569,6 +1574,7 @@ static void mana_gd_remove(struct pci_dev *pdev) { struct gdma_context *gc =3D pci_get_drvdata(pdev); =20 + mana_rdma_remove(&gc->mana_ib); mana_remove(&gc->mana, false); =20 mana_gd_cleanup(pdev); @@ -1588,6 +1594,7 @@ static int mana_gd_suspend(struct pci_dev *pdev, pm_m= essage_t state) { struct gdma_context *gc =3D pci_get_drvdata(pdev); =20 + mana_rdma_remove(&gc->mana_ib); mana_remove(&gc->mana, true); =20 mana_gd_cleanup(pdev); @@ -1612,6 +1619,10 @@ static int mana_gd_resume(struct pci_dev *pdev) if (err) return err; =20 + err =3D mana_rdma_probe(&gc->mana_ib); + if (err) + return err; + return 0; } =20 @@ -1622,6 +1633,7 @@ static void mana_gd_shutdown(struct pci_dev *pdev) =20 dev_info(&pdev->dev, "Shutdown was called\n"); =20 + mana_rdma_remove(&gc->mana_ib); mana_remove(&gc->mana, true); =20 mana_gd_cleanup(pdev); diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/et= hernet/microsoft/mana/mana_en.c index 4e870b1..e08b43f 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -2936,7 +2936,7 @@ static void remove_adev(struct gdma_dev *gd) gd->adev =3D NULL; } =20 -static int add_adev(struct gdma_dev *gd) +static int add_adev(struct gdma_dev *gd, const char *name) { struct auxiliary_device *adev; struct mana_adev *madev; @@ -2952,7 +2952,7 @@ static int add_adev(struct gdma_dev *gd) goto idx_fail; adev->id =3D ret; =20 - adev->name =3D "rdma"; + adev->name =3D name; adev->dev.parent =3D gd->gdma_context->dev; adev->dev.release =3D adev_release; madev->mdev =3D gd; @@ -3064,7 +3064,7 @@ int mana_probe(struct gdma_dev *gd, bool resuming) } } =20 - err =3D add_adev(gd); + err =3D add_adev(gd, "eth"); out: if (err) mana_remove(gd, false); @@ -3131,6 +3131,39 @@ out: kfree(ac); } =20 +int mana_rdma_probe(struct gdma_dev *gd) +{ + int err =3D 0; + + if (gd->dev_id.type !=3D GDMA_DEVICE_MANA_IB) { + /* RDMA device is not detected on pci */ + return err; + } + + err =3D mana_gd_register_device(gd); + if (err) + return err; + + err =3D add_adev(gd, "rdma"); + if (err) + mana_gd_deregister_device(gd); + + return err; +} + +void mana_rdma_remove(struct gdma_dev *gd) +{ + if (gd->dev_id.type !=3D GDMA_DEVICE_MANA_IB) { + /* RDMA device is not detected on pci */ + return; + } + + if (gd->adev) + remove_adev(gd); + + mana_gd_deregister_device(gd); +} + struct net_device *mana_get_primary_netdev(struct mana_context *ac, u32 port_index, netdevice_tracker *tracker) diff --git a/include/net/mana/mana.h b/include/net/mana/mana.h index 0f78065..5857efc 100644 --- a/include/net/mana/mana.h +++ b/include/net/mana/mana.h @@ -488,6 +488,9 @@ int mana_detach(struct net_device *ndev, bool from_clos= e); int mana_probe(struct gdma_dev *gd, bool resuming); void mana_remove(struct gdma_dev *gd, bool suspending); =20 +int mana_rdma_probe(struct gdma_dev *gd); +void mana_rdma_remove(struct gdma_dev *gd); + void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev); int mana_xdp_xmit(struct net_device *ndev, int n, struct xdp_frame **frame= s, u32 flags); --=20 1.8.3.1 From nobody Sat Feb 7 22:06:33 2026 Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 26927280038; Tue, 6 May 2025 18:52:22 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=13.77.154.182 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746557545; cv=none; b=Rkb0pRPfQZk65G3Lw+wiK2HB2fU1eG0YqegA8MXkQfdUD5ibfDMRN3+fLn1PJhqM0F+0oHvxCg5CsIrQjfnvTXl07VU37r3AgYwk1Ob622ysjzZsL2sPldazXBQcdhV7xZlt2rs+xVr3OMizbMcLbMFdq1Rk0G3/x+qTmdEDsl0= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746557545; c=relaxed/simple; bh=8av+odCsxcGu5/U6tUJo7W61y54EDAWCzVziSuwmYfA=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References; b=TP91wGtaDHO8BmyhhE6K08QRURhcsSt91CrkBNsFazk02Y2COn9V30KKW+N9t+OKjwJ1hDwlLT9NCz7Yxkd4RjtaaLfZpuXDCYOez8CfKQt4fFMG9PDDlRA3qAX7CkbpPdcnidNcTqqhygq6zOIQTPMVaOVRLYZM7mPFkD+zCPA= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com; spf=pass smtp.mailfrom=linux.microsoft.com; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b=N3zfVKRa; arc=none smtp.client-ip=13.77.154.182 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b="N3zfVKRa" Received: by linux.microsoft.com (Postfix, from userid 1186) id C3EAE2115DD6; Tue, 6 May 2025 11:52:21 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com C3EAE2115DD6 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1746557541; bh=32IyxjG1RE0ymV8g1q5/Nay/X3lYqQWjWyJLElBlLIo=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=N3zfVKRaB14hKrdguIsPza7xRGGXnVxrQ3D777edQg5DyA0EcMVoiCZ10bKLHHW2/ WSuDHX718j0MbEM9s8RTz3yD+iOWZO1KzBOf09/4ZXT70tYKqw4OvAwojvPN6BoaSD JiaiJIXJ1Qi3UYrIE5tjldoDHho80bzvlafmEVio= From: Konstantin Taranov To: kotaranov@microsoft.com, pabeni@redhat.com, haiyangz@microsoft.com, kys@microsoft.com, edumazet@google.com, kuba@kernel.org, davem@davemloft.net, decui@microsoft.com, wei.liu@kernel.org, longli@microsoft.com, jgg@ziepe.ca, leon@kernel.org Cc: linux-rdma@vger.kernel.org, linux-kernel@vger.kernel.org, netdev@vger.kernel.org Subject: [PATCH rdma-next v3 2/4] RDMA/mana_ib: Add support of mana_ib for RNIC and ETH nic Date: Tue, 6 May 2025 11:52:19 -0700 Message-Id: <1746557541-3617-3-git-send-email-kotaranov@linux.microsoft.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1746557541-3617-1-git-send-email-kotaranov@linux.microsoft.com> References: <1746557541-3617-1-git-send-email-kotaranov@linux.microsoft.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" From: Konstantin Taranov Allow mana_ib to be created over ethernet gdma device and over rnic gdma device. The HW has two devices with different capabilities and different use-cases. Initialize required resources depending on the used gdma device. Signed-off-by: Konstantin Taranov --- drivers/infiniband/hw/mana/device.c | 174 +++++++++++++++++--------------= ---- drivers/infiniband/hw/mana/main.c | 55 +++++++++-- drivers/infiniband/hw/mana/mana_ib.h | 6 ++ 3 files changed, 138 insertions(+), 97 deletions(-) diff --git a/drivers/infiniband/hw/mana/device.c b/drivers/infiniband/hw/ma= na/device.c index b310893..165c0a1 100644 --- a/drivers/infiniband/hw/mana/device.c +++ b/drivers/infiniband/hw/mana/device.c @@ -101,103 +101,95 @@ static int mana_ib_probe(struct auxiliary_device *ad= ev, const struct auxiliary_device_id *id) { struct mana_adev *madev =3D container_of(adev, struct mana_adev, adev); + struct gdma_context *gc =3D madev->mdev->gdma_context; + struct mana_context *mc =3D gc->mana.driver_data; struct gdma_dev *mdev =3D madev->mdev; struct net_device *ndev; - struct mana_context *mc; struct mana_ib_dev *dev; u8 mac_addr[ETH_ALEN]; int ret; =20 - mc =3D mdev->driver_data; - dev =3D ib_alloc_device(mana_ib_dev, ib_dev); if (!dev) return -ENOMEM; =20 ib_set_device_ops(&dev->ib_dev, &mana_ib_dev_ops); - - dev->ib_dev.phys_port_cnt =3D mc->num_ports; - - ibdev_dbg(&dev->ib_dev, "mdev=3D%p id=3D%d num_ports=3D%d\n", mdev, - mdev->dev_id.as_uint32, dev->ib_dev.phys_port_cnt); - dev->ib_dev.node_type =3D RDMA_NODE_IB_CA; - - /* - * num_comp_vectors needs to set to the max MSIX index - * when interrupts and event queues are implemented - */ - dev->ib_dev.num_comp_vectors =3D mdev->gdma_context->max_num_queues; - dev->ib_dev.dev.parent =3D mdev->gdma_context->dev; - - ndev =3D mana_get_primary_netdev(mc, 0, &dev->dev_tracker); - if (!ndev) { - ret =3D -ENODEV; - ibdev_err(&dev->ib_dev, "Failed to get netdev for IB port 1"); - goto free_ib_device; - } - ether_addr_copy(mac_addr, ndev->dev_addr); - addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, ndev->dev_addr); - ret =3D ib_device_set_netdev(&dev->ib_dev, ndev, 1); - /* mana_get_primary_netdev() returns ndev with refcount held */ - netdev_put(ndev, &dev->dev_tracker); - if (ret) { - ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret); - goto free_ib_device; - } - - ret =3D mana_gd_register_device(&mdev->gdma_context->mana_ib); - if (ret) { - ibdev_err(&dev->ib_dev, "Failed to register device, ret %d", - ret); - goto free_ib_device; - } - dev->gdma_dev =3D &mdev->gdma_context->mana_ib; - - dev->nb.notifier_call =3D mana_ib_netdev_event; - ret =3D register_netdevice_notifier(&dev->nb); - if (ret) { - ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", - ret); - goto deregister_device; - } - - ret =3D mana_ib_gd_query_adapter_caps(dev); - if (ret) { - ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", - ret); - goto deregister_net_notifier; - } - - ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops); - - ret =3D mana_ib_create_eqs(dev); - if (ret) { - ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret); - goto deregister_net_notifier; - } - - ret =3D mana_ib_gd_create_rnic_adapter(dev); - if (ret) - goto destroy_eqs; - + dev->ib_dev.num_comp_vectors =3D gc->max_num_queues; + dev->ib_dev.dev.parent =3D gc->dev; + dev->gdma_dev =3D mdev; xa_init_flags(&dev->qp_table_wq, XA_FLAGS_LOCK_IRQ); - ret =3D mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr); - if (ret) { - ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", - ret); - goto destroy_rnic; + + if (mana_ib_is_rnic(dev)) { + dev->ib_dev.phys_port_cnt =3D 1; + ndev =3D mana_get_primary_netdev(mc, 0, &dev->dev_tracker); + if (!ndev) { + ret =3D -ENODEV; + ibdev_err(&dev->ib_dev, "Failed to get netdev for IB port 1"); + goto free_ib_device; + } + ether_addr_copy(mac_addr, ndev->dev_addr); + addrconf_addr_eui48((u8 *)&dev->ib_dev.node_guid, ndev->dev_addr); + ret =3D ib_device_set_netdev(&dev->ib_dev, ndev, 1); + /* mana_get_primary_netdev() returns ndev with refcount held */ + netdev_put(ndev, &dev->dev_tracker); + if (ret) { + ibdev_err(&dev->ib_dev, "Failed to set ib netdev, ret %d", ret); + goto free_ib_device; + } + + dev->nb.notifier_call =3D mana_ib_netdev_event; + ret =3D register_netdevice_notifier(&dev->nb); + if (ret) { + ibdev_err(&dev->ib_dev, "Failed to register net notifier, %d", + ret); + goto free_ib_device; + } + + ret =3D mana_ib_gd_query_adapter_caps(dev); + if (ret) { + ibdev_err(&dev->ib_dev, "Failed to query device caps, ret %d", ret); + goto deregister_net_notifier; + } + + ib_set_device_ops(&dev->ib_dev, &mana_ib_stats_ops); + + ret =3D mana_ib_create_eqs(dev); + if (ret) { + ibdev_err(&dev->ib_dev, "Failed to create EQs, ret %d", ret); + goto deregister_net_notifier; + } + + ret =3D mana_ib_gd_create_rnic_adapter(dev); + if (ret) + goto destroy_eqs; + + ret =3D mana_ib_gd_config_mac(dev, ADDR_OP_ADD, mac_addr); + if (ret) { + ibdev_err(&dev->ib_dev, "Failed to add Mac address, ret %d", ret); + goto destroy_rnic; + } + } else { + dev->ib_dev.phys_port_cnt =3D mc->num_ports; + ret =3D mana_eth_query_adapter_caps(dev); + if (ret) { + ibdev_err(&dev->ib_dev, "Failed to query ETH device caps, ret %d", ret); + goto free_ib_device; + } } =20 - dev->av_pool =3D dma_pool_create("mana_ib_av", mdev->gdma_context->dev, - MANA_AV_BUFFER_SIZE, MANA_AV_BUFFER_SIZE, 0); + dev->av_pool =3D dma_pool_create("mana_ib_av", gc->dev, MANA_AV_BUFFER_SI= ZE, + MANA_AV_BUFFER_SIZE, 0); if (!dev->av_pool) { ret =3D -ENOMEM; goto destroy_rnic; } =20 - ret =3D ib_register_device(&dev->ib_dev, "mana_%d", - mdev->gdma_context->dev); + ibdev_dbg(&dev->ib_dev, "mdev=3D%p id=3D%d num_ports=3D%d\n", mdev, + mdev->dev_id.as_uint32, dev->ib_dev.phys_port_cnt); + + ret =3D ib_register_device(&dev->ib_dev, mana_ib_is_rnic(dev) ? "mana_%d"= : "manae_%d", + gc->dev); if (ret) goto deallocate_pool; =20 @@ -208,15 +200,16 @@ static int mana_ib_probe(struct auxiliary_device *ade= v, deallocate_pool: dma_pool_destroy(dev->av_pool); destroy_rnic: - xa_destroy(&dev->qp_table_wq); - mana_ib_gd_destroy_rnic_adapter(dev); + if (mana_ib_is_rnic(dev)) + mana_ib_gd_destroy_rnic_adapter(dev); destroy_eqs: - mana_ib_destroy_eqs(dev); + if (mana_ib_is_rnic(dev)) + mana_ib_destroy_eqs(dev); deregister_net_notifier: - unregister_netdevice_notifier(&dev->nb); -deregister_device: - mana_gd_deregister_device(dev->gdma_dev); + if (mana_ib_is_rnic(dev)) + unregister_netdevice_notifier(&dev->nb); free_ib_device: + xa_destroy(&dev->qp_table_wq); ib_dealloc_device(&dev->ib_dev); return ret; } @@ -227,25 +220,24 @@ static void mana_ib_remove(struct auxiliary_device *a= dev) =20 ib_unregister_device(&dev->ib_dev); dma_pool_destroy(dev->av_pool); + if (mana_ib_is_rnic(dev)) { + mana_ib_gd_destroy_rnic_adapter(dev); + mana_ib_destroy_eqs(dev); + unregister_netdevice_notifier(&dev->nb); + } xa_destroy(&dev->qp_table_wq); - mana_ib_gd_destroy_rnic_adapter(dev); - mana_ib_destroy_eqs(dev); - unregister_netdevice_notifier(&dev->nb); - mana_gd_deregister_device(dev->gdma_dev); ib_dealloc_device(&dev->ib_dev); } =20 static const struct auxiliary_device_id mana_id_table[] =3D { - { - .name =3D "mana.rdma", - }, + { .name =3D "mana.rdma", }, + { .name =3D "mana.eth", }, {}, }; =20 MODULE_DEVICE_TABLE(auxiliary, mana_id_table); =20 static struct auxiliary_driver mana_driver =3D { - .name =3D "rdma", .probe =3D mana_ib_probe, .remove =3D mana_ib_remove, .id_table =3D mana_id_table, diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana= /main.c index a28b712..dd8bfb8 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -4,6 +4,7 @@ */ =20 #include "mana_ib.h" +#include "linux/pci.h" =20 void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd, u32 port) @@ -551,6 +552,7 @@ int mana_ib_mmap(struct ib_ucontext *ibcontext, struct = vm_area_struct *vma) int mana_ib_get_port_immutable(struct ib_device *ibdev, u32 port_num, struct ib_port_immutable *immutable) { + struct mana_ib_dev *dev =3D container_of(ibdev, struct mana_ib_dev, ib_de= v); struct ib_port_attr attr; int err; =20 @@ -560,10 +562,12 @@ int mana_ib_get_port_immutable(struct ib_device *ibde= v, u32 port_num, =20 immutable->pkey_tbl_len =3D attr.pkey_tbl_len; immutable->gid_tbl_len =3D attr.gid_tbl_len; - immutable->core_cap_flags =3D RDMA_CORE_PORT_RAW_PACKET; - if (port_num =3D=3D 1) { - immutable->core_cap_flags |=3D RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; + + if (mana_ib_is_rnic(dev)) { + immutable->core_cap_flags =3D RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP; immutable->max_mad_size =3D IB_MGMT_MAD_SIZE; + } else { + immutable->core_cap_flags =3D RDMA_CORE_PORT_RAW_PACKET; } =20 return 0; @@ -572,10 +576,12 @@ int mana_ib_get_port_immutable(struct ib_device *ibde= v, u32 port_num, int mana_ib_query_device(struct ib_device *ibdev, struct ib_device_attr *p= rops, struct ib_udata *uhw) { - struct mana_ib_dev *dev =3D container_of(ibdev, - struct mana_ib_dev, ib_dev); + struct mana_ib_dev *dev =3D container_of(ibdev, struct mana_ib_dev, ib_de= v); + struct pci_dev *pdev =3D to_pci_dev(mdev_to_gc(dev)->dev); =20 memset(props, 0, sizeof(*props)); + props->vendor_id =3D pdev->vendor; + props->vendor_part_id =3D dev->gdma_dev->dev_id.type; props->max_mr_size =3D MANA_IB_MAX_MR_SIZE; props->page_size_cap =3D dev->adapter_caps.page_size_cap; props->max_qp =3D dev->adapter_caps.max_qp_count; @@ -596,6 +602,8 @@ int mana_ib_query_device(struct ib_device *ibdev, struc= t ib_device_attr *props, props->max_ah =3D INT_MAX; props->max_pkeys =3D 1; props->local_ca_ack_delay =3D MANA_CA_ACK_DELAY; + if (!mana_ib_is_rnic(dev)) + props->raw_packet_caps =3D IB_RAW_PACKET_CAP_IP_CSUM; =20 return 0; } @@ -603,6 +611,7 @@ int mana_ib_query_device(struct ib_device *ibdev, struc= t ib_device_attr *props, int mana_ib_query_port(struct ib_device *ibdev, u32 port, struct ib_port_attr *props) { + struct mana_ib_dev *dev =3D container_of(ibdev, struct mana_ib_dev, ib_de= v); struct net_device *ndev =3D mana_ib_get_netdev(ibdev, port); =20 if (!ndev) @@ -623,7 +632,7 @@ int mana_ib_query_port(struct ib_device *ibdev, u32 por= t, props->active_width =3D IB_WIDTH_4X; props->active_speed =3D IB_SPEED_EDR; props->pkey_tbl_len =3D 1; - if (port =3D=3D 1) { + if (mana_ib_is_rnic(dev)) { props->gid_tbl_len =3D 16; props->port_cap_flags =3D IB_PORT_CM_SUP; props->ip_gids =3D true; @@ -703,6 +712,37 @@ int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *= dev) return 0; } =20 +int mana_eth_query_adapter_caps(struct mana_ib_dev *dev) +{ + struct mana_ib_adapter_caps *caps =3D &dev->adapter_caps; + struct gdma_query_max_resources_resp resp =3D {}; + struct gdma_general_req req =3D {}; + int err; + + mana_gd_init_req_hdr(&req.hdr, GDMA_QUERY_MAX_RESOURCES, + sizeof(req), sizeof(resp)); + + err =3D mana_gd_send_request(mdev_to_gc(dev), sizeof(req), &req, sizeof(r= esp), &resp); + if (err) { + ibdev_err(&dev->ib_dev, + "Failed to query adapter caps err %d", err); + return err; + } + + caps->max_qp_count =3D min_t(u32, resp.max_sq, resp.max_rq); + caps->max_cq_count =3D resp.max_cq; + caps->max_mr_count =3D resp.max_mst; + caps->max_pd_count =3D 0x6000; + caps->max_qp_wr =3D min_t(u32, + 0x100000 / GDMA_MAX_SQE_SIZE, + 0x100000 / GDMA_MAX_RQE_SIZE); + caps->max_send_sge_count =3D 30; + caps->max_recv_sge_count =3D 15; + caps->page_size_cap =3D PAGE_SZ_BM; + + return 0; +} + static void mana_ib_event_handler(void *ctx, struct gdma_queue *q, struct gdma_event *= event) { @@ -921,6 +961,9 @@ int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, stru= ct mana_ib_cq *cq, u32 do struct mana_rnic_create_cq_req req =3D {}; int err; =20 + if (!mdev->eqs) + return -EINVAL; + mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_CQ, sizeof(req), sizeof(res= p)); req.hdr.dev_id =3D gc->mana_ib.dev_id; req.adapter =3D mdev->adapter_handle; diff --git a/drivers/infiniband/hw/mana/mana_ib.h b/drivers/infiniband/hw/m= ana/mana_ib.h index f0dbd90..42bebd6 100644 --- a/drivers/infiniband/hw/mana/mana_ib.h +++ b/drivers/infiniband/hw/mana/mana_ib.h @@ -544,6 +544,11 @@ static inline void mana_put_qp_ref(struct mana_ib_qp *= qp) complete(&qp->free); } =20 +static inline bool mana_ib_is_rnic(struct mana_ib_dev *mdev) +{ + return mdev->gdma_dev->dev_id.type =3D=3D GDMA_DEVICE_MANA_IB; +} + static inline struct net_device *mana_ib_get_netdev(struct ib_device *ibde= v, u32 port) { struct mana_ib_dev *mdev =3D container_of(ibdev, struct mana_ib_dev, ib_d= ev); @@ -643,6 +648,7 @@ int mana_ib_query_gid(struct ib_device *ibdev, u32 port= , int index, void mana_ib_disassociate_ucontext(struct ib_ucontext *ibcontext); =20 int mana_ib_gd_query_adapter_caps(struct mana_ib_dev *mdev); +int mana_eth_query_adapter_caps(struct mana_ib_dev *mdev); =20 int mana_ib_create_eqs(struct mana_ib_dev *mdev); =20 --=20 1.8.3.1 From nobody Sat Feb 7 22:06:33 2026 Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 26A18286413; Tue, 6 May 2025 18:52:22 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=13.77.154.182 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746557544; cv=none; b=J+pdYKhe7K1pRuxWXqhjlUE712fxIRvk8om8lCA8kEmyOKq7mao27rkiLWFrKHOrJOkNXaajH1B4OjOWXzEtwg2wku/Qxhqxv7aCClLBLWDG5MIlmaPMwwSg+O5nkaXMILaFMU9upjr2FJFSEjkXzg0crDXlrauZi0rqtP9Xhkc= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746557544; c=relaxed/simple; bh=E9zyVeuLSd/Q3m8or8d/zlcpfY40P06hv95j4xTg6Fg=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References; b=LOe8FDGBw37hou7fVlKeqpexF+fRvpGgLfxBveihtaKh50unsJNjGY8sOzmA2GtiJcdG6slOmLMmuxvl/zTiHYK2jHwpXFBQjv02ENe37AT1RmvaSrpvQNSOHbf511RUEzUPgHOV761Gb4k2UKJOy/SU5uRqaawykVn6XhKN6E4= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com; spf=pass smtp.mailfrom=linux.microsoft.com; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b=n4k6L3I2; arc=none smtp.client-ip=13.77.154.182 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b="n4k6L3I2" Received: by linux.microsoft.com (Postfix, from userid 1186) id D16962115DD3; Tue, 6 May 2025 11:52:21 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com D16962115DD3 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1746557541; bh=2FDkAduyf1th1FuQPp51hcm7T+5yFvKHn4bWMU4AQ40=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=n4k6L3I2CmHdDfjwhMbO2xcUO4qShRy285EEdPGTjg2ozHjdFCmY+phYQwO3p6Sza b/xQqXNyAITvPf18P1aVfPQ7wQy2F6wRUM/kEyrZINbzWiKZG6jnkYx709afOYQED4 tnq2BfTBjHERRuiIsVYwblaSe7Y/30X9xnjLiCmM= From: Konstantin Taranov To: kotaranov@microsoft.com, pabeni@redhat.com, haiyangz@microsoft.com, kys@microsoft.com, edumazet@google.com, kuba@kernel.org, davem@davemloft.net, decui@microsoft.com, wei.liu@kernel.org, longli@microsoft.com, jgg@ziepe.ca, leon@kernel.org Cc: linux-rdma@vger.kernel.org, linux-kernel@vger.kernel.org, netdev@vger.kernel.org Subject: [PATCH rdma-next v3 3/4] RDMA/mana_ib: unify mana_ib functions to support any gdma device Date: Tue, 6 May 2025 11:52:20 -0700 Message-Id: <1746557541-3617-4-git-send-email-kotaranov@linux.microsoft.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1746557541-3617-1-git-send-email-kotaranov@linux.microsoft.com> References: <1746557541-3617-1-git-send-email-kotaranov@linux.microsoft.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" From: Konstantin Taranov Use the installed gdma_device instead of hard-coded device in requests to the HW. Signed-off-by: Konstantin Taranov --- drivers/infiniband/hw/mana/cq.c | 4 +--- drivers/infiniband/hw/mana/main.c | 27 +++++++++++++-------------- drivers/infiniband/hw/mana/qp.c | 5 ++--- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/drivers/infiniband/hw/mana/cq.c b/drivers/infiniband/hw/mana/c= q.c index 0fc4e26..28e154b 100644 --- a/drivers/infiniband/hw/mana/cq.c +++ b/drivers/infiniband/hw/mana/cq.c @@ -15,14 +15,12 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct = ib_cq_init_attr *attr, struct ib_device *ibdev =3D ibcq->device; struct mana_ib_create_cq ucmd =3D {}; struct mana_ib_dev *mdev; - struct gdma_context *gc; bool is_rnic_cq; u32 doorbell; u32 buf_size; int err; =20 mdev =3D container_of(ibdev, struct mana_ib_dev, ib_dev); - gc =3D mdev_to_gc(mdev); =20 cq->comp_vector =3D attr->comp_vector % ibdev->num_comp_vectors; cq->cq_handle =3D INVALID_MANA_HANDLE; @@ -65,7 +63,7 @@ int mana_ib_create_cq(struct ib_cq *ibcq, const struct ib= _cq_init_attr *attr, ibdev_dbg(ibdev, "Failed to create kernel queue for create cq, %d\n", e= rr); return err; } - doorbell =3D gc->mana_ib.doorbell; + doorbell =3D mdev->gdma_dev->doorbell; } =20 if (is_rnic_cq) { diff --git a/drivers/infiniband/hw/mana/main.c b/drivers/infiniband/hw/mana= /main.c index dd8bfb8..db60979 100644 --- a/drivers/infiniband/hw/mana/main.c +++ b/drivers/infiniband/hw/mana/main.c @@ -244,7 +244,6 @@ void mana_ib_dealloc_ucontext(struct ib_ucontext *ibcon= text) int mana_ib_create_kernel_queue(struct mana_ib_dev *mdev, u32 size, enum g= dma_queue_type type, struct mana_ib_queue *queue) { - struct gdma_context *gc =3D mdev_to_gc(mdev); struct gdma_queue_spec spec =3D {}; int err; =20 @@ -253,7 +252,7 @@ int mana_ib_create_kernel_queue(struct mana_ib_dev *mde= v, u32 size, enum gdma_qu spec.type =3D type; spec.monitor_avl_buf =3D false; spec.queue_size =3D size; - err =3D mana_gd_create_mana_wq_cq(&gc->mana_ib, &spec, &queue->kmem); + err =3D mana_gd_create_mana_wq_cq(mdev->gdma_dev, &spec, &queue->kmem); if (err) return err; /* take ownership into mana_ib from mana */ @@ -784,7 +783,7 @@ int mana_ib_create_eqs(struct mana_ib_dev *mdev) spec.eq.log2_throttle_limit =3D LOG2_EQ_THROTTLE; spec.eq.msix_index =3D 0; =20 - err =3D mana_gd_create_mana_eq(&gc->mana_ib, &spec, &mdev->fatal_err_eq); + err =3D mana_gd_create_mana_eq(mdev->gdma_dev, &spec, &mdev->fatal_err_eq= ); if (err) return err; =20 @@ -835,7 +834,7 @@ int mana_ib_gd_create_rnic_adapter(struct mana_ib_dev *= mdev) =20 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_ADAPTER, sizeof(req), sizeo= f(resp)); req.hdr.req.msg_version =3D GDMA_MESSAGE_V2; - req.hdr.dev_id =3D gc->mana_ib.dev_id; + req.hdr.dev_id =3D mdev->gdma_dev->dev_id; req.notify_eq_id =3D mdev->fatal_err_eq->id; =20 if (mdev->adapter_caps.feature_flags & MANA_IB_FEATURE_CLIENT_ERROR_CQE_S= UPPORT) @@ -860,7 +859,7 @@ int mana_ib_gd_destroy_rnic_adapter(struct mana_ib_dev = *mdev) =20 gc =3D mdev_to_gc(mdev); mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_ADAPTER, sizeof(req), size= of(resp)); - req.hdr.dev_id =3D gc->mana_ib.dev_id; + req.hdr.dev_id =3D mdev->gdma_dev->dev_id; req.adapter =3D mdev->adapter_handle; =20 err =3D mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); @@ -887,7 +886,7 @@ int mana_ib_gd_add_gid(const struct ib_gid_attr *attr, = void **context) } =20 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_IP_ADDR, sizeof(req), sizeo= f(resp)); - req.hdr.dev_id =3D gc->mana_ib.dev_id; + req.hdr.dev_id =3D mdev->gdma_dev->dev_id; req.adapter =3D mdev->adapter_handle; req.op =3D ADDR_OP_ADD; req.sgid_type =3D (ntype =3D=3D RDMA_NETWORK_IPV6) ? SGID_TYPE_IPV6 : SGI= D_TYPE_IPV4; @@ -917,7 +916,7 @@ int mana_ib_gd_del_gid(const struct ib_gid_attr *attr, = void **context) } =20 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_IP_ADDR, sizeof(req), sizeo= f(resp)); - req.hdr.dev_id =3D gc->mana_ib.dev_id; + req.hdr.dev_id =3D mdev->gdma_dev->dev_id; req.adapter =3D mdev->adapter_handle; req.op =3D ADDR_OP_REMOVE; req.sgid_type =3D (ntype =3D=3D RDMA_NETWORK_IPV6) ? SGID_TYPE_IPV6 : SGI= D_TYPE_IPV4; @@ -940,7 +939,7 @@ int mana_ib_gd_config_mac(struct mana_ib_dev *mdev, enu= m mana_ib_addr_op op, u8 int err; =20 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CONFIG_MAC_ADDR, sizeof(req), size= of(resp)); - req.hdr.dev_id =3D gc->mana_ib.dev_id; + req.hdr.dev_id =3D mdev->gdma_dev->dev_id; req.adapter =3D mdev->adapter_handle; req.op =3D op; copy_in_reverse(req.mac_addr, mac, ETH_ALEN); @@ -965,7 +964,7 @@ int mana_ib_gd_create_cq(struct mana_ib_dev *mdev, stru= ct mana_ib_cq *cq, u32 do return -EINVAL; =20 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_CQ, sizeof(req), sizeof(res= p)); - req.hdr.dev_id =3D gc->mana_ib.dev_id; + req.hdr.dev_id =3D mdev->gdma_dev->dev_id; req.adapter =3D mdev->adapter_handle; req.gdma_region =3D cq->queue.gdma_region; req.eq_id =3D mdev->eqs[cq->comp_vector]->id; @@ -997,7 +996,7 @@ int mana_ib_gd_destroy_cq(struct mana_ib_dev *mdev, str= uct mana_ib_cq *cq) return 0; =20 mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_CQ, sizeof(req), sizeof(re= sp)); - req.hdr.dev_id =3D gc->mana_ib.dev_id; + req.hdr.dev_id =3D mdev->gdma_dev->dev_id; req.adapter =3D mdev->adapter_handle; req.cq_handle =3D cq->cq_handle; =20 @@ -1023,7 +1022,7 @@ int mana_ib_gd_create_rc_qp(struct mana_ib_dev *mdev,= struct mana_ib_qp *qp, int err, i; =20 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_RC_QP, sizeof(req), sizeof(= resp)); - req.hdr.dev_id =3D gc->mana_ib.dev_id; + req.hdr.dev_id =3D mdev->gdma_dev->dev_id; req.adapter =3D mdev->adapter_handle; req.pd_handle =3D pd->pd_handle; req.send_cq_handle =3D send_cq->cq_handle; @@ -1059,7 +1058,7 @@ int mana_ib_gd_destroy_rc_qp(struct mana_ib_dev *mdev= , struct mana_ib_qp *qp) int err; =20 mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_RC_QP, sizeof(req), sizeof= (resp)); - req.hdr.dev_id =3D gc->mana_ib.dev_id; + req.hdr.dev_id =3D mdev->gdma_dev->dev_id; req.adapter =3D mdev->adapter_handle; req.rc_qp_handle =3D qp->qp_handle; err =3D mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); @@ -1082,7 +1081,7 @@ int mana_ib_gd_create_ud_qp(struct mana_ib_dev *mdev,= struct mana_ib_qp *qp, int err, i; =20 mana_gd_init_req_hdr(&req.hdr, MANA_IB_CREATE_UD_QP, sizeof(req), sizeof(= resp)); - req.hdr.dev_id =3D gc->mana_ib.dev_id; + req.hdr.dev_id =3D mdev->gdma_dev->dev_id; req.adapter =3D mdev->adapter_handle; req.pd_handle =3D pd->pd_handle; req.send_cq_handle =3D send_cq->cq_handle; @@ -1117,7 +1116,7 @@ int mana_ib_gd_destroy_ud_qp(struct mana_ib_dev *mdev= , struct mana_ib_qp *qp) int err; =20 mana_gd_init_req_hdr(&req.hdr, MANA_IB_DESTROY_UD_QP, sizeof(req), sizeof= (resp)); - req.hdr.dev_id =3D gc->mana_ib.dev_id; + req.hdr.dev_id =3D mdev->gdma_dev->dev_id; req.adapter =3D mdev->adapter_handle; req.qp_handle =3D qp->qp_handle; err =3D mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); diff --git a/drivers/infiniband/hw/mana/qp.c b/drivers/infiniband/hw/mana/q= p.c index c928af5..14fd7d6 100644 --- a/drivers/infiniband/hw/mana/qp.c +++ b/drivers/infiniband/hw/mana/qp.c @@ -635,7 +635,6 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, str= uct ib_pd *ibpd, { struct mana_ib_dev *mdev =3D container_of(ibpd->device, struct mana_ib_de= v, ib_dev); struct mana_ib_qp *qp =3D container_of(ibqp, struct mana_ib_qp, ibqp); - struct gdma_context *gc =3D mdev_to_gc(mdev); u32 doorbell, queue_size; int i, err; =20 @@ -654,7 +653,7 @@ static int mana_ib_create_ud_qp(struct ib_qp *ibqp, str= uct ib_pd *ibpd, goto destroy_queues; } } - doorbell =3D gc->mana_ib.doorbell; + doorbell =3D mdev->gdma_dev->doorbell; =20 err =3D create_shadow_queue(&qp->shadow_rq, attr->cap.max_recv_wr, sizeof(struct ud_rq_shadow_wqe)); @@ -736,7 +735,7 @@ static int mana_ib_gd_modify_qp(struct ib_qp *ibqp, str= uct ib_qp_attr *attr, int err; =20 mana_gd_init_req_hdr(&req.hdr, MANA_IB_SET_QP_STATE, sizeof(req), sizeof(= resp)); - req.hdr.dev_id =3D gc->mana_ib.dev_id; + req.hdr.dev_id =3D mdev->gdma_dev->dev_id; req.adapter =3D mdev->adapter_handle; req.qp_handle =3D qp->qp_handle; req.qp_state =3D attr->qp_state; --=20 1.8.3.1 From nobody Sat Feb 7 22:06:33 2026 Received: from linux.microsoft.com (linux.microsoft.com [13.77.154.182]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 4B5DF286438; Tue, 6 May 2025 18:52:22 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=13.77.154.182 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746557543; cv=none; b=UUrP46Pgpjwnw1kur9jjK6cuS3xAoX51lM8i8tmUr6ipw7lWh3+YOUsMovytiow2RXRIdrtPueY5n95ROTEM3RYlArwIaFNuUj9vN7VMyxEtQUUHwLkzrSJicSeFjMPscwB8FlrpN9N/lceksNERhnH42uuAKDVA2Iy0RzCCEk4= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1746557543; c=relaxed/simple; bh=X9s38AFitvss5w9rj6xKtTNuhrCtPgInhOR92ySVVYI=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References; b=k/hite1YhE+ShXHxCLANZf/fhDmyllHF3Qbxqb+vM/EJFQ7XCxpFTNxAo6T011XDQzdc+ofrR7ogmDZ3iUTaoSLGKFwFKXEnAyeLefIZVv56drGJh2AkoZOqeOvSLeRTQVQ9tIIY2cVjdv7cBsie3hTZYfRoTyAsb4WiCstdyJE= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com; spf=pass smtp.mailfrom=linux.microsoft.com; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b=OZHCZ1Eu; arc=none smtp.client-ip=13.77.154.182 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=linux.microsoft.com Authentication-Results: smtp.subspace.kernel.org; dkim=pass (1024-bit key) header.d=linux.microsoft.com header.i=@linux.microsoft.com header.b="OZHCZ1Eu" Received: by linux.microsoft.com (Postfix, from userid 1186) id DE13C2115DD7; Tue, 6 May 2025 11:52:21 -0700 (PDT) DKIM-Filter: OpenDKIM Filter v2.11.0 linux.microsoft.com DE13C2115DD7 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=linux.microsoft.com; s=default; t=1746557541; bh=RiOFjcGtYKwB80UEqpnpaVJHrzdwUYUItgslE4NxIVw=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=OZHCZ1Euus0+4zmPKVLZhR+fF+RBAH54pNJPXNNCGdYJaeIvhiY5RyxUma9dk99nC L8qaCCF+Y/8brCQAaqNAxIXs+9uYU2r/w6PirwqDh7t2Zi2RRTdVoMQqlXFogZf+C1 R1T1+vK6LbOrJUzG196NrU8X5Eop60Fq4/jpPuyw= From: Konstantin Taranov To: kotaranov@microsoft.com, pabeni@redhat.com, haiyangz@microsoft.com, kys@microsoft.com, edumazet@google.com, kuba@kernel.org, davem@davemloft.net, decui@microsoft.com, wei.liu@kernel.org, longli@microsoft.com, jgg@ziepe.ca, leon@kernel.org Cc: linux-rdma@vger.kernel.org, linux-kernel@vger.kernel.org, netdev@vger.kernel.org Subject: [PATCH rdma-next v3 4/4] net: mana: Add support for auxiliary device servicing events Date: Tue, 6 May 2025 11:52:21 -0700 Message-Id: <1746557541-3617-5-git-send-email-kotaranov@linux.microsoft.com> X-Mailer: git-send-email 1.8.3.1 In-Reply-To: <1746557541-3617-1-git-send-email-kotaranov@linux.microsoft.com> References: <1746557541-3617-1-git-send-email-kotaranov@linux.microsoft.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: Content-Transfer-Encoding: quoted-printable MIME-Version: 1.0 Content-Type: text/plain; charset="utf-8" From: Shiraz Saleem Handle soc servcing events which require the rdma auxiliary device resource= s to be cleaned up during a suspend, and re-initialized during a resume. Signed-off-by: Shiraz Saleem Signed-off-by: Konstantin Taranov --- drivers/net/ethernet/microsoft/mana/gdma_main.c | 11 +++- drivers/net/ethernet/microsoft/mana/hw_channel.c | 19 +++++++ drivers/net/ethernet/microsoft/mana/mana_en.c | 69 ++++++++++++++++++++= ++++ include/net/mana/gdma.h | 19 +++++++ include/net/mana/hw_channel.h | 9 ++++ 5 files changed, 126 insertions(+), 1 deletion(-) diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/= ethernet/microsoft/mana/gdma_main.c index 1caf73c..1d98dd6 100644 --- a/drivers/net/ethernet/microsoft/mana/gdma_main.c +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -385,6 +385,7 @@ static void mana_gd_process_eqe(struct gdma_queue *eq) case GDMA_EQE_HWC_INIT_EQ_ID_DB: case GDMA_EQE_HWC_INIT_DATA: case GDMA_EQE_HWC_INIT_DONE: + case GDMA_EQE_HWC_SOC_SERVICE: case GDMA_EQE_RNIC_QP_FATAL: if (!eq->eq.callback) break; @@ -1438,9 +1439,13 @@ static int mana_gd_setup(struct pci_dev *pdev) mana_gd_init_registers(pdev); mana_smc_init(&gc->shm_channel, gc->dev, gc->shm_base); =20 + gc->service_wq =3D alloc_ordered_workqueue("gdma_service_wq", 0); + if (!gc->service_wq) + return -ENOMEM; + err =3D mana_gd_setup_irqs(pdev); if (err) - return err; + goto free_workqueue; =20 err =3D mana_hwc_create_channel(gc); if (err) @@ -1464,6 +1469,8 @@ destroy_hwc: mana_hwc_destroy_channel(gc); remove_irq: mana_gd_remove_irqs(pdev); +free_workqueue: + destroy_workqueue(gc->service_wq); return err; } =20 @@ -1474,6 +1481,8 @@ static void mana_gd_cleanup(struct pci_dev *pdev) mana_hwc_destroy_channel(gc); =20 mana_gd_remove_irqs(pdev); + + destroy_workqueue(gc->service_wq); } =20 static bool mana_is_pf(unsigned short dev_id) diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net= /ethernet/microsoft/mana/hw_channel.c index a00f915..407b46e 100644 --- a/drivers/net/ethernet/microsoft/mana/hw_channel.c +++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c @@ -112,11 +112,13 @@ out: static void mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_se= lf, struct gdma_event *event) { + union hwc_init_soc_service_type service_data; struct hw_channel_context *hwc =3D ctx; struct gdma_dev *gd =3D hwc->gdma_dev; union hwc_init_type_data type_data; union hwc_init_eq_id_db eq_db; u32 type, val; + int ret; =20 switch (event->type) { case GDMA_EQE_HWC_INIT_EQ_ID_DB: @@ -199,7 +201,24 @@ static void mana_hwc_init_event_handler(void *ctx, str= uct gdma_queue *q_self, } =20 break; + case GDMA_EQE_HWC_SOC_SERVICE: + service_data.as_uint32 =3D event->details[0]; + type =3D service_data.type; + val =3D service_data.value; =20 + switch (type) { + case GDMA_SERVICE_TYPE_RDMA_SUSPEND: + case GDMA_SERVICE_TYPE_RDMA_RESUME: + ret =3D mana_rdma_service_event(gd->gdma_context, type); + if (ret) + dev_err(hwc->dev, "Failed to schedule adev service event: %d\n", ret); + break; + default: + dev_warn(hwc->dev, "Received unknown SOC service type %u\n", type); + break; + } + + break; default: dev_warn(hwc->dev, "Received unknown gdma event %u\n", event->type); /* Ignore unknown events, which should never happen. */ diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/et= hernet/microsoft/mana/mana_en.c index e08b43f..d35b7fd 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -2982,6 +2982,70 @@ idx_fail: return ret; } =20 +static void mana_handle_rdma_servicing(struct work_struct *work) +{ + struct mana_service_work *serv_work =3D + container_of(work, struct mana_service_work, work); + struct gdma_dev *gd =3D serv_work->gdma_dev; + struct device *dev =3D gd->gdma_context->dev; + int ret; + + if (READ_ONCE(gd->rdma_teardown)) + goto out; + + switch (serv_work->event) { + case GDMA_SERVICE_TYPE_RDMA_SUSPEND: + if (!gd->adev || gd->is_suspended) + break; + + remove_adev(gd); + gd->is_suspended =3D true; + break; + + case GDMA_SERVICE_TYPE_RDMA_RESUME: + if (!gd->is_suspended) + break; + + ret =3D add_adev(gd, "rdma"); + if (ret) + dev_err(dev, "Failed to add adev on resume: %d\n", ret); + else + gd->is_suspended =3D false; + break; + + default: + dev_warn(dev, "unknown adev service event %u\n", + serv_work->event); + break; + } + +out: + kfree(serv_work); +} + +int mana_rdma_service_event(struct gdma_context *gc, enum gdma_service_typ= e event) +{ + struct gdma_dev *gd =3D &gc->mana_ib; + struct mana_service_work *serv_work; + + if (gd->dev_id.type !=3D GDMA_DEVICE_MANA_IB) { + /* RDMA device is not detected on pci */ + return 0; + } + + serv_work =3D kzalloc(sizeof(*serv_work), GFP_ATOMIC); + if (!serv_work) + return -ENOMEM; + + serv_work->event =3D event; + serv_work->gdma_dev =3D gd; + + INIT_WORK(&serv_work->work, mana_handle_rdma_servicing); + queue_work(gc->service_wq, &serv_work->work); + + return 0; +} + int mana_probe(struct gdma_dev *gd, bool resuming) { struct gdma_context *gc =3D gd->gdma_context; @@ -3153,11 +3217,16 @@ int mana_rdma_probe(struct gdma_dev *gd) =20 void mana_rdma_remove(struct gdma_dev *gd) { + struct gdma_context *gc =3D gd->gdma_context; + if (gd->dev_id.type !=3D GDMA_DEVICE_MANA_IB) { /* RDMA device is not detected on pci */ return; } =20 + WRITE_ONCE(gd->rdma_teardown, true); + flush_workqueue(gc->service_wq); + if (gd->adev) remove_adev(gd); =20 diff --git a/include/net/mana/gdma.h b/include/net/mana/gdma.h index 89abf98..6b79788 100644 --- a/include/net/mana/gdma.h +++ b/include/net/mana/gdma.h @@ -60,6 +60,7 @@ enum gdma_eqe_type { GDMA_EQE_HWC_INIT_DONE =3D 131, GDMA_EQE_HWC_SOC_RECONFIG =3D 132, GDMA_EQE_HWC_SOC_RECONFIG_DATA =3D 133, + GDMA_EQE_HWC_SOC_SERVICE =3D 134, GDMA_EQE_RNIC_QP_FATAL =3D 176, }; =20 @@ -70,6 +71,18 @@ enum { GDMA_DEVICE_MANA_IB =3D 3, }; =20 +enum gdma_service_type { + GDMA_SERVICE_TYPE_NONE =3D 0, + GDMA_SERVICE_TYPE_RDMA_SUSPEND =3D 1, + GDMA_SERVICE_TYPE_RDMA_RESUME =3D 2, +}; + +struct mana_service_work { + struct work_struct work; + struct gdma_dev *gdma_dev; + enum gdma_service_type event; +}; + struct gdma_resource { /* Protect the bitmap */ spinlock_t lock; @@ -224,6 +237,8 @@ struct gdma_dev { void *driver_data; =20 struct auxiliary_device *adev; + bool is_suspended; + bool rdma_teardown; }; =20 /* MANA_PAGE_SIZE is the DMA unit */ @@ -409,6 +424,8 @@ struct gdma_context { struct gdma_dev mana_ib; =20 u64 pf_cap_flags1; + + struct workqueue_struct *service_wq; }; =20 #define MAX_NUM_GDMA_DEVICES 4 @@ -888,4 +905,6 @@ int mana_gd_destroy_dma_region(struct gdma_context *gc,= u64 dma_region_handle); void mana_register_debugfs(void); void mana_unregister_debugfs(void); =20 +int mana_rdma_service_event(struct gdma_context *gc, enum gdma_service_typ= e event); + #endif /* _GDMA_H */ diff --git a/include/net/mana/hw_channel.h b/include/net/mana/hw_channel.h index 158b125..83cf933 100644 --- a/include/net/mana/hw_channel.h +++ b/include/net/mana/hw_channel.h @@ -49,6 +49,15 @@ union hwc_init_type_data { }; }; /* HW DATA */ =20 +union hwc_init_soc_service_type { + u32 as_uint32; + + struct { + u32 value : 28; + u32 type : 4; + }; +}; /* HW DATA */ + struct hwc_rx_oob { u32 type : 6; u32 eom : 1; --=20 1.8.3.1